add entries in robots.txt for new bots

GoteoFoundation · Sep 6, 2024 · a8ef67e · a8ef67e
1 parent 6ccfe8a
commit a8ef67e
Showing 1 changed file with 16 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,4 +1,20 @@
+User-agent: GPTBot
+Disallow: /
+
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: AhrefsBot
+Disallow: /
+
+User-Agent: ImagesiftBot
+Crawl-delay: 60
+
 User-agent: *
 Disallow: /admin
 Disallow: /admin/
 Disallow: /dashboard/
+