[robots.txt] Block pods with high requests

datopian · Jul 19, 2022 · 01bb5b6 · 01bb5b6
1 parent c8cba20
commit 01bb5b6
Showing 1 changed file with 10 additions and 1 deletion.
diff --git a/public/robots.txt b/public/robots.txt
@@ -13,7 +13,13 @@ Disallow: /
 User-agent: MJ12bot
 Disallow: /
 
-User-agent: Bingbot
+User-agent: MJ12bot/v1.4.8   
+Disallow: /
+
+User-agent: serpstatbot/2.1       
+Disallow: /
+
+User-agent: PetalBot  
 Disallow: /
 
 User-agent: Pingdom
@@ -31,6 +37,9 @@ Disallow: /
 User-agent: dotbot
 Disallow: /
 
+User-agent: DotBot/1.2
+Disallow: /
+
 User-Agent: trendictionbot
 Disallow: /