User-agent: *
Disallow: /test/
Disallow: /terms/
Disallow: /web_log/
Disallow: /__jr/
Disallow: /cgi-bin/__jr/
Disallow: /SDD/
Disallow: /cgi-bin/un.pl
Disallow: /cgi-bin/forums/cancel_notify.pl


User-agent: GPTBot
Crawl-delay: 60

#User-agent: bingbot
#Crawl-delay: 30

User-agent: Slurp
Crawl-delay: 30

User-agent: 008
Crawl-delay: 10

User-agent: Ocelli 
Crawl-delay: 30	

#JR - added to slow down aggressive robot
User-agent: Aghaven/Nutch-1.2 
Crawl-delay: 60	

#JR - added to slow down aggressive robot 04/20/2015
User-agent: Typhoeus 
Crawl-delay: 60	

#JR - added to slow down aggressive robot 05/01/2015
User-agent: Domain Re-Animator Bot 
Crawl-delay: 60	

User-agent: SemrushBot
Disallow: /
User-agent: SiteAuditBot
Disallow: /
User-agent: SemrushBot-BA
Disallow: /

User-agent: omgili/0.5 +http://omgili.com
Disallow: /

User-agent: AhrefsBot 
Crawl-Delay: 15