User-agent: AhrefsBot
Disallow: /

User-agent: BLEXBot
Disallow: /
# wfarrell 2022-03-25 Disallowed BLEXBot crawler as it uses too many phpBB sessions.

User-agent: Riddler
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: lssbot
Disallow: /

User-agent: librabot
Disallow: /

# Bing also checks for msnbot - cpeel 2020-03-08
# https://blogs.bing.com/webmaster/2012/05/03/to-crawl-or-not-to-crawl-that-is-bingbots-question/
#User-agent: msnbot
#Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: "Wells Search II"
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: NaverBot
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: Vortex
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: discobot
Disallow: /

# SEMrushBot is taking up a lot of bandwidth - cpeel 2020-03-08
User-agent: SemrushBot
Disallow: /

#---------------------------------------------
# Disable AI scraping - cpeel 2023-08-23
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Omgilibot
Disallow: /

User-Agent: FacebookBot
Disallow: /
#---------------------------------------------

User-agent: *
Crawl-delay: 30
Disallow: /feed/
# Allow dp_branding to be crawled as our social media previews use it
Allow: /dp_branding/
Disallow: /c/accounts/
Disallow: /c/crontab/
Disallow: /c/graphics/
Disallow: /c/locale/
Disallow: /c.new/
Disallow: /c.bak/
Disallow: /c/pinc/
Disallow: /c/SETUP/
Disallow: /c/stats/
Disallow: /c/tools/
Disallow: /c/users/
Disallow: /dpmail/
Disallow: /d/
Disallow: /out/
Disallow: /jpgraph/
Disallow: /jpgraph-1.14
Disallow: /archive
Disallow: /projects/
Disallow: /mailman/
Disallow: /mailman3/
Disallow: /noncvs/
Disallow: /phpBB2/
Disallow: /phpBB3/
Disallow: /sawiki/
Disallow: /squirrels/
Disallow: /stats/
Disallow: /tools/
Disallow: /w/
Disallow: /wikiheiro/