# search crawler delay set

User-agent: search.marginalia.nu
Crawl-delay: 0

User-agent: MojeekBot
Crawl-delay: 0

User-agent: Yandex
Crawl-delay: 10

# blocking? non-search crawlers that regularly hit the site

User-agent: SemrushBot
Disallow: /

User-agent: Ahrefs
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: MJ12bot
Crawl-delay: 30

User-agent: MegaIndex.ru
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: ClaudeBot
Crawl-delay: 30

User-agent: Mediapartners-Google
Disallow: /

User-agent: Google-Display-Ads-Bot
Disallow: /

# general rules to follow for all bots

User-agent: *
Crawl-delay: 5
Disallow: /projects/dl/*