# Search engines and AI answer engines that link back to us are welcome.
# AI model-training crawlers and third-party SEO/backlink crawlers send no
# visitors, so they may not crawl (2026-10-16). Google-Extended and
# Applebot-Extended opt out of AI training without affecting search.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: Diffbot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: PetalBot
# SEO content tool that crawled the whole sitemap unannounced (2026-10-19).
User-agent: QlyzeBot
# Alibaba Cloud crawler that took 4,492 pages in two hours (2026-10-26).
User-agent: TokenFoundryBot
Disallow: /

User-agent: *
Allow: /
Disallow: /api/

Sitemap: https://thedialatlas.com/sitemap.xml