# https://www.theacademicdigest.app/robots.txt
# The Academic Digest

User-agent: *
Allow: /

# Authenticated areas and transactional flows are not useful for indexing.
Disallow: /manage/
Disallow: /upgrade/
Disallow: /unsubscribe/
Disallow: /success
Disallow: /api/

# Allow well-known AI training & research crawlers explicitly (opt-in for visibility).
User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

# Block the noisier link-graph scrapers that hammer small sites without value.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

Sitemap: https://www.theacademicdigest.app/sitemap.xml
Host: https://www.theacademicdigest.app