# Standard crawlers
User-agent: *
Allow: /

# ── AI search / retrieval crawlers ──
# These power live citations in AI chat answers (ChatGPT, Claude, Perplexity).
# Allowed so Glassdart can actually be found and recommended through them.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ── AI training crawlers ──
# Citation/retrieval bots above are welcome; these are dedicated model-training
# crawlers and are disallowed, this site should be citable, not training data.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

# Bytespider is widely reported to ignore robots.txt and to generate
# disproportionate, low-value scraping traffic. Disallowed as a stated
# intent even though it may not be honored, costs nothing to try.
User-agent: Bytespider
Disallow: /

Sitemap: https://glassdart.com/sitemap.xml