# BriteSprout crawler policy. The reader-facing version is /terms/.
#
# Search engines are welcome. Pages that should not be indexed carry a
# noindex meta tag instead of a Disallow rule here, so that a crawler can
# read the tag rather than guess from an absent page.

User-agent: *
Allow: /

# Search and citation crawlers, named so that their absence from the list
# below reads as a decision. They send readers to the site.

# OpenAI: surfaces pages in ChatGPT search
User-agent: OAI-SearchBot
Allow: /

# OpenAI: fetches a page a person asked for
User-agent: ChatGPT-User
Allow: /

# Anthropic: improves search results
User-agent: Claude-SearchBot
Allow: /

# Anthropic: fetches a page a person asked for
User-agent: Claude-User
Allow: /

# Perplexity: indexes pages to link to them
User-agent: PerplexityBot
Allow: /

# Perplexity: fetches a page a person asked for
User-agent: Perplexity-User
Allow: /

# Crawlers documented by their operators as collecting model training data.
# BriteSprout content may not be used to train or improve a machine learning
# model without permission. See /terms/.

# OpenAI: trains foundation models
User-agent: GPTBot
Disallow: /

# Anthropic: collects content for model training
User-agent: ClaudeBot
Disallow: /

# Google: Gemini training; no effect on Google Search
User-agent: Google-Extended
Disallow: /

# Apple: Apple Intelligence training; no effect on Applebot indexing
User-agent: Applebot-Extended
Disallow: /

# Meta: trains foundation models
User-agent: meta-externalagent
Disallow: /

# ByteDance: collects training data
User-agent: Bytespider
Disallow: /

# Common Crawl: builds a corpus widely used for training
User-agent: CCBot
Disallow: /

Sitemap: https://britesprout.com/sitemap.xml