# pelles.ai robots.txt
# Preferred host: https://www.pelles.ai

# Default: allow all crawlers
# /_next/static/ must stay crawlable — Google needs the JS/CSS to render the site.
# /thank-you is handled by its own noindex meta tag; a Disallow here would
# prevent crawlers from ever seeing that noindex.
#
# Content signals (contentsignals.org): how our content may be used once it's
# fetched. We want Pelles in search results, quoted in AI answers, and known
# to the models themselves, so all three are yes.
#   search    building a search index and showing links and snippets
#   ai-input  feeding content into AI answers (RAG, grounding, AI Overviews)
#   ai-train  training or fine-tuning AI models
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Search and AI crawlers, allowed by name so the intent is on record for GEO.
# A crawler that finds its own name reads only that group and ignores `*`,
# which is why they share one group that repeats the content signal.
User-agent: Googlebot             # Google Search, AI Overviews, AI Mode
User-agent: Google-Extended       # Gemini training and grounding
User-agent: Bingbot               # Bing, Copilot; ChatGPT search also draws on Bing
User-agent: GPTBot                # OpenAI training
User-agent: OAI-SearchBot         # ChatGPT search index
User-agent: ChatGPT-User          # ChatGPT fetching a page for a user
User-agent: ClaudeBot             # Anthropic training
User-agent: Claude-SearchBot      # Claude search index
User-agent: Claude-User           # Claude fetching a page for a user (the homepage's "ask Claude" link)
User-agent: PerplexityBot         # Perplexity search index
User-agent: Perplexity-User       # Perplexity fetching a page for a user
User-agent: Applebot              # Siri and Spotlight search
User-agent: Applebot-Extended     # Apple Intelligence training
User-agent: Amazonbot             # Alexa+ and Amazon AI features
User-agent: Meta-ExternalAgent    # Meta AI training and search
User-agent: Meta-ExternalFetcher  # Meta AI fetching a page for a user
User-agent: DuckAssistBot         # DuckDuckGo AI answers
User-agent: MistralAI-User        # Mistral Le Chat fetching a page for a user
User-agent: Bytespider            # ByteDance (Doubao)
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

Sitemap: https://www.pelles.ai/sitemap.xml

# LLM context files (not robots directives — discovered at their URLs):
# https://www.pelles.ai/llms.txt
# https://www.pelles.ai/llms-full.txt
# Every page as Markdown: the page URL plus .md (https://www.pelles.ai/index.md for the homepage)