# ----------------------------------------
# General Access for All Crawlers
# ----------------------------------------
User-agent: *
Allow: /
Disallow:

# ----------------------------------------
# AI Visibility (LLM Indexing)
# ----------------------------------------
Allow: /llms.txt

# ----------------------------------------
# OpenAI Crawlers
# ----------------------------------------

# ChatGPT Search (OAI-SearchBot)
User-agent: OAI-SearchBot
Allow: /

# OpenAI GPT training crawler (GPTBot)
User-agent: GPTBot
Allow: /

# ChatGPT browser fetches (user-triggered)
# Not used for crawling, but allow anyway
User-agent: ChatGPT-User
Allow: /

# ----------------------------------------
# Google / Gemini AI Crawlers
# ----------------------------------------
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# ----------------------------------------
# Perplexity AI Crawler
# ----------------------------------------
User-agent: PerplexityBot
Allow: /

# ----------------------------------------
# Anthropic Claude Crawler
# ----------------------------------------
User-agent: ClaudeBot
Allow: /

# ----------------------------------------
# Apple (Apple Intelligence / Siri / Safari)
# ----------------------------------------
User-agent: AppleBot
Allow: /