# ==========================================
# robots.txt
# AI Search Visible | AI Training Restricted
# ==========================================

# ------------------------------------------
# OPENAI / CHATGPT
# ------------------------------------------

# Search, citations and user browsing
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Required if running ChatGPT ads
User-agent: OAI-AdsBot
Allow: /

# AI model training
User-agent: GPTBot
Disallow: /

# ------------------------------------------
# GOOGLE
# ------------------------------------------

# Google Search and AI Overviews visibility
User-agent: Googlebot
Allow: /

# Google AI retrieval systems
User-agent: GoogleOther
Allow: /

# Prevent Gemini model training
User-agent: Google-Extended
Disallow: /

# Prevent enterprise agent ingestion
User-agent: Google-CloudVertexBot
Disallow: /

# ------------------------------------------
# MICROSOFT / BING / COPILOT
# ------------------------------------------

# Bing Search and Microsoft Copilot grounding
User-agent: Bingbot
Allow: /

User-agent: AdIdxBot
Allow: /

# Autonomous browsing agents
User-agent: AutoGen-Agent
Disallow: /

# ------------------------------------------
# ANTHROPIC / CLAUDE
# ------------------------------------------

# Claude Search and citations
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

# Foundation model training
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Developer and autonomous agents
User-agent: Claude-Code
Disallow: /

User-agent: Anthropic-ComputerUse
Disallow: /

User-agent: Claude-Researcher
Disallow: /

# ------------------------------------------
# APPLE INTELLIGENCE
# ------------------------------------------

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Disallow: /

# ------------------------------------------
# PERPLEXITY
# ------------------------------------------

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ------------------------------------------
# MISTRAL AI
# ------------------------------------------

User-agent: MistralAI-Index
Allow: /

User-agent: MistralAI-User
Allow: /

# ------------------------------------------
# YOU.COM
# ------------------------------------------

User-agent: YouBot
Allow: /

# ------------------------------------------
# META AI
# ------------------------------------------

# User-requested fetches from WhatsApp,
# Facebook and Instagram AI experiences

User-agent: Meta-ExternalFetcher
Allow: /

# Agentic browsing
User-agent: Meta-ExternalAgent
Disallow: /

# Training crawler
User-agent: FacebookBot
Disallow: /

# ------------------------------------------
# MASS SCRAPERS / TRAINING CRAWLERS
# ------------------------------------------

User-agent: CCBot
Disallow: /

User-agent: ByteSpider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Cohere-ai
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# ------------------------------------------
# TRADITIONAL SEARCH ENGINES
# ------------------------------------------

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: YandexBot
Allow: /

User-agent: baiduspider
Allow: /

# ------------------------------------------
# SOCIAL PREVIEW BOTS
# ------------------------------------------

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

# ------------------------------------------
# DEFAULT POLICY
# ------------------------------------------

# Block all unknown bots unless explicitly approved

User-agent: *
Disallow: /

# Optimizely specific stuff
User-agent: *
Disallow: /ui/
Disallow: /episerver/
Disallow: /utils/

# ------------------------------------------
# SEMRUSH
# ------------------------------------------
 
User-agent: SemrushBot
Allow: /
 
User-agent: SemrushBot-SA
Allow: /
 
User-agent: SemrushBot-BA
Allow: /