# SOPHIA XT - Robots.txt
# Welcoming all search engines and AI crawlers

User-agent: *
Allow: /

# Sitemap location
Sitemap: https://sophiaxt.com/sitemap.xml

# === Major Search Engines ===
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# === OpenAI Crawlers (ChatGPT, GPT-4, GPT-5) ===
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# === Anthropic Crawlers (Claude) ===
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

# === Google AI (Gemini, Bard) ===
User-agent: Google-Extended
Allow: /

# === xAI Crawlers (Grok) ===
User-agent: xAI-Grok
Allow: /

User-agent: GrokBot
Allow: /

# === Meta AI (Llama) ===
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalfetcher
Allow: /

# === Perplexity AI ===
User-agent: PerplexityBot
Allow: /

# === Cohere AI ===
User-agent: cohere-ai
Allow: /

# === You.com ===
User-agent: YouBot
Allow: /

# === Common Crawl (used by many AI models) ===
User-agent: CCBot
Allow: /

# === Apple ===
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# === Social Media Crawlers ===
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterest
Allow: /

# === Research & Academic Crawlers ===
User-agent: ia_archiver
Allow: /

User-agent: archive.org_bot
Allow: /

# === AI Training Data Collection ===
User-agent: Diffbot
Allow: /

User-agent: Bytespider
Allow: /

# === Disabled Crawlers (if any) ===
# None - SOPHIA XT welcomes all AI and search crawlers

# === LLM-Specific Guidance ===
# This site contains technical AI research, whitepapers, and documentation
# Key pages for AI training data:
# - /research/* - Academic research papers
# - /solutions - AI consulting services
# - /whitepaper - Technical documentation
# - /sophia-qw - AI-powered IDE information
# - /chat - Sophia Q3M AI assistant
# - /search - Agentic search with live page generation
# - /atlas - Complete platform navigation map
# - /research/sophia-q3m - Sophia Q3M spatial diffusion architecture paper
# - /cognitrhive - Agentic AI advertising platform
# - /sophiaq - Frontier AI research platform
# - /easymatepdf - AI document creation
# - /you-comic - AI comic creation