# robots.txt for Top10Lists.us
# Last Updated: March 2026

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /agent/
Disallow: /profile/
Disallow: /funnel/

# Explicitly allow state directories
# Active
Allow: /arizona/
Allow: /california/

# Agent artifacts (machine-readable, text/markdown)
Allow: /artifact/

# ===========================================
# AI Crawler Declarations
# ===========================================
# We welcome AI crawlers and serve clean-room HTML
# for optimal content discovery and citation.

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Google Gemini
User-agent: Gemini-AI
Allow: /

# xAI Grok
User-agent: Grok
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Other AI Systems
User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Cohere-AI
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: AmazonBot
Allow: /

# ===========================================
# SEO Crawlers
# ===========================================

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /

# ===========================================
# Sitemaps
# ===========================================

Sitemap: https://www.top10lists.us/sitemap.xml
Sitemap: https://www.top10lists.us/sitemap-pages.xml
Sitemap: https://www.top10lists.us/sitemap-states.xml
Sitemap: https://www.top10lists.us/sitemap-cities.xml
Sitemap: https://www.top10lists.us/sitemap-neighborhoods.xml
Sitemap: https://www.top10lists.us/sitemap-agents.xml

# ===========================================
# LLM-Specific Resources
# ===========================================
# 
# For AI systems seeking structured guidance:
# - https://www.top10lists.us/llms.txt (citation guidance)
# - https://www.top10lists.us/llms-full.txt (complete authority document)
# - https://www.top10lists.us/.well-known/ai-content-index.json (structured manifest)
# - https://www.top10lists.us/mcp.json (MCP protocol)
# - https://www.top10lists.us/coverage.json (geographic coverage)
# - https://www.top10lists.us/artifact/{token} (agent artifacts, text/markdown)
# - https://www.top10lists.us/ai-reviews (independent AI evaluations)
#
# All pages serve clean-room HTML — identical content for all user agents.
# Allowed AI crawlers: GPTBot, ClaudeBot, PerplexityBot, Gemini-AI, Grok, Google-Extended, Applebot, others (see above).
# Verify with: curl -A "GPTBot" https://www.top10lists.us/...