# Recommended.app robots.txt

# Content Signals (https://contentsignals.org)
Content-Signal: ai-train=no, search=yes, ai-input=yes

# AI Search Engine Crawlers — ALLOWED
# These bots power AI-generated answers in ChatGPT, Perplexity,
# Google Gemini, Claude, Copilot, and other AI search products.

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /dashboard
Disallow: /auth
Disallow: /api

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: DeepSeekBot
Allow: /

User-agent: GrokBot
Allow: /

User-agent: CopilotBot
Allow: /

# Traditional search engine crawlers — ALLOWED
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: YandexBot
Allow: /

# SEO tools we pay for — ALLOWED
# AhrefsBot powers our own Site Audit verification, backlink monitoring,
# IndexNow verification, and content snapshots. Blocking it breaks our
# paid Ahrefs subscription. Do not re-add to the blocked list.
User-agent: AhrefsBot
Allow: /
Disallow: /admin
Disallow: /dashboard
Disallow: /auth
Disallow: /api
Disallow: /merchant

User-agent: AhrefsSiteAudit
Allow: /

# Blocked bots — competitor SEO tools we don't use + AI training scrapers
# Reconsider unblocking if we ever subscribe to these tools ourselves.
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Default rule for all other bots
User-agent: *
Allow: /
Disallow: /admin
Disallow: /dashboard
Disallow: /auth
Disallow: /api
Disallow: /merchant

# Sitemaps
Sitemap: https://www.recommended.app/sitemap.xml
Sitemap: https://www.recommended.app/sitemaps/core.xml
Sitemap: https://www.recommended.app/sitemaps/blog.xml
Sitemap: https://www.recommended.app/sitemaps/cities.xml
Sitemap: https://www.recommended.app/sitemaps/compare.xml
Sitemap: https://www.recommended.app/sitemaps/guides.xml
Sitemap: https://www.recommended.app/sitemaps/explore.xml
Sitemap: https://www.recommended.app/sitemaps/join.xml
Sitemap: https://www.recommended.app/feed.xml