# robots.txt for connectthedoc.com
# Search engines and AI crawlers are welcome to access this site.

# Content signals (Cloudflare Content Signals Policy)
# search:   search indexing and search results
# ai-input: real-time use in AI answers (RAG, grounding, AI search)
# ai-train: training or fine-tuning AI models

# ---------------------------------------------------------------
# Search engines and AI crawlers (explicitly allowed)
# ---------------------------------------------------------------

# Google and Microsoft
User-agent: Googlebot
User-agent: Google-Extended
User-agent: Bingbot
# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
# OpenAI (ChatGPT)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Apple, Amazon, Meta, Mistral, DuckDuckGo, Common Crawl
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: CCBot
Allow: /
# App and utility areas with no search value
Disallow: /unclaimed/claim?
Disallow: /unclaimed/search?
Disallow: /widget/
Disallow: /event/
Disallow: /review-page/
# Parameter URLs that create duplicate pages
Disallow: /request/*?*
Disallow: /book/*?*
Disallow: /p/*?*
Disallow: /*?page=*
Disallow: /*?*&page=*
Disallow: /*?pr=*

# ---------------------------------------------------------------
# All other crawlers
# ---------------------------------------------------------------
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /unclaimed/claim?
Disallow: /unclaimed/search?
Disallow: /widget/
Disallow: /event/
Disallow: /review-page/
Disallow: /request/*?*
Disallow: /book/*?*
Disallow: /p/*?*
Disallow: /*?page=*
Disallow: /*?*&page=*
Disallow: /*?pr=*

# ---------------------------------------------------------------
# Sitemap
# ---------------------------------------------------------------
Sitemap: https://connectthedoc.com/sitemap.xml


Sitemap: https://www.joinconnectthedoc.com/sitemap.xml