# IMPORTANT — read before editing:
# In production, Cloudflare PREPENDS a "Cloudflare Managed content" block to this
# file (see https://docstreamai.com/robots.txt). That managed block currently sets
# `Disallow: /` for GPTBot, ClaudeBot, Google-Extended, Amazonbot, and others.
# Because a crawler obeys its OWN named user-agent group and ignores `User-agent: *`,
# those per-bot Disallow rules WIN over anything below — appending an `Allow: /`
# here only creates a conflicting duplicate group (resolved inconsistently per crawler).
#
# To actually let ChatGPT, Claude, and Google AI crawl this site you MUST disable or
# edit the Cloudflare AI-bot / "Managed robots.txt" setting in the Cloudflare dashboard
# (Bots / AI Scrapers & Crawlers). Once that managed block is removed, the policy below
# governs. This file documents the intended policy.
#
# Content-Signal grants: search (search indexing) + ai-input (use in AI answers /
# RAG — this is what drives AI visibility) while withholding ai-train (model training).
# Set ai-train=yes if you also want to permit training use.

# ---- Default policy ----
User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

# ---- Search engines ----
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# ---- Social preview / unfurl bots ----
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# ---- OpenAI / ChatGPT ----
User-agent: GPTBot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

User-agent: ChatGPT-User
Allow: /

# ---- Anthropic / Claude ----
User-agent: ClaudeBot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot-User
Allow: /

# ---- Google AI (Gemini / AI Overviews grounding) ----
User-agent: Google-Extended
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

# ---- Perplexity ----
User-agent: PerplexityBot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

User-agent: Perplexity-User
Allow: /

# ---- Training-only / aggressive scrapers: kept blocked (no visibility benefit) ----
# These don't power any major AI search/answer surface. Remove a block to allow it.
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

Sitemap: https://docstreamai.com/sitemap.xml