# This file used to not exist at this host: /robots.txt fell through to the
# SPA's index.html (via vercel.json's catch-all rewrite), and Cloudflare's
# "Managed robots.txt" feature was gluing its own AI-crawler rules onto that
# HTML document. A real static file here is served ahead of the rewrite
# (Vercel serves anything under public/ before falling back to the SPA), so
# this is now the actual policy, not a side effect of what Cloudflare
# happened to inject.
#
# Policy: AI systems fetching a page to answer a specific user's question
# (search/citation) are welcome. Bulk crawlers whose only purpose is
# harvesting training data are not. See https://contentsignals.org

User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

# --- AI: citation and live-reference crawlers (allowed) ---
# These fetch a page on behalf of a specific user's live query, the same
# way a person would visit the link. No bot-specific rule needed: they're
# covered by the default Allow above and listed here only for clarity.
# OAI-SearchBot, ChatGPT-User (OpenAI), Claude-User, Claude-SearchBot
# (Anthropic), Perplexity-User, PerplexityBot (Perplexity).

# --- AI: bulk training crawlers (blocked) ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

Sitemap: https://icallagent.com/sitemap.xml