# Content-Signal declares preferences for how this content may be used.
# Spec: https://contentsignals.org/  draft: draft-romm-aipref-contentsignals
#   search=yes    — index and surface this content in search results
#   ai-input=yes  — inference-time agents (RAG, citation, Q&A) may use this content
#   ai-train=no   — do not use this content for AI training corpora
# Per the draft, Content-Signal lines are scoped to the preceding User-agent group.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Inference-time AI agents and assistants — explicitly welcome.
# These bots fetch pages on behalf of a user asking a question, not for training.
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: YouBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Training-time crawlers — paid research JSON is excluded per PRD §7.5
# (do not surface premium memo/fundamental bodies in Common Crawl → LLM training).
# Public pages and the free primer remain crawlable, but Content-Signal
# explicitly forbids using them for model training.
User-agent: CCBot
Content-Signal: search=yes, ai-input=no, ai-train=no
Disallow: /api/v1/research/
Allow: /

# Discovery — pointers for inference-time agents to find the machine surfaces.
Sitemap: https://marginofinsight.com/sitemap.xml
OpenAPI: https://marginofinsight.com/openapi.json
# llms.txt: https://marginofinsight.com/llms.txt