# CVE Security: crawler policy
#
# INDEXED CORE (stage 1, gated by INDEX_MAX_STAGE in src/lib/detectable.ts):
# KEV-listed ∪ Nuclei-detectable CVE pages (~5.4k, pipeline-owned via
# cves.seo_index) plus the hub pages — /, /detectable, /scoreboard, /coverage,
# /attack (+ per-technique pages), /api docs, /glossary, /metrics. Every other
# CVE page is served but noindex; the browse/search endpoints below are
# disallowed to conserve crawl budget and keep the indexed footprint focused.
# The sitemap (sitemap.xml) lists exactly this core.
#
# Posture: allow general search engines + AI *citation*
# crawlers (which drive user traffic by linking back); block AI *training*
# crawlers (which take content without returning visitors). Re-confirmed by the
# owner 2026-07-20 when the Content-Signal below was added: search and agent
# retrieval yes, training no — the signal and the per-bot blocks agree.
#
# The AI citation/retrieval groups additionally get the OPEN-DATA machine
# endpoints (scoreboard JSON/CSV, keyless search, the MCP endpoint): those are
# built for agent retrieval (ai-input=yes), and some agent fetchers honour
# robots.txt for user-requested GETs. General crawlers keep the full /api/
# block — JSON endpoints are not search-index material.
#
# Note: Cloudflare may inject its managed "AI Audit" robots.txt at the edge.
# To make this file the source of truth, disable AI Crawl Control in the
# Cloudflare dashboard (Security → Bots).

# Disallowed for ALL crawlers: browse/search/api + orphaned sitemap shards.
# (Placed before the wildcard Allow blocks so specific rules apply per group.)
#
# /kev-watch is open (owner, 2026-09-27). "Disallow: /kev" is a prefix rule and
# had covered /kev-watch as well, while sitemap.xml listed the page; the longer
# "Allow: /kev-watch" now wins for it. The paths under /kev-watch/, /chains/,
# /kcv-watch/ and /detectable/ are the list pages' own row data (JSON), closed
# like /api/. /kev itself stays closed, and with it /kev/rows.json.

User-agent: *
# AI usage policy (contentsignals.org): searchable yes, loadable into an
# agent's context at a user's request yes, model training no.
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /kev
Allow: /kev-watch
Disallow: /kev-watch/
Disallow: /chains/
Disallow: /kcv-watch/
Disallow: /detectable/
Disallow: /epss-top
Disallow: /latest
Disallow: /search
Disallow: /egress
Disallow: /sources
Disallow: /vendor/
Disallow: /malware/
Disallow: /vendors
Disallow: /product/
Disallow: /cwe
Disallow: /api/
Disallow: /sitemaps/
Allow: /

# ─── AI citation / retrieval crawlers (drive traffic via cited links) ────
# Same limited HTML surface as everyone else, PLUS the open-data machine
# endpoints (longest-match Allow beats the /api/ Disallow): the CC BY 4.0
# scoreboard report + per-metric CSVs, keyless structured search, and the
# MCP endpoint. See /llms.txt for how to use them.

User-agent: OAI-SearchBot
Disallow: /kev
Allow: /kev-watch
Disallow: /kev-watch/
Disallow: /chains/
Disallow: /kcv-watch/
Disallow: /detectable/
Disallow: /epss-top
Disallow: /latest
Disallow: /search
Disallow: /sources
Disallow: /vendor/
Disallow: /malware/
Disallow: /vendors
Disallow: /product/
Disallow: /cwe
Disallow: /api/
Allow: /api/scoreboard/
Allow: /api/search
Allow: /api/mcp
Disallow: /sitemaps/
Allow: /

User-agent: ChatGPT-User
Disallow: /kev
Allow: /kev-watch
Disallow: /kev-watch/
Disallow: /chains/
Disallow: /kcv-watch/
Disallow: /detectable/
Disallow: /epss-top
Disallow: /latest
Disallow: /search
Disallow: /sources
Disallow: /vendor/
Disallow: /malware/
Disallow: /vendors
Disallow: /product/
Disallow: /cwe
Disallow: /api/
Allow: /api/scoreboard/
Allow: /api/search
Allow: /api/mcp
Disallow: /sitemaps/
Allow: /

User-agent: PerplexityBot
Disallow: /kev
Allow: /kev-watch
Disallow: /kev-watch/
Disallow: /chains/
Disallow: /kcv-watch/
Disallow: /detectable/
Disallow: /epss-top
Disallow: /latest
Disallow: /search
Disallow: /sources
Disallow: /vendor/
Disallow: /malware/
Disallow: /vendors
Disallow: /product/
Disallow: /cwe
Disallow: /api/
Allow: /api/scoreboard/
Allow: /api/search
Allow: /api/mcp
Disallow: /sitemaps/
Allow: /

User-agent: Perplexity-User
Disallow: /kev
Allow: /kev-watch
Disallow: /kev-watch/
Disallow: /chains/
Disallow: /kcv-watch/
Disallow: /detectable/
Disallow: /epss-top
Disallow: /latest
Disallow: /search
Disallow: /sources
Disallow: /vendor/
Disallow: /malware/
Disallow: /vendors
Disallow: /product/
Disallow: /cwe
Disallow: /api/
Allow: /api/scoreboard/
Allow: /api/search
Allow: /api/mcp
Disallow: /sitemaps/
Allow: /

User-agent: Claude-User
Disallow: /kev
Allow: /kev-watch
Disallow: /kev-watch/
Disallow: /chains/
Disallow: /kcv-watch/
Disallow: /detectable/
Disallow: /epss-top
Disallow: /latest
Disallow: /search
Disallow: /sources
Disallow: /vendor/
Disallow: /malware/
Disallow: /vendors
Disallow: /product/
Disallow: /cwe
Disallow: /api/
Allow: /api/scoreboard/
Allow: /api/search
Allow: /api/mcp
Disallow: /sitemaps/
Allow: /

User-agent: Claude-Web
Disallow: /kev
Allow: /kev-watch
Disallow: /kev-watch/
Disallow: /chains/
Disallow: /kcv-watch/
Disallow: /detectable/
Disallow: /epss-top
Disallow: /latest
Disallow: /search
Disallow: /sources
Disallow: /vendor/
Disallow: /malware/
Disallow: /vendors
Disallow: /product/
Disallow: /cwe
Disallow: /api/
Allow: /api/scoreboard/
Allow: /api/search
Allow: /api/mcp
Disallow: /sitemaps/
Allow: /

# Anthropic's search-index crawler (distinct from ClaudeBot the training
# crawler, which stays blocked below, and from Claude-User above).
User-agent: Claude-SearchBot
Disallow: /kev
Allow: /kev-watch
Disallow: /kev-watch/
Disallow: /chains/
Disallow: /kcv-watch/
Disallow: /detectable/
Disallow: /epss-top
Disallow: /latest
Disallow: /search
Disallow: /sources
Disallow: /vendor/
Disallow: /malware/
Disallow: /vendors
Disallow: /product/
Disallow: /cwe
Disallow: /api/
Allow: /api/scoreboard/
Allow: /api/search
Allow: /api/mcp
Disallow: /sitemaps/
Allow: /

# ─── AI training crawlers (no return traffic) ────────────────────────────

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: FacebookBot
Disallow: /

# ─── Sitemaps ────────────────────────────────────────────────────────────
Sitemap: https://cve-security.com/sitemap.xml