{
  "study_id": "ai-crawler-policy-2026",
  "kind": "experiment",
  "status": "publish-ready",
  "title": "19,998 Sites Block GPTBot While Allowing OpenAI Search",
  "measured_at": "2026-08-18",
  "measurement_window": "90 days ending 18 August 2026",
  "canonical_url": "https://searchscore.io/research/ai-crawler-policy-2026/",
  "csv_url": "https://searchscore.io/data/research/ai-crawler-policy-2026.csv",
  "license": "CC BY 4.0",
  "population": {
    "unique_domains": 347988,
    "geography": "Worldwide",
    "sampling_note": "SearchScore audit corpus; not a random sample of the entire web."
  },
  "crawler_rates": [
    {"crawler": "GPTBot", "role": "OpenAI model-training crawler", "blocked": 46984, "rate_pct": 13.50},
    {"crawler": "CCBot", "role": "Common Crawl open-web dataset crawler", "blocked": 46128, "rate_pct": 13.26},
    {"crawler": "Bytespider", "role": "ByteDance crawler; exact public purpose not relied on in the headline", "blocked": 44595, "rate_pct": 12.82},
    {"crawler": "ClaudeBot", "role": "Anthropic model-development/training crawler", "blocked": 44561, "rate_pct": 12.81},
    {"crawler": "Google-Extended", "role": "Google robots control token for Gemini training and grounding", "blocked": 43827, "rate_pct": 12.59},
    {"crawler": "PerplexityBot", "role": "Perplexity search-indexing crawler", "blocked": 28325, "rate_pct": 8.14},
    {"crawler": "OAI-SearchBot", "role": "OpenAI search crawler", "blocked": 27256, "rate_pct": 7.83},
    {"crawler": "DeepSeekBot", "role": "Other measured crawler token; role not relied on in the headline", "blocked": 24914, "rate_pct": 7.16},
    {"crawler": "GrqBot", "role": "Other measured crawler token; role not relied on in the headline", "blocked": 24296, "rate_pct": 6.98}
  ],
  "headline_findings": {
    "gptbot_block_rate_pct": 13.50,
    "oai_searchbot_block_rate_pct": 7.83,
    "gptbot_relative_block_rate_higher_pct": 72.4,
    "gptbot_blockers": 46984,
    "gptbot_blockers_allowing_oai_searchbot": 19998,
    "gptbot_blockers_allowing_oai_searchbot_pct": 42.6,
    "oai_searchbot_blockers_allowing_gptbot": 270,
    "direction_ratio_display": "74 to 1"
  },
  "claim_limits": [
    "Snapshot only; do not present as a change-over-time trend.",
    "Do not generalise the sample to the entire web.",
    "Robots policy does not establish publisher intent.",
    "Do not generalise all nine measured tokens into a binary training-versus-retrieval taxonomy.",
    "Google-Extended is a robots control token rather than a standalone crawler user agent."
  ],
  "correction_record": {
    "previous_claim": "7.8% of sites block GPTBot",
    "corrected_claim": "13.5% of sites in the 347,988-domain study block GPTBot",
    "note": "The earlier 7.8% figure closely matches the measured OAI-SearchBot rate of 7.83%."
  },
  "citation": {
    "journalistic": "SearchScore, AI Crawler Policy Study 2026, analysis of 347,988 websites, measured 18 August 2026.",
    "academic": "Huss, R. (2026). AI Crawler Policy Study 2026. SearchScore.",
    "attribution": "Source: SearchScore, AI Crawler Policy Study 2026"
  }
}