{
  "study_id": "ai-crawler-policy-2026-10",
  "kind": "experiment",
  "status": "publish-ready",
  "title": "AI crawler policy, re-measured after a parser fix (October 2026)",
  "measured_at": "2026-10-20",
  "published_at": "2026-10-21",
  "measurement_window": "Latest audit per domain on or after 26 August 2026, measured 20 October 2026",
  "temporal_coverage": "2026-08-26/2026-10-20",
  "canonical_url": "https://searchscore.io/research/ai-crawler-policy-2026-10/",
  "csv_url": "https://searchscore.io/data/research/ai-crawler-policy-2026-10.csv",
  "license": "CC BY 4.0",
  "relationship_to_earlier_study": "The 18 August 2026 study (/research/ai-crawler-policy-2026/) is kept as published. This is a separate, dated measurement.",
  "parser_fix": {
    "date": "2026-08-25",
    "summary": "Rules from a later user-agent group leaked onto earlier agents, so a file that allowed OAI-SearchBot and then disallowed GPTBot read OAI-SearchBot as blocked. A Content-Signal: line also cut the wildcard (*) group off from its rules."
  },
  "population": {
    "domains_with_post_fix_audit": 164635,
    "robots_txt_status": {
      "read": 134858,
      "missing": 24679,
      "not_recorded": 3112,
      "unreachable": 1986
    },
    "all_sites_n": 162649,
    "with_policy_n": 120190,
    "no_policy": 42459,
    "denominator_rule": "Headline rates divide by the 120,190 sites whose robots.txt blocks or allows at least one measured AI crawler, by name or through the wildcard (*) group. The all-sites rates divide by the 162,649 sites whose robots.txt was not unreachable, including sites with no robots.txt, sites with no rule reaching AI crawlers, and 3,112 audits that recorded no robots.txt status (counted from the crawler verdicts those audits did record). All-sites rates describe this post-fix sample only.",
    "geography": "Worldwide",
    "sampling_note": "Post-fix audits only: the latest audit per domain on or after 26 August 2026. These audits come from SearchScore's own scanning campaigns, so the sample is not random and is not the whole SearchScore corpus."
  },
  "crawler_rates": [
    {
      "crawler": "GPTBot",
      "role": "OpenAI model-training crawler",
      "blocked": 12490,
      "rate_pct_with_policy": 10.39,
      "rate_pct_all_sites": 7.68
    },
    {
      "crawler": "ClaudeBot",
      "role": "Anthropic model-development/training crawler",
      "blocked": 11614,
      "rate_pct_with_policy": 9.66,
      "rate_pct_all_sites": 7.14
    },
    {
      "crawler": "Google-Extended",
      "role": "Google robots control token for Gemini training and grounding",
      "blocked": 10942,
      "rate_pct_with_policy": 9.1,
      "rate_pct_all_sites": 6.73
    },
    {
      "crawler": "CCBot",
      "role": "Common Crawl open-web dataset crawler",
      "blocked": 10285,
      "rate_pct_with_policy": 8.56,
      "rate_pct_all_sites": 6.32
    },
    {
      "crawler": "Bytespider",
      "role": "ByteDance crawler; exact public purpose not relied on in the headline",
      "blocked": 10104,
      "rate_pct_with_policy": 8.41,
      "rate_pct_all_sites": 6.21
    },
    {
      "crawler": "PerplexityBot",
      "role": "Perplexity search-indexing crawler",
      "blocked": 7240,
      "rate_pct_with_policy": 6.02,
      "rate_pct_all_sites": 4.45
    },
    {
      "crawler": "OAI-SearchBot",
      "role": "OpenAI search crawler",
      "blocked": 6583,
      "rate_pct_with_policy": 5.48,
      "rate_pct_all_sites": 4.05
    },
    {
      "crawler": "GoogleOther",
      "role": "Google generic crawler used by its product teams; role not relied on in the headline",
      "blocked": 6472,
      "rate_pct_with_policy": 5.38,
      "rate_pct_all_sites": 3.98
    },
    {
      "crawler": "Claude-SearchBot",
      "role": "Anthropic search crawler",
      "blocked": 6260,
      "rate_pct_with_policy": 5.21,
      "rate_pct_all_sites": 3.85
    },
    {
      "crawler": "DeepSeekBot",
      "role": "Other measured crawler token; role not relied on in the headline",
      "blocked": 6220,
      "rate_pct_with_policy": 5.18,
      "rate_pct_all_sites": 3.82
    }
  ],
  "headline_findings": {
    "with_policy_n": 120190,
    "all_sites_n": 162649,
    "gptbot_block_rate_pct": 10.39,
    "oai_searchbot_block_rate_pct": 5.48,
    "gptbot_block_rate_pct_all_sites": 7.68,
    "oai_searchbot_block_rate_pct_all_sites": 4.05,
    "gptbot_blockers": 12490,
    "oai_searchbot_blockers": 6583,
    "gptbot_blockers_allowing_oai_searchbot": 5782,
    "gptbot_blockers_allowing_oai_searchbot_pct": 46.3,
    "oai_searchbot_blockers_allowing_gptbot": 49,
    "allowed_definition": "Allowed means the crawler is not blocked and a rule reaches it, either a group naming it or the wildcard (*) group. It does not mean the site named the crawler.",
    "interpretation": "Within OpenAI's documented pair, the model-training crawler is blocked more often than the search crawler in this sample. Of the sites blocking GPTBot, 46.3% leave OAI-SearchBot allowed, by name or through the wildcard group. The data does not establish why those policies were configured."
  },
  "claim_limits": [
    "Do not compare these figures with the 18 August 2026 study as a trend or change over time: the samples and denominators differ.",
    "Do not generalise the sample to 'the web', or to the whole SearchScore corpus: it is the latest post-fix audit per domain from SearchScore's own scanning campaigns.",
    "Allowed includes crawlers reached only by the wildcard (*) group, so do not describe an allowed crawler as one the site named.",
    "Do not infer publisher intent from robots.txt configuration.",
    "Do not generalise the measured tokens into a binary training-versus-retrieval taxonomy; some have mixed, indirect or insufficiently documented roles.",
    "Google-Extended is a robots control token rather than a standalone crawler user agent."
  ],
  "citation": {
    "journalistic": "SearchScore, AI crawler policy re-measure (October 2026), 120,190 websites with a robots.txt rule reaching AI crawlers, latest audit per domain on or after 26 August 2026, measured 20 October 2026.",
    "academic": "Huss, R. (2026). AI crawler policy, re-measured after a parser fix (October 2026). SearchScore.",
    "attribution": "Source: SearchScore, AI crawler policy re-measure, October 2026"
  }
}