{
 "sites_in_file": 20000,
 "fetched_from": "2026-11-08T14:56:12.356Z",
 "fetched_to": "2026-11-08T15:09:32.247Z",
 "tiers": {
  "top_1k": {
   "scanned": 1000,
   "reachable": 699,
   "valid_robots": 511,
   "empty_robots": 4,
   "star_blocks_all": 51,
   "llms_present": 130,
   "llms_h1": 121,
   "content_signal": 20,
   "usage_ai_training": 9,
   "tokens": {
    "GPTBot": {
     "named": 138,
     "named_block": 92,
     "named_partial": 27,
     "named_allow": 19,
     "star_only": 35,
     "blocked_pct": 24.85,
     "named_block_pct": 18.0
    },
    "OAI-SearchBot": {
     "named": 90,
     "named_block": 37,
     "named_partial": 38,
     "named_allow": 15,
     "star_only": 36,
     "blocked_pct": 14.29,
     "named_block_pct": 7.24
    },
    "ChatGPT-User": {
     "named": 102,
     "named_block": 49,
     "named_partial": 36,
     "named_allow": 17,
     "star_only": 36,
     "blocked_pct": 16.63,
     "named_block_pct": 9.59
    },
    "ClaudeBot": {
     "named": 133,
     "named_block": 91,
     "named_partial": 25,
     "named_allow": 17,
     "star_only": 37,
     "blocked_pct": 25.05,
     "named_block_pct": 17.81
    },
    "Claude-SearchBot": {
     "named": 76,
     "named_block": 43,
     "named_partial": 28,
     "named_allow": 5,
     "star_only": 40,
     "blocked_pct": 16.24,
     "named_block_pct": 8.41
    },
    "Claude-User": {
     "named": 75,
     "named_block": 47,
     "named_partial": 23,
     "named_allow": 5,
     "star_only": 39,
     "blocked_pct": 16.83,
     "named_block_pct": 9.2
    },
    "PerplexityBot": {
     "named": 122,
     "named_block": 68,
     "named_partial": 35,
     "named_allow": 19,
     "star_only": 34,
     "blocked_pct": 19.96,
     "named_block_pct": 13.31
    },
    "Perplexity-User": {
     "named": 67,
     "named_block": 45,
     "named_partial": 15,
     "named_allow": 7,
     "star_only": 46,
     "blocked_pct": 17.81,
     "named_block_pct": 8.81
    },
    "Google-Extended": {
     "named": 117,
     "named_block": 77,
     "named_partial": 22,
     "named_allow": 18,
     "star_only": 35,
     "blocked_pct": 21.92,
     "named_block_pct": 15.07
    },
    "Applebot-Extended": {
     "named": 91,
     "named_block": 71,
     "named_partial": 13,
     "named_allow": 7,
     "star_only": 39,
     "blocked_pct": 21.53,
     "named_block_pct": 13.89
    },
    "Meta-ExternalAgent": {
     "named": 92,
     "named_block": 76,
     "named_partial": 14,
     "named_allow": 2,
     "star_only": 42,
     "blocked_pct": 23.09,
     "named_block_pct": 14.87
    },
    "Meta-WebIndexer": {
     "named": 32,
     "named_block": 26,
     "named_partial": 5,
     "named_allow": 1,
     "star_only": 48,
     "blocked_pct": 14.48,
     "named_block_pct": 5.09
    },
    "Meta-ExternalFetcher": {
     "named": 47,
     "named_block": 40,
     "named_partial": 7,
     "named_allow": 0,
     "star_only": 44,
     "blocked_pct": 16.44,
     "named_block_pct": 7.83
    },
    "Amazonbot": {
     "named": 78,
     "named_block": 62,
     "named_partial": 12,
     "named_allow": 4,
     "star_only": 41,
     "blocked_pct": 20.16,
     "named_block_pct": 12.13
    },
    "Amzn-SearchBot": {
     "named": 15,
     "named_block": 10,
     "named_partial": 2,
     "named_allow": 3,
     "star_only": 50,
     "blocked_pct": 11.74,
     "named_block_pct": 1.96
    },
    "Amzn-User": {
     "named": 13,
     "named_block": 8,
     "named_partial": 3,
     "named_allow": 2,
     "star_only": 49,
     "blocked_pct": 11.15,
     "named_block_pct": 1.57
    },
    "Bytespider": {
     "named": 113,
     "named_block": 100,
     "named_partial": 11,
     "named_allow": 2,
     "star_only": 42,
     "blocked_pct": 27.79,
     "named_block_pct": 19.57
    },
    "CCBot": {
     "named": 121,
     "named_block": 97,
     "named_partial": 16,
     "named_allow": 8,
     "star_only": 42,
     "blocked_pct": 27.2,
     "named_block_pct": 18.98
    },
    "MistralAI-Training": {
     "named": 6,
     "named_block": 6,
     "named_partial": 0,
     "named_allow": 0,
     "star_only": 51,
     "blocked_pct": 11.15,
     "named_block_pct": 1.17
    },
    "MistralAI-Index": {
     "named": 2,
     "named_block": 2,
     "named_partial": 0,
     "named_allow": 0,
     "star_only": 51,
     "blocked_pct": 10.37,
     "named_block_pct": 0.39
    },
    "MistralAI-User": {
     "named": 41,
     "named_block": 35,
     "named_partial": 4,
     "named_allow": 2,
     "star_only": 47,
     "blocked_pct": 16.05,
     "named_block_pct": 6.85
    },
    "DuckAssistBot": {
     "named": 60,
     "named_block": 43,
     "named_partial": 16,
     "named_allow": 1,
     "star_only": 42,
     "blocked_pct": 16.63,
     "named_block_pct": 8.41
    },
    "Googlebot": {
     "named": 83,
     "named_block": 5,
     "named_partial": 66,
     "named_allow": 12,
     "star_only": 21,
     "blocked_pct": 5.09,
     "named_block_pct": 0.98
    },
    "Bingbot": {
     "named": 66,
     "named_block": 6,
     "named_partial": 50,
     "named_allow": 10,
     "star_only": 26,
     "blocked_pct": 6.26,
     "named_block_pct": 1.17
    },
    "Applebot": {
     "named": 40,
     "named_block": 8,
     "named_partial": 26,
     "named_allow": 6,
     "star_only": 32,
     "blocked_pct": 7.83,
     "named_block_pct": 1.57
    },
    "anthropic-ai": {
     "named": 71,
     "named_block": 53,
     "named_partial": 9,
     "named_allow": 9,
     "star_only": 46,
     "blocked_pct": 19.37,
     "named_block_pct": 10.37
    },
    "Claude-Web": {
     "named": 65,
     "named_block": 56,
     "named_partial": 5,
     "named_allow": 4,
     "star_only": 46,
     "blocked_pct": 19.96,
     "named_block_pct": 10.96
    },
    "FacebookBot": {
     "named": 58,
     "named_block": 48,
     "named_partial": 8,
     "named_allow": 2,
     "star_only": 44,
     "blocked_pct": 18.0,
     "named_block_pct": 9.39
    },
    "cohere-ai": {
     "named": 77,
     "named_block": 66,
     "named_partial": 9,
     "named_allow": 2,
     "star_only": 46,
     "blocked_pct": 21.92,
     "named_block_pct": 12.92
    },
    "Google-NotebookLM": {
     "named": 22,
     "named_block": 18,
     "named_partial": 4,
     "named_allow": 0,
     "star_only": 47,
     "blocked_pct": 12.72,
     "named_block_pct": 3.52
    },
    "GrokBot": {
     "named": 21,
     "named_block": 18,
     "named_partial": 3,
     "named_allow": 0,
     "star_only": 50,
     "blocked_pct": 13.31,
     "named_block_pct": 3.52
    },
    "xAI-Grok": {
     "named": 19,
     "named_block": 17,
     "named_partial": 2,
     "named_allow": 0,
     "star_only": 51,
     "blocked_pct": 13.31,
     "named_block_pct": 3.33
    },
    "DeepSeekBot": {
     "named": 37,
     "named_block": 32,
     "named_partial": 3,
     "named_allow": 2,
     "star_only": 47,
     "blocked_pct": 15.46,
     "named_block_pct": 6.26
    }
   },
   "any_ai_named": 191,
   "any_ai_block": 140,
   "pairs": {
    "openai": {
     "train_block": 92,
     "train_only": 52,
     "train_and_search": 40,
     "search_only_backwards": 0,
     "user_block": 49
    },
    "anthropic": {
     "train_block": 91,
     "train_only": 43,
     "train_and_search": 48,
     "search_only_backwards": 0,
     "user_block": 47
    }
   },
   "stale_anthropic": 25,
   "names_anthropic_legacy": 91,
   "blocks_claudebot_not_new": 39
  },
  "top_10k": {
   "scanned": 10000,
   "reachable": 7307,
   "valid_robots": 5472,
   "empty_robots": 97,
   "star_blocks_all": 271,
   "llms_present": 1034,
   "llms_h1": 960,
   "content_signal": 347,
   "usage_ai_training": 17,
   "tokens": {
    "GPTBot": {
     "named": 1185,
     "named_block": 845,
     "named_partial": 200,
     "named_allow": 140,
     "star_only": 222,
     "blocked_pct": 19.5,
     "named_block_pct": 15.44
    },
    "OAI-SearchBot": {
     "named": 604,
     "named_block": 252,
     "named_partial": 200,
     "named_allow": 152,
     "star_only": 223,
     "blocked_pct": 8.68,
     "named_block_pct": 4.61
    },
    "ChatGPT-User": {
     "named": 698,
     "named_block": 366,
     "named_partial": 189,
     "named_allow": 143,
     "star_only": 221,
     "blocked_pct": 10.73,
     "named_block_pct": 6.69
    },
    "ClaudeBot": {
     "named": 1048,
     "named_block": 780,
     "named_partial": 152,
     "named_allow": 116,
     "star_only": 237,
     "blocked_pct": 18.59,
     "named_block_pct": 14.25
    },
    "Claude-SearchBot": {
     "named": 397,
     "named_block": 227,
     "named_partial": 111,
     "named_allow": 59,
     "star_only": 238,
     "blocked_pct": 8.5,
     "named_block_pct": 4.15
    },
    "Claude-User": {
     "named": 411,
     "named_block": 251,
     "named_partial": 96,
     "named_allow": 64,
     "star_only": 237,
     "blocked_pct": 8.92,
     "named_block_pct": 4.59
    },
    "PerplexityBot": {
     "named": 750,
     "named_block": 434,
     "named_partial": 169,
     "named_allow": 147,
     "star_only": 226,
     "blocked_pct": 12.06,
     "named_block_pct": 7.93
    },
    "Perplexity-User": {
     "named": 401,
     "named_block": 255,
     "named_partial": 92,
     "named_allow": 54,
     "star_only": 248,
     "blocked_pct": 9.19,
     "named_block_pct": 4.66
    },
    "Google-Extended": {
     "named": 978,
     "named_block": 694,
     "named_partial": 144,
     "named_allow": 140,
     "star_only": 230,
     "blocked_pct": 16.89,
     "named_block_pct": 12.68
    },
    "Applebot-Extended": {
     "named": 776,
     "named_block": 652,
     "named_partial": 77,
     "named_allow": 47,
     "star_only": 243,
     "blocked_pct": 16.36,
     "named_block_pct": 11.92
    },
    "Meta-ExternalAgent": {
     "named": 817,
     "named_block": 696,
     "named_partial": 84,
     "named_allow": 37,
     "star_only": 244,
     "blocked_pct": 17.18,
     "named_block_pct": 12.72
    },
    "Meta-WebIndexer": {
     "named": 192,
     "named_block": 164,
     "named_partial": 25,
     "named_allow": 3,
     "star_only": 262,
     "blocked_pct": 7.79,
     "named_block_pct": 3.0
    },
    "Meta-ExternalFetcher": {
     "named": 310,
     "named_block": 266,
     "named_partial": 28,
     "named_allow": 16,
     "star_only": 254,
     "blocked_pct": 9.5,
     "named_block_pct": 4.86
    },
    "Amazonbot": {
     "named": 778,
     "named_block": 653,
     "named_partial": 84,
     "named_allow": 41,
     "star_only": 242,
     "blocked_pct": 16.36,
     "named_block_pct": 11.93
    },
    "Amzn-SearchBot": {
     "named": 120,
     "named_block": 103,
     "named_partial": 11,
     "named_allow": 6,
     "star_only": 262,
     "blocked_pct": 6.67,
     "named_block_pct": 1.88
    },
    "Amzn-User": {
     "named": 124,
     "named_block": 110,
     "named_partial": 8,
     "named_allow": 6,
     "star_only": 264,
     "blocked_pct": 6.83,
     "named_block_pct": 2.01
    },
    "Bytespider": {
     "named": 955,
     "named_block": 868,
     "named_partial": 63,
     "named_allow": 24,
     "star_only": 246,
     "blocked_pct": 20.36,
     "named_block_pct": 15.86
    },
    "CCBot": {
     "named": 1036,
     "named_block": 879,
     "named_partial": 105,
     "named_allow": 52,
     "star_only": 241,
     "blocked_pct": 20.47,
     "named_block_pct": 16.06
    },
    "MistralAI-Training": {
     "named": 214,
     "named_block": 214,
     "named_partial": 0,
     "named_allow": 0,
     "star_only": 269,
     "blocked_pct": 8.83,
     "named_block_pct": 3.91
    },
    "MistralAI-Index": {
     "named": 50,
     "named_block": 47,
     "named_partial": 3,
     "named_allow": 0,
     "star_only": 269,
     "blocked_pct": 5.77,
     "named_block_pct": 0.86
    },
    "MistralAI-User": {
     "named": 253,
     "named_block": 206,
     "named_partial": 32,
     "named_allow": 15,
     "star_only": 259,
     "blocked_pct": 8.5,
     "named_block_pct": 3.76
    },
    "DuckAssistBot": {
     "named": 324,
     "named_block": 249,
     "named_partial": 59,
     "named_allow": 16,
     "star_only": 249,
     "blocked_pct": 9.1,
     "named_block_pct": 4.55
    },
    "Googlebot": {
     "named": 530,
     "named_block": 11,
     "named_partial": 365,
     "named_allow": 154,
     "star_only": 164,
     "blocked_pct": 3.2,
     "named_block_pct": 0.2
    },
    "Bingbot": {
     "named": 407,
     "named_block": 45,
     "named_partial": 240,
     "named_allow": 122,
     "star_only": 186,
     "blocked_pct": 4.22,
     "named_block_pct": 0.82
    },
    "Applebot": {
     "named": 278,
     "named_block": 99,
     "named_partial": 112,
     "named_allow": 67,
     "star_only": 216,
     "blocked_pct": 5.76,
     "named_block_pct": 1.81
    },
    "anthropic-ai": {
     "named": 751,
     "named_block": 639,
     "named_partial": 65,
     "named_allow": 47,
     "star_only": 250,
     "blocked_pct": 16.25,
     "named_block_pct": 11.68
    },
    "Claude-Web": {
     "named": 626,
     "named_block": 535,
     "named_partial": 49,
     "named_allow": 42,
     "star_only": 254,
     "blocked_pct": 14.42,
     "named_block_pct": 9.78
    },
    "FacebookBot": {
     "named": 445,
     "named_block": 374,
     "named_partial": 48,
     "named_allow": 23,
     "star_only": 249,
     "blocked_pct": 11.39,
     "named_block_pct": 6.83
    },
    "cohere-ai": {
     "named": 710,
     "named_block": 638,
     "named_partial": 51,
     "named_allow": 21,
     "star_only": 251,
     "blocked_pct": 16.25,
     "named_block_pct": 11.66
    },
    "Google-NotebookLM": {
     "named": 103,
     "named_block": 84,
     "named_partial": 14,
     "named_allow": 5,
     "star_only": 261,
     "blocked_pct": 6.3,
     "named_block_pct": 1.54
    },
    "GrokBot": {
     "named": 70,
     "named_block": 65,
     "named_partial": 4,
     "named_allow": 1,
     "star_only": 266,
     "blocked_pct": 6.05,
     "named_block_pct": 1.19
    },
    "xAI-Grok": {
     "named": 51,
     "named_block": 48,
     "named_partial": 2,
     "named_allow": 1,
     "star_only": 269,
     "blocked_pct": 5.79,
     "named_block_pct": 0.88
    },
    "DeepSeekBot": {
     "named": 191,
     "named_block": 174,
     "named_partial": 11,
     "named_allow": 6,
     "star_only": 262,
     "blocked_pct": 7.97,
     "named_block_pct": 3.18
    }
   },
   "any_ai_named": 1548,
   "any_ai_block": 1177,
   "pairs": {
    "openai": {
     "train_block": 845,
     "train_only": 586,
     "train_and_search": 259,
     "search_only_backwards": 0,
     "user_block": 366
    },
    "anthropic": {
     "train_block": 780,
     "train_only": 545,
     "train_and_search": 235,
     "search_only_backwards": 0,
     "user_block": 251
    }
   },
   "stale_anthropic": 185,
   "names_anthropic_legacy": 804,
   "blocks_claudebot_not_new": 503
  },
  "top_20k": {
   "scanned": 20000,
   "reachable": 14671,
   "valid_robots": 10870,
   "empty_robots": 186,
   "star_blocks_all": 507,
   "llms_present": 2053,
   "llms_h1": 1902,
   "content_signal": 762,
   "usage_ai_training": 22,
   "tokens": {
    "GPTBot": {
     "named": 2301,
     "named_block": 1647,
     "named_partial": 371,
     "named_allow": 283,
     "star_only": 435,
     "blocked_pct": 19.15,
     "named_block_pct": 15.15
    },
    "OAI-SearchBot": {
     "named": 1053,
     "named_block": 416,
     "named_partial": 351,
     "named_allow": 286,
     "star_only": 436,
     "blocked_pct": 7.84,
     "named_block_pct": 3.83
    },
    "ChatGPT-User": {
     "named": 1274,
     "named_block": 634,
     "named_partial": 349,
     "named_allow": 291,
     "star_only": 434,
     "blocked_pct": 9.83,
     "named_block_pct": 5.83
    },
    "ClaudeBot": {
     "named": 2055,
     "named_block": 1509,
     "named_partial": 304,
     "named_allow": 242,
     "star_only": 456,
     "blocked_pct": 18.08,
     "named_block_pct": 13.88
    },
    "Claude-SearchBot": {
     "named": 695,
     "named_block": 362,
     "named_partial": 208,
     "named_allow": 125,
     "star_only": 455,
     "blocked_pct": 7.52,
     "named_block_pct": 3.33
    },
    "Claude-User": {
     "named": 716,
     "named_block": 396,
     "named_partial": 185,
     "named_allow": 135,
     "star_only": 457,
     "blocked_pct": 7.85,
     "named_block_pct": 3.64
    },
    "PerplexityBot": {
     "named": 1323,
     "named_block": 696,
     "named_partial": 329,
     "named_allow": 298,
     "star_only": 442,
     "blocked_pct": 10.47,
     "named_block_pct": 6.4
    },
    "Perplexity-User": {
     "named": 702,
     "named_block": 401,
     "named_partial": 176,
     "named_allow": 125,
     "star_only": 469,
     "blocked_pct": 8.0,
     "named_block_pct": 3.69
    },
    "Google-Extended": {
     "named": 1868,
     "named_block": 1323,
     "named_partial": 271,
     "named_allow": 274,
     "star_only": 443,
     "blocked_pct": 16.25,
     "named_block_pct": 12.17
    },
    "Applebot-Extended": {
     "named": 1541,
     "named_block": 1277,
     "named_partial": 155,
     "named_allow": 109,
     "star_only": 467,
     "blocked_pct": 16.04,
     "named_block_pct": 11.75
    },
    "Meta-ExternalAgent": {
     "named": 1612,
     "named_block": 1338,
     "named_partial": 183,
     "named_allow": 91,
     "star_only": 467,
     "blocked_pct": 16.61,
     "named_block_pct": 12.31
    },
    "Meta-WebIndexer": {
     "named": 334,
     "named_block": 270,
     "named_partial": 49,
     "named_allow": 15,
     "star_only": 495,
     "blocked_pct": 7.04,
     "named_block_pct": 2.48
    },
    "Meta-ExternalFetcher": {
     "named": 524,
     "named_block": 421,
     "named_partial": 59,
     "named_allow": 44,
     "star_only": 485,
     "blocked_pct": 8.33,
     "named_block_pct": 3.87
    },
    "Amazonbot": {
     "named": 1560,
     "named_block": 1288,
     "named_partial": 179,
     "named_allow": 93,
     "star_only": 464,
     "blocked_pct": 16.12,
     "named_block_pct": 11.85
    },
    "Amzn-SearchBot": {
     "named": 213,
     "named_block": 182,
     "named_partial": 24,
     "named_allow": 7,
     "star_only": 497,
     "blocked_pct": 6.25,
     "named_block_pct": 1.67
    },
    "Amzn-User": {
     "named": 208,
     "named_block": 182,
     "named_partial": 18,
     "named_allow": 8,
     "star_only": 499,
     "blocked_pct": 6.26,
     "named_block_pct": 1.67
    },
    "Bytespider": {
     "named": 1821,
     "named_block": 1637,
     "named_partial": 123,
     "named_allow": 61,
     "star_only": 467,
     "blocked_pct": 19.36,
     "named_block_pct": 15.06
    },
    "CCBot": {
     "named": 2010,
     "named_block": 1677,
     "named_partial": 212,
     "named_allow": 121,
     "star_only": 462,
     "blocked_pct": 19.68,
     "named_block_pct": 15.43
    },
    "MistralAI-Training": {
     "named": 517,
     "named_block": 514,
     "named_partial": 2,
     "named_allow": 1,
     "star_only": 505,
     "blocked_pct": 9.37,
     "named_block_pct": 4.73
    },
    "MistralAI-Index": {
     "named": 89,
     "named_block": 76,
     "named_partial": 11,
     "named_allow": 2,
     "star_only": 504,
     "blocked_pct": 5.34,
     "named_block_pct": 0.7
    },
    "MistralAI-User": {
     "named": 432,
     "named_block": 328,
     "named_partial": 70,
     "named_allow": 34,
     "star_only": 488,
     "blocked_pct": 7.51,
     "named_block_pct": 3.02
    },
    "DuckAssistBot": {
     "named": 522,
     "named_block": 370,
     "named_partial": 111,
     "named_allow": 41,
     "star_only": 474,
     "blocked_pct": 7.76,
     "named_block_pct": 3.4
    },
    "Googlebot": {
     "named": 989,
     "named_block": 20,
     "named_partial": 675,
     "named_allow": 294,
     "star_only": 318,
     "blocked_pct": 3.11,
     "named_block_pct": 0.18
    },
    "Bingbot": {
     "named": 774,
     "named_block": 80,
     "named_partial": 456,
     "named_allow": 238,
     "star_only": 370,
     "blocked_pct": 4.14,
     "named_block_pct": 0.74
    },
    "Applebot": {
     "named": 529,
     "named_block": 206,
     "named_partial": 188,
     "named_allow": 135,
     "star_only": 427,
     "blocked_pct": 5.82,
     "named_block_pct": 1.9
    },
    "anthropic-ai": {
     "named": 1472,
     "named_block": 1248,
     "named_partial": 122,
     "named_allow": 102,
     "star_only": 475,
     "blocked_pct": 15.85,
     "named_block_pct": 11.48
    },
    "Claude-Web": {
     "named": 1232,
     "named_block": 1049,
     "named_partial": 94,
     "named_allow": 89,
     "star_only": 484,
     "blocked_pct": 14.1,
     "named_block_pct": 9.65
    },
    "FacebookBot": {
     "named": 790,
     "named_block": 640,
     "named_partial": 91,
     "named_allow": 59,
     "star_only": 474,
     "blocked_pct": 10.25,
     "named_block_pct": 5.89
    },
    "cohere-ai": {
     "named": 1367,
     "named_block": 1226,
     "named_partial": 92,
     "named_allow": 49,
     "star_only": 477,
     "blocked_pct": 15.67,
     "named_block_pct": 11.28
    },
    "Google-NotebookLM": {
     "named": 162,
     "named_block": 130,
     "named_partial": 24,
     "named_allow": 8,
     "star_only": 491,
     "blocked_pct": 5.71,
     "named_block_pct": 1.2
    },
    "GrokBot": {
     "named": 111,
     "named_block": 104,
     "named_partial": 6,
     "named_allow": 1,
     "star_only": 502,
     "blocked_pct": 5.57,
     "named_block_pct": 0.96
    },
    "xAI-Grok": {
     "named": 74,
     "named_block": 67,
     "named_partial": 6,
     "named_allow": 1,
     "star_only": 504,
     "blocked_pct": 5.25,
     "named_block_pct": 0.62
    },
    "DeepSeekBot": {
     "named": 300,
     "named_block": 271,
     "named_partial": 17,
     "named_allow": 12,
     "star_only": 497,
     "blocked_pct": 7.07,
     "named_block_pct": 2.49
    }
   },
   "any_ai_named": 2981,
   "any_ai_block": 2247,
   "pairs": {
    "openai": {
     "train_block": 1647,
     "train_only": 1224,
     "train_and_search": 423,
     "search_only_backwards": 2,
     "user_block": 634
    },
    "anthropic": {
     "train_block": 1509,
     "train_only": 1138,
     "train_and_search": 371,
     "search_only_backwards": 1,
     "user_block": 396
    }
   },
   "stale_anthropic": 363,
   "names_anthropic_legacy": 1559,
   "blocks_claudebot_not_new": 1074
  }
 },
 "top_agent_names": [
  [
   "*",
   10515
  ],
  [
   "gptbot",
   2301
  ],
  [
   "claudebot",
   2055
  ],
  [
   "ccbot",
   2010
  ],
  [
   "google-extended",
   1868
  ],
  [
   "bytespider",
   1821
  ],
  [
   "meta-externalagent",
   1612
  ],
  [
   "amazonbot",
   1560
  ],
  [
   "applebot-extended",
   1541
  ],
  [
   "anthropic-ai",
   1472
  ],
  [
   "cohere-ai",
   1367
  ],
  [
   "perplexitybot",
   1323
  ],
  [
   "diffbot",
   1321
  ],
  [
   "chatgpt-user",
   1274
  ],
  [
   "claude-web",
   1232
  ],
  [
   "omgili",
   1225
  ],
  [
   "petalbot",
   1183
  ],
  [
   "oai-searchbot",
   1053
  ],
  [
   "baiduspider",
   1024
  ],
  [
   "googlebot",
   989
  ],
  [
   "google-cloudvertexbot",
   826
  ],
  [
   "yandex",
   792
  ],
  [
   "magpie-crawler",
   792
  ],
  [
   "facebookbot",
   790
  ],
  [
   "googleother",
   790
  ],
  [
   "bingbot",
   774
  ],
  [
   "icc-crawler",
   768
  ],
  [
   "adsbot-google",
   763
  ],
  [
   "youbot",
   736
  ],
  [
   "ahrefsbot",
   728
  ],
  [
   "awariosmartbot",
   722
  ],
  [
   "claude-user",
   716
  ],
  [
   "perplexity-user",
   702
  ],
  [
   "twitterbot",
   698
  ],
  [
   "awariorssbot",
   698
  ],
  [
   "claude-searchbot",
   695
  ],
  [
   "omgilibot",
   693
  ],
  [
   "cotoyogi",
   669
  ],
  [
   "facebookexternalhit",
   665
  ],
  [
   "qualifiedbot",
   605
  ],
  [
   "semrushbot-swa",
   602
  ],
  [
   "mj12bot",
   579
  ],
  [
   "timpibot",
   577
  ],
  [
   "semrushbot",
   576
  ],
  [
   "mediapartners-google",
   561
  ],
  [
   "imagesiftbot",
   560
  ],
  [
   "atlassian-bot",
   559
  ],
  [
   "dotbot",
   546
  ],
  [
   "wardbot",
   541
  ],
  [
   "applebot",
   529
  ],
  [
   "meta-externalfetcher",
   524
  ],
  [
   "duckassistbot",
   522
  ],
  [
   "mistralai-training",
   517
  ],
  [
   "kimibot",
   510
  ],
  [
   "scrapy",
   483
  ],
  [
   "ai2bot",
   473
  ],
  [
   "amazonadbot",
   466
  ],
  [
   "fishbot",
   466
  ],
  [
   "citibotsitecrawler",
   466
  ],
  [
   "navubot",
   466
  ],
  [
   "borderxbot",
   466
  ],
  [
   "webzio-extended",
   443
  ],
  [
   "mistralai-user",
   432
  ],
  [
   "cohere-training-data-crawler",
   428
  ],
  [
   "googlebot-image",
   405
  ],
  [
   "blexbot",
   389
  ],
  [
   "ia_archiver",
   382
  ],
  [
   "dataforseobot",
   364
  ],
  [
   "pangubot",
   353
  ],
  [
   "friendlycrawler",
   341
  ],
  [
   "img2dataset",
   339
  ],
  [
   "meta-webindexer",
   334
  ],
  [
   "turnitinbot",
   328
  ],
  [
   "velenpublicwebcrawler",
   324
  ],
  [
   "sogou",
   305
  ],
  [
   "deepseekbot",
   300
  ],
  [
   "ai2bot-dolma",
   296
  ],
  [
   "slurp",
   291
  ],
  [
   "kangaroo",
   279
  ],
  [
   "firecrawlagent",
   268
  ],
  [
   "googlebot-news",
   255
  ],
  [
   "brightbot",
   255
  ],
  [
   "exabot",
   253
  ],
  [
   "adsbot-google-mobile",
   252
  ],
  [
   "yandexbot",
   249
  ],
  [
   "archive.org_bot",
   223
  ],
  [
   "httrack",
   221
  ],
  [
   "shapbot",
   220
  ],
  [
   "msnbot",
   217
  ],
  [
   "iaskspider",
   217
  ],
  [
   "amzn-searchbot",
   213
  ],
  [
   "offline",
   209
  ],
  [
   "amzn-user",
   208
  ],
  [
   "tiktokspider",
   205
  ],
  [
   "yisouspider",
   204
  ],
  [
   "webcopier",
   197
  ],
  [
   "webzip",
   192
  ],
  [
   "linkedinbot",
   191
  ],
  [
   "teleport",
   185
  ],
  [
   "webstripper",
   184
  ],
  [
   "wget",
   184
  ],
  [
   "proximic",
   180
  ],
  [
   "gemini-deep-research",
   177
  ],
  [
   "teleportpro",
   177
  ],
  [
   "sitesnagger",
   177
  ],
  [
   "nutch",
   176
  ],
  [
   "pinterestbot",
   175
  ],
  [
   "rogerbot",
   173
  ],
  [
   "netestate",
   173
  ],
  [
   "aihitbot",
   172
  ],
  [
   "isscyberriskcrawler",
   170
  ],
  [
   "duckduckbot",
   168
  ],
  [
   "peer39_crawler",
   168
  ],
  [
   "msiecrawler",
   168
  ],
  [
   "chatglm-spider",
   167
  ],
  [
   "barkrowler",
   166
  ],
  [
   "larbin",
   164
  ],
  [
   "bravebot",
   164
  ],
  [
   "google-notebooklm",
   162
  ],
  [
   "grapeshot",
   161
  ],
  [
   "psbot",
   159
  ],
  [
   "factset_spyderbot",
   158
  ],
  [
   "sbintuitionsbot",
   157
  ],
  [
   "yeti",
   155
  ],
  [
   "newsnow",
   150
  ],
  [
   "phindbot",
   149
  ],
  [
   "sidetrade",
   145
  ],
  [
   "semrushbot-ocob",
   143
  ],
  [
   "seznambot",
   142
  ],
  [
   "news-please",
   142
  ],
  [
   "yandexadditionalbot",
   137
  ],
  [
   "npbot",
   135
  ],
  [
   "008",
   134
  ],
  [
   "heritrix",
   131
  ],
  [
   "linerbot",
   130
  ],
  [
   "datenbank",
   129
  ],
  [
   "360spider",
   128
  ],
  [
   "webreaper",
   128
  ],
  [
   "devin",
   125
  ],
  [
   "imagespider",
   124
  ],
  [
   "googlebot-video",
   123
  ],
  [
   "linkupbot",
   123
  ],
  [
   "iaskbot",
   122
  ],
  [
   "echoboxbot",
   122
  ],
  [
   "xenu",
   122
  ],
  [
   "download",
   122
  ],
  [
   "yandexadditional",
   122
  ],
  [
   "grub-client",
   118
  ],
  [
   "zyborg",
   118
  ],
  [
   "trendictionbot",
   118
  ]
 ]
}