"""Reads robots.jsonl from collect.ts and writes summary.json and the per-site CSV.

Parsing follows RFC 9309: groups of consecutive user-agent lines, all groups naming a token merged,
`*` used only when no group names the token, longest match wins, Allow wins a tie.

A token is "blocked" when an ordinary page (PROBE) is disallowed for it.
"""

import collections
import csv
import json
import re
import sys

PROBE = "/invisit-probe/some-article"

BOTS = {
    "GPTBot": ("OpenAI", "training"),
    "OAI-SearchBot": ("OpenAI", "search"),
    "ChatGPT-User": ("OpenAI", "user"),
    "ClaudeBot": ("Anthropic", "training"),
    "Claude-SearchBot": ("Anthropic", "search"),
    "Claude-User": ("Anthropic", "user"),
    "PerplexityBot": ("Perplexity", "search"),
    "Perplexity-User": ("Perplexity", "user"),
    "Google-Extended": ("Google", "training"),
    "Applebot-Extended": ("Apple", "training"),
    "Meta-ExternalAgent": ("Meta", "training"),
    "Meta-WebIndexer": ("Meta", "search"),
    "Meta-ExternalFetcher": ("Meta", "user"),
    "Amazonbot": ("Amazon", "training"),
    "Amzn-SearchBot": ("Amazon", "search"),
    "Amzn-User": ("Amazon", "user"),
    "Bytespider": ("ByteDance", "training"),
    "CCBot": ("Common Crawl", "training"),
    "MistralAI-Training": ("Mistral", "training"),
    "MistralAI-Index": ("Mistral", "search"),
    "MistralAI-User": ("Mistral", "user"),
    "DuckAssistBot": ("DuckDuckGo", "user"),
    "Googlebot": ("Google", "engine"),
    "Bingbot": ("Microsoft", "engine"),
    "Applebot": ("Apple", "engine"),
}

# Names operators have retired or never documented, still copied between robots.txt files.
LEGACY = ["anthropic-ai", "Claude-Web", "FacebookBot", "cohere-ai", "Google-NotebookLM", "GrokBot", "xAI-Grok", "DeepSeekBot"]

TIERS = [("top_1k", 1_000), ("top_10k", 10_000), ("top_20k", 20_000)]


def norm(value):
    return re.split(r"[/\s]", value.strip(), maxsplit=1)[0].lower()


def parse(text):
    text = text.lstrip("")
    groups, agents, rules, open_agents = [], [], [], False
    extras = {"content_signal": False}
    for raw in text.splitlines():
        line = raw.split("#", 1)[0].strip()
        if ":" not in line:
            continue
        key, value = line.split(":", 1)
        key, value = key.strip().lower().replace("_", "-"), value.strip()
        if key == "user-agent":
            if not open_agents:
                if agents:
                    groups.append((agents, rules))
                agents, rules = [], []
            agents.append(norm(value))
            open_agents = True
        elif key in ("allow", "disallow"):
            open_agents = False
            if agents and value:
                rules.append((key == "allow", value))
        elif key == "content-signal":
            extras["content_signal"] = True
    if agents:
        groups.append((agents, rules))
    return groups, extras


def matches(pattern, path):
    """RFC 9309 wildcards without a regex: a hostile pattern cannot make this backtrack."""
    anchored = pattern.endswith("$")
    parts = (pattern[:-1] if anchored else pattern).split("*")
    if not path.startswith(parts[0]):
        return False
    if len(parts) == 1:
        return path == parts[0] if anchored else True
    position = len(parts[0])
    for part in parts[1:-1]:
        found = path.find(part, position)
        if found < 0:
            return False
        position = found + len(part)
    last = parts[-1]
    if anchored:
        return len(path) - len(last) >= position and path.endswith(last)
    return path.find(last, position) >= 0


def allowed(rules, path):
    best = None
    for allow, pattern in rules:
        if not matches(pattern, path):
            continue
        rank = (len(pattern), allow)
        if best is None or rank > best:
            best = rank
    return best is None or best[1]


def state(groups, token):
    """none: nothing applies. star: blocked by `*` alone. named_*: a group names the token."""
    key = token.lower()
    named = [rule for agents, rules in groups if key in agents for rule in rules]
    is_named = any(key in agents for agents, _ in groups)
    if is_named:
        if not allowed(named, PROBE):
            return "named_block"
        return "named_partial" if any(not allow for allow, _ in named) else "named_allow"
    star = [rule for agents, rules in groups if "*" in agents for rule in rules]
    return "star" if not allowed(star, PROBE) else "none"


BLOCKED = {"named_block", "star"}


def main(source, summary_path, csv_path):
    rows = [json.loads(line) for line in open(source, encoding="utf8")]
    token_names = list(BOTS) + LEGACY
    sites, name_counts = [], collections.Counter()
    for row in rows:
        site = {
            "rank": row["rank"],
            "domain": row["domain"],
            "valid": row["robots"] is not None,
            "has_llms": row["llms_present"],
            "llms_h1": row["llms_spec_h1"],
            "reachable": row["error"] is None,
            "status": row["status"],
        }
        if site["valid"]:
            groups, extras = parse(row["robots"])
            site["states"] = {token: state(groups, token) for token in token_names}
            site["content_signal"] = extras["content_signal"]
            site["usage_ai_training"] = any("__usage_ai-training" in agents for agents, _ in groups)
            site["star_blocks"] = state(groups, "invisit-unnamed-bot") == "star"
            site["empty"] = not groups
            for name in {agent for agents, _ in groups for agent in agents}:
                name_counts[name] += 1
        sites.append(site)

    def tier_summary(limit):
        scope = [s for s in sites if s["rank"] <= limit]
        valid = [s for s in scope if s["valid"]]
        n = len(valid) or 1
        out = {
            "scanned": len(scope),
            "reachable": sum(s["reachable"] for s in scope),
            "valid_robots": len(valid),
            "empty_robots": sum(s["empty"] for s in valid),
            "star_blocks_all": sum(s["star_blocks"] for s in valid),
            "llms_present": sum(s["has_llms"] for s in scope),
            "llms_h1": sum(s["llms_h1"] for s in scope),
            "content_signal": sum(s["content_signal"] for s in valid),
            "usage_ai_training": sum(s["usage_ai_training"] for s in valid),
            "tokens": {},
        }
        for token in token_names:
            count = collections.Counter(s["states"][token] for s in valid)
            out["tokens"][token] = {
                "named": sum(v for k, v in count.items() if k.startswith("named")),
                "named_block": count["named_block"],
                "named_partial": count["named_partial"],
                "named_allow": count["named_allow"],
                "star_only": count["star"],
                "blocked_pct": round(100 * (count["named_block"] + count["star"]) / n, 2),
                "named_block_pct": round(100 * count["named_block"] / n, 2),
            }

        def blocked(site, token):
            return site["states"][token] in BLOCKED

        def explicit(site, token):
            return site["states"][token] == "named_block"

        any_named_ai = [s for s in valid if any(s["states"][t].startswith("named") for t, m in BOTS.items() if m[1] != "engine")]
        any_block_ai = [s for s in valid if any(explicit(s, t) for t, m in BOTS.items() if m[1] != "engine")]
        out["any_ai_named"] = len(any_named_ai)
        out["any_ai_block"] = len(any_block_ai)
        pairs = {"openai": ("GPTBot", "OAI-SearchBot", "ChatGPT-User"), "anthropic": ("ClaudeBot", "Claude-SearchBot", "Claude-User")}
        out["pairs"] = {}
        for name, (train, search, user) in pairs.items():
            out["pairs"][name] = {
                "train_block": sum(explicit(s, train) for s in valid),
                "train_only": sum(explicit(s, train) and not blocked(s, search) for s in valid),
                "train_and_search": sum(explicit(s, train) and blocked(s, search) for s in valid),
                "search_only_backwards": sum(explicit(s, search) and not blocked(s, train) for s in valid),
                "user_block": sum(explicit(s, user) for s in valid),
            }
        out["stale_anthropic"] = sum(
            (s["states"]["anthropic-ai"].startswith("named") or s["states"]["Claude-Web"].startswith("named")) and not blocked(s, "ClaudeBot")
            for s in valid
        )
        out["names_anthropic_legacy"] = sum(
            s["states"]["anthropic-ai"].startswith("named") or s["states"]["Claude-Web"].startswith("named") for s in valid
        )
        out["blocks_claudebot_not_new"] = sum(
            explicit(s, "ClaudeBot") and not blocked(s, "Claude-SearchBot") and not blocked(s, "Claude-User") for s in valid
        )
        return out

    summary = {
        "sites_in_file": len(sites),
        "fetched_from": min(r["fetched_at"] for r in rows),
        "fetched_to": max(r["fetched_at"] for r in rows),
        "tiers": {name: tier_summary(limit) for name, limit in TIERS},
        "top_agent_names": name_counts.most_common(150),
    }
    json.dump(summary, open(summary_path, "w"), indent=1)

    with open(csv_path, "w", newline="", encoding="utf8") as handle:
        writer = csv.writer(handle)
        writer.writerow(["rank", "domain", "robots_valid", "llms_txt", "content_signal", "usage_ai_training"] + token_names)
        for site in sorted(sites, key=lambda s: s["rank"]):
            base = [site["rank"], site["domain"], int(site["valid"]), int(site["has_llms"])]
            if site["valid"]:
                writer.writerow(base + [int(site["content_signal"]), int(site["usage_ai_training"])] + [site["states"][t] for t in token_names])
            else:
                writer.writerow(base + ["", ""] + [""] * len(token_names))


if __name__ == "__main__":
    main(*sys.argv[1:4])