"""Write the cleaned, publishable dataset into the site repo's static dir."""
import csv, json, os, shutil
from urllib.parse import urlparse

HERE = os.path.dirname(os.path.abspath(__file__))
SLUG = "manufactured-sources-behind-ai-recommendations"
SITE = os.environ.get("SITE_ROOT", os.path.join(HERE, "..", "..", "..", ".."))
DEST = os.path.join(SITE, "static", "data", SLUG)
os.makedirs(DEST, exist_ok=True)

recs = [json.loads(l) for l in open(os.path.join(HERE, "main_raw.jsonl")) if l.strip()]
cited = {c["domain"]: c for c in json.load(open(os.path.join(HERE, "cited_domains.json")))}
vend = json.load(open(os.path.join(HERE, "vendor_domains_merged.json")))


def reg(h):
    p = h.split(".")
    return ".".join(p[-2:]) if len(p) > 2 else h


def wb(ts):
    return "%s-%s-%s" % (ts[:4], ts[4:6], ts[6:8]) if ts else ""


# 1. citations.csv
with open(os.path.join(DEST, "citations.csv"), "w", newline="") as f:
    w = csv.writer(f)
    w.writerow(["model", "category", "citation_url", "domain",
                "tranco_rank", "wayback_first_capture"])
    for r in recs:
        for u in r.get("citations", []):
            d = reg(urlparse(u).netloc.lower().replace("www.", ""))
            c = cited.get(d, {})
            w.writerow([r["model"], r["cat"], u, d,
                        c.get("tranco_rank") or "", wb(c.get("wayback_first"))])

# 2. answers.csv
with open(os.path.join(DEST, "answers.csv"), "w", newline="") as f:
    w = csv.writer(f)
    w.writerow(["model", "category", "position", "product_name", "vendor_domain"])
    for r in recs:
        for i, p in enumerate(r.get("picks", []), 1):
            w.writerow([r["model"], r["cat"], i, p["name"], p.get("domain", "")])

# 3. cited_domains.csv
with open(os.path.join(DEST, "cited_domains.csv"), "w", newline="") as f:
    w = csv.writer(f)
    w.writerow(["domain", "citations", "categories_cited_in", "tranco_rank",
                "wayback_first_capture"])
    for d, c in sorted(cited.items(), key=lambda x: -x[1]["citations"]):
        w.writerow([d, c["citations"], c["categories"], c.get("tranco_rank") or "",
                    wb(c.get("wayback_first"))])

# 4. vendor_domains.csv
with open(os.path.join(DEST, "vendor_domains.csv"), "w", newline="") as f:
    w = csv.writer(f)
    w.writerow(["domain", "times_recommended", "dns_resolves", "http_status",
                "final_url", "direct_status", "proxy_status"])
    for v in sorted(vend, key=lambda x: -x["n"]):
        w.writerow([v["domain"], v["n"], "yes" if v.get("ip") else "no",
                    v.get("status") or "", v.get("final") or "",
                    v.get("direct_status") or "", v.get("proxy_status") or ""])

# 5. supporting json + scripts
shutil.copy(os.path.join(HERE, "numbers.json"), DEST)
shutil.copy(os.path.join(HERE, "sitemaps.json"), DEST)
shutil.copy(os.path.join(HERE, "redirect_check.json"), DEST)
os.makedirs(os.path.join(DEST, "scripts"), exist_ok=True)
for s in ["cats300.py", "run_main.py", "enrich.py", "recheck_vendors.py",
          "merge_vendors.py", "redirect_check.py", "analyze.py", "farm_facts.py",
          "export.py", "proxyfetch.py", "refetch_evidence.py", "spend.py"]:
    shutil.copy(os.path.join(HERE, s), os.path.join(DEST, "scripts", s))
# orlib.py is republished with the key read from the environment rather than
# from a local dotenv path.
_orlib = open(os.path.join(os.path.dirname(HERE), "orlib.py")).read()
_head, _rest = _orlib.split("K=key()", 1)
_orlib = ('import os,json,urllib.request,re,time\n'
          'K = os.environ["OPENROUTER_API_KEY"]\n' + _rest.lstrip("\n"))
open(os.path.join(DEST, "scripts", "orlib.py"), "w").write(_orlib)
shutil.copy(os.path.join(os.path.dirname(HERE), "cats.py"),
            os.path.join(DEST, "scripts", "cats.py"))
shutil.copy(os.path.join(HERE, "METHOD.md"), DEST)

# raw answers, jsonl, with the model's prose trimmed out
with open(os.path.join(DEST, "answers_raw.jsonl"), "w") as f:
    for r in recs:
        f.write(json.dumps({k: v for k, v in r.items()
                            if k in ("model", "cat", "ts", "picks", "citations")}) + "\n")

total = 0
for root, _, files in os.walk(DEST):
    for fn in files:
        total += os.path.getsize(os.path.join(root, fn))
print("wrote", DEST, "%.1f MB" % (total / 1e6))
for root, _, files in os.walk(DEST):
    for fn in sorted(files):
        p = os.path.join(root, fn)
        print("  %-42s %8.1f kB" % (os.path.relpath(p, DEST), os.path.getsize(p) / 1e3))

# 6. a plain directory index so the /data/<slug>/ link resolves
rows = []
for root, _, files in sorted(os.walk(DEST)):
    for fn in sorted(files):
        if fn == "index.html":
            continue
        p = os.path.join(root, fn)
        rel = os.path.relpath(p, DEST)
        rows.append((rel, os.path.getsize(p)))


def _kb(n):
    return "%.0f kB" % (n / 1e3) if n < 1e6 else "%.1f MB" % (n / 1e6)


LINKS = "\n".join(
    '<li><a href="%s">%s</a> <span class="s">%s</span></li>' % (r, r, _kb(s))
    for r, s in rows)
open(os.path.join(DEST, "index.html"), "w").write("""<!doctype html>
<html lang="en"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Dataset: manufactured sources behind AI recommendations | Trellner Research</title>
<meta name="robots" content="noindex">
<link rel="stylesheet" href="/style.css">
<style>ul{list-style:none;padding:0}li{padding:.25rem 0;border-bottom:1px solid var(--rule)}
.s{color:var(--muted);font-size:.85em;float:right}main{max-width:740px;margin:3rem auto;padding:0 1.25rem}</style>
</head><body><main>
<h1>Dataset</h1>
<p>Supporting data and scripts for
<a href="/reports/%s/">Facts and Grounding Pages: What AI Reads Before Recommending Software</a>.
Collected 2 September 2026. Released under CC BY 4.0.
See <a href="README.md">README.md</a> for the columns and <a href="METHOD.md">METHOD.md</a> for the method.</p>
<ul>
%s
</ul>
</main></body></html>
""" % (SLUG, LINKS))
print("wrote index.html")

# 7. evidence: plain-text extracts of every page named in the report, plus the
# fetch manifest, so each named claim can be checked without refetching.
import html as _html
import re as _re
EVSRC = os.path.join(HERE, "evidence")
EVDST = os.path.join(DEST, "evidence")
os.makedirs(EVDST, exist_ok=True)
_man = json.load(open(os.path.join(EVSRC, "manifest.json")))
json.dump(_man, open(os.path.join(EVDST, "manifest.json"), "w"), indent=1)
for m in _man:
    src = os.path.join(EVSRC, m["name"] + ".html")
    if not os.path.exists(src):
        continue
    b = open(src, encoding="utf8", errors="ignore").read()
    ttl = _re.search(r"<title>(.*?)</title>", b, _re.S)
    desc = _re.search(r'<meta name=["\']description["\'] content=["\'](.*?)["\']', b, _re.S)
    t = _re.sub(r"<script.*?</script>|<style.*?</style>", "", b, flags=_re.S)
    t = _html.unescape(_re.sub(r"<[^>]+>", "\n", t))
    lines = [x.strip() for x in t.split("\n") if x.strip()]
    head = ["# %s" % m["name"],
            "url: %s" % m["url"],
            "final_url: %s" % (m["final"] or ""),
            "http_status: %s" % m["status"],
            "fetched_at: %s (via %s)" % (m["fetched_at"], m["via"]),
            "title: %s" % (_html.unescape(ttl.group(1).strip()) if ttl else ""),
            "meta_description: %s" % (_html.unescape(desc.group(1).strip()) if desc else ""),
            "", "--- visible text (truncated) ---", ""]
    open(os.path.join(EVDST, m["name"] + ".txt"), "w", encoding="utf8").write(
        "\n".join(head) + "\n".join(lines)[:40000])
print("wrote evidence/", len(os.listdir(EVDST)), "files")