"""Merge the direct and proxied vendor checks into one conservative verdict.

Each vendor homepage was fetched twice: once from the research host directly and
once through a rotating proxy. A domain counts as reachable if EITHER path
reached it, so neither a blocked research IP nor a blocked datacentre proxy IP
can be mistaken for a dead site. Where both paths returned a status, the more
successful one (2xx/3xx over 4xx/5xx) is kept.
"""
import json, os

HERE = os.path.dirname(os.path.abspath(__file__))
direct = {v["domain"]: v for v in json.load(open(os.path.join(HERE, "vendor_domains_direct.json")))}
proxied = {v["domain"]: v for v in json.load(open(os.path.join(HERE, "vendor_domains.json")))}


def score(v):
    """Lower is better."""
    s = v.get("status")
    if s is None:
        return 4
    if 200 <= s < 400:
        return 0
    if s in (403, 429):
        return 2      # blocked, not dead
    return 3          # 404/410/5xx


out = []
for d in sorted(set(direct) | set(proxied)):
    a, b = direct.get(d), proxied.get(d)
    cands = [x for x in (a, b) if x]
    best = min(cands, key=score)
    rec = dict(best)
    rec["domain"] = d
    rec["direct_status"] = a.get("status") if a else None
    rec["proxy_status"] = b.get("status") if b else None
    rec["ip"] = (a or {}).get("ip") or (b or {}).get("ip")
    rec["checked_both_paths"] = bool(a and b)
    out.append(rec)

json.dump(out, open(os.path.join(HERE, "vendor_domains_merged.json"), "w"), indent=1)
print("merged", len(out))
import collections
print("no DNS:", sum(1 for r in out if not r["ip"]))
print("no response either path:", sum(1 for r in out if r["ip"] and r["status"] is None))
print(collections.Counter(r["status"] for r in out
                          if isinstance(r["status"], int) and r["status"] >= 400).most_common())
print("blocked on proxy but fine direct:",
      sum(1 for r in out if r["proxy_status"] in (403, 429)
          and isinstance(r["direct_status"], int) and r["direct_status"] < 400))
print("blocked direct but fine on proxy:",
      sum(1 for r in out if r["direct_status"] in (403, 429)
          and isinstance(r["proxy_status"], int) and r["proxy_status"] < 400))