"""Re-check every recommended vendor domain through the rotating proxy.

DNS is resolved locally (a proxy cannot change whether a name exists); the HTTP
fetch goes out through a fresh proxy IP so that host-level blocking of a single
research IP does not get mistaken for a dead site.
"""
import json, os, socket, sys, time, threading
import concurrent.futures as cf
from urllib.parse import urlparse
import proxyfetch as pf

HERE = os.path.dirname(os.path.abspath(__file__))
socket.setdefaulttimeout(10)

recs = [json.loads(l) for l in open(os.path.join(HERE, "main_raw.jsonl")) if l.strip()]
import collections
vendors = collections.Counter()
for r in recs:
    for p in r.get("picks", []):
        d = (p.get("domain") or "").lower()
        if d and "." in d and " " not in d:
            vendors[d] += 1
doms = sorted(vendors)
print("vendor domains:", len(doms))

lock = threading.Lock()
n = [0]


def check(d):
    o = {"domain": d, "n": vendors[d], "ip": None, "status": None,
         "final": None, "err": None}
    try:
        o["ip"] = socket.gethostbyname(d)
    except Exception as e:
        o["dns_err"] = str(e)[:60]
    if o["ip"]:
        for scheme in ("https", "http"):
            r = pf.get(f"{scheme}://{d}/", timeout=25)
            if r["status"] is not None:
                o["status"] = r["status"]
                o["final"] = r["final"]
                o["bytes"] = len(r["text"])
                break
            o["err"] = r["err"]
    with lock:
        n[0] += 1
        if n[0] % 100 == 0:
            print(n[0], len(doms), flush=True)
    return o


with cf.ThreadPoolExecutor(16) as ex:
    res = list(ex.map(check, doms))
json.dump(res, open(os.path.join(HERE, "vendor_domains.json"), "w"), indent=1)
print("wrote vendor_domains.json")
print("no DNS:", sum(1 for r in res if not r["ip"]))
print("no response:", sum(1 for r in res if r["ip"] and r["status"] is None))
print(collections.Counter(r["status"] for r in res if isinstance(r["status"], int)
                          and r["status"] >= 400).most_common())