"""For every recommended homepage that redirects off its own domain, test
whether the destination page still mentions the recommended product.

A rebrand or acquisition normally keeps the product name on the landing page
(getcensus.com -> fivetran.com still says "Census"). A domain that has been
dropped and re-registered by someone else does not. Fetched through the
rotating proxy.
"""
import json, os, re, html, collections
import concurrent.futures as cf
from urllib.parse import urlparse
import proxyfetch as pf

HERE = os.path.dirname(os.path.abspath(__file__))
recs = [json.loads(l) for l in open(os.path.join(HERE, "main_raw.jsonl")) if l.strip()]
vend = json.load(open(os.path.join(HERE, "vendor_domains_merged.json")))


def reg(h):
    p = h.split(".")
    return ".".join(p[-2:]) if len(p) > 2 else h


def host_of(url):
    h = urlparse(url).netloc.lower().split("@")[-1].split(":")[0]
    return h[4:] if h.startswith("www.") else h


names = collections.defaultdict(set)
for r in recs:
    for p in r.get("picks", []):
        d = (p.get("domain") or "").lower()
        if d:
            names[d].add(p["name"].strip())

targets = [v for v in vend if v.get("final")
           and reg(host_of(v["final"])) != reg(v["domain"])]
print("off-domain redirects:", len(targets))


def norm(s):
    return re.sub(r"[^a-z0-9]", "", s.lower())


def check(v):
    d = v["domain"]
    r = pf.get(v["final"], timeout=30)
    txt = re.sub(r"<script.*?</script>|<style.*?</style>", "", r["text"], flags=re.S)
    txt = html.unescape(re.sub(r"<[^>]+>", " ", txt))
    flat = norm(txt)
    # the product name, and the distinctive part of the old domain
    cands = set(names[d]) | {reg(d).split(".")[0]}
    found = sorted(c for c in cands if len(norm(c)) >= 4 and norm(c) in flat)
    return {"domain": d, "names": sorted(names[d]), "final": v["final"],
            "dest_status": r["status"], "dest_bytes": len(r["text"]),
            "name_found_on_destination": bool(found), "matched": found}


with cf.ThreadPoolExecutor(8) as ex:
    res = list(ex.map(check, targets))
json.dump(res, open(os.path.join(HERE, "redirect_check.json"), "w"), indent=1)

lost = [r for r in res if not r["name_found_on_destination"] and r["dest_status"] == 200]
print("destination page does NOT mention the recommended product:", len(lost))
for r in sorted(lost, key=lambda x: x["domain"]):
    print("   %-28s %-26s -> %s" % (r["domain"], "/".join(r["names"])[:26],
                                    r["final"][:60]))
print("destination unreachable:", sum(1 for r in res if r["dest_status"] != 200))