"""Re-fetch every page named in the report through the rotating proxy and save
it under evidence/, with a manifest recording status, final URL and fetch time.
"""
import json, os, time
import concurrent.futures as cf
import proxyfetch as pf

HERE = os.path.dirname(os.path.abspath(__file__))
EV = os.path.join(HERE, "evidence")
os.makedirs(EV, exist_ok=True)

URLS = [
    # the three grounding-farm brands
    ("worldmetrics.org", "https://worldmetrics.org/"),
    ("gitnux.org", "https://gitnux.org/"),
    ("wifitalents.com", "https://wifitalents.com/"),
    ("worldmetrics.org.best-project-estimation-software",
     "https://worldmetrics.org/best/project-estimation-software/"),
    ("gitnux.org.best-project-estimation-software",
     "https://gitnux.org/best/project-estimation-software/"),
    ("wifitalents.com.best-project-estimation-software",
     "https://wifitalents.com/best/project-estimation-software/"),
    # the most-cited domain
    ("guideflow.com", "https://www.guideflow.com/"),
    ("guideflow.com.blog-survey-software-tools",
     "https://www.guideflow.com/blog/survey-software-tools"),
    # publishers that blocked the direct fetch
    ("thedigitalprojectmanager.com", "https://thedigitalprojectmanager.com/"),
    ("thectoclub.com", "https://thectoclub.com/"),
    ("thecxlead.com", "https://thecxlead.com/"),
    ("solutionsreview.com", "https://solutionsreview.com/"),
    ("peoplemanagingpeople.com", "https://peoplemanagingpeople.com/"),
    ("devopsschool.com", "https://www.devopsschool.com/"),
    ("softwarereviews.com", "https://www.softwarereviews.com/"),
    # dead / wrong vendor destinations
    ("vendor-dryad.co", "https://dryad.co/"),
    ("vendor-montecarlo.com", "https://montecarlo.com/"),
    ("vendor-auditboard.com", "https://auditboard.com/"),
    ("vendor-datadryad.org", "https://datadryad.org/"),
    # pilot exhibits, re-checked at write time
    ("top-5-solutions.com", "https://top-5-solutions.com/"),
    ("zendikt.com", "https://zendikt.com/"),
]


def one(item):
    name, url = item
    r = pf.get(url, timeout=30)
    path = os.path.join(EV, name + ".html")
    if r["text"]:
        open(path, "w", encoding="utf8").write(r["text"])
    return {"name": name, "url": url, "status": r["status"], "final": r["final"],
            "bytes": len(r["text"]), "err": r["err"],
            "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
            "via": "rotating proxy"}


with cf.ThreadPoolExecutor(8) as ex:
    man = list(ex.map(one, URLS))
json.dump(man, open(os.path.join(EV, "manifest.json"), "w"), indent=1)
for m in man:
    print("%-46s %-5s %8d  %s" % (m["name"], m["status"], m["bytes"],
                                  (m["final"] or m["err"] or "")[:60]))