"""Fetch helper: every request goes out through a rotating proxy (fresh IP).

The proxy URL is read from proxy_secret.txt, which is NOT part of the published
dataset. Set FETCH_PROXY in the environment to override.
"""
import os
import requests

UA = ("Mozilla/5.0 (compatible; TrellnerResearchBot/1.0; "
      "+https://trellner.com/research)")

_HERE = os.path.dirname(os.path.abspath(__file__))


def _proxy():
    p = os.environ.get("FETCH_PROXY")
    if p:
        return p.strip()
    path = os.path.join(_HERE, "proxy_secret.txt")
    if os.path.exists(path):
        return open(path).read().strip()
    return None


P = _proxy()
PROXIES = {"http": P, "https": P} if P else None


def get(url, timeout=25, allow_redirects=True, stream_bytes=300000, retries=2):
    """GET through the proxy. Returns a dict; retries once on proxy error."""
    out = {"url": url, "status": None, "final": None, "text": "", "err": None}
    last = None
    for attempt in range(retries):
        try:
            r = requests.get(url, headers={"User-Agent": UA}, proxies=PROXIES,
                             timeout=timeout, allow_redirects=allow_redirects,
                             verify=False)
            out["status"] = r.status_code
            out["final"] = r.url
            out["text"] = r.text[:stream_bytes]
            out["err"] = None
            return out
        except requests.exceptions.RequestException as e:
            last = e
            out["err"] = type(e).__name__ + ": " + str(e)[:120]
    out["err"] = out["err"] or (type(last).__name__ if last else "unknown")
    return out


requests.packages.urllib3.disable_warnings()