#!/usr/bin/env python3
"""Fetch a fixed Show HN cohort from the public Algolia HN Search API."""

from __future__ import annotations

import argparse
import datetime as dt
import gzip
import hashlib
import json
import pathlib
import time
import urllib.parse
import urllib.request

ENDPOINT = "https://hn.algolia.com/api/v1/search_by_date"
USER_AGENT = "TrendiumoResearch/1.0 (+https://trendiumo.com/methodology)"


def epoch(value: str) -> int:
    return int(dt.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp())


def windows(start: dt.datetime, end: dt.datetime, days: int):
    cursor = start
    while cursor < end:
        nxt = min(cursor + dt.timedelta(days=days), end)
        yield cursor, nxt
        cursor = nxt


def fetch_window(start_i: int, end_i: int, pause: float):
    page = 0
    requests = 0
    while True:
        params = {
            "tags": "show_hn",
            "hitsPerPage": "1000",
            "page": str(page),
            "numericFilters": f"created_at_i>={start_i},created_at_i<{end_i}",
        }
        url = f"{ENDPOINT}?{urllib.parse.urlencode(params)}"
        request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
        with urllib.request.urlopen(request, timeout=45) as response:
            payload = json.load(response)
        requests += 1
        hits = payload.get("hits", [])
        for hit in hits:
            yield hit, requests
        page += 1
        if not hits or page >= int(payload.get("nbPages", 0)):
            break
        time.sleep(pause)


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--start", default="2025-07-01T00:00:00Z")
    parser.add_argument("--end", default="2026-07-01T00:00:00Z")
    parser.add_argument("--output", default="evidence/data/show-hn-2025-07-01_2026-06-30.jsonl.gz")
    parser.add_argument("--metadata", default="evidence/fetch-metadata.json")
    parser.add_argument("--pause", type=float, default=0.12)
    parser.add_argument("--window-days", type=int, default=7)
    args = parser.parse_args()

    start = dt.datetime.fromisoformat(args.start.replace("Z", "+00:00"))
    end = dt.datetime.fromisoformat(args.end.replace("Z", "+00:00"))
    rows: dict[str, dict] = {}
    request_total = 0

    for window_start, window_end in windows(start, end, args.window_days):
        last_request_count = 0
        window_rows: dict[str, dict] = {}
        for hit, request_count in fetch_window(epoch(window_start.isoformat()), epoch(window_end.isoformat()), args.pause):
            object_id = str(hit.get("objectID") or hit.get("story_id") or "")
            if object_id:
                window_rows[object_id] = hit
            last_request_count = request_count
        if len(window_rows) >= 1000:
            raise RuntimeError(
                f"API result cap reached for {window_start.date()}..{window_end.date()} "
                f"({len(window_rows)} rows). Reduce --window-days; refusing a partial snapshot."
            )
        rows.update(window_rows)
        request_total += last_request_count
        print(f"{window_start.date()}..{window_end.date()}: window={len(window_rows)}, total unique={len(rows)}")

    ordered = sorted(rows.values(), key=lambda row: (int(row.get("created_at_i") or 0), str(row.get("objectID") or "")))
    output = pathlib.Path(args.output)
    output.parent.mkdir(parents=True, exist_ok=True)
    with gzip.open(output, "wt", encoding="utf-8", newline="\n") as stream:
        for row in ordered:
            stream.write(json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n")

    digest = hashlib.sha256(output.read_bytes()).hexdigest()
    metadata = {
        "schemaVersion": 1,
        "source": ENDPOINT,
        "queryTag": "show_hn",
        "periodStart": args.start,
        "periodEndExclusive": args.end,
        "fetchedAt": dt.datetime.now(dt.timezone.utc).isoformat(),
        "requestCount": request_total,
        "windowDays": args.window_days,
        "rowCount": len(ordered),
        "output": str(output),
        "sha256": digest,
    }
    metadata_path = pathlib.Path(args.metadata)
    metadata_path.parent.mkdir(parents=True, exist_ok=True)
    metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(metadata, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()