#!/usr/bin/env python3
"""
Download the 200-paper evaluation corpus from corpus_papers.csv.

Reads the manifest, fetches each paper from its source_url, validates that the
response is a PDF, and writes it to {output}/{bucket}/{filename}. Skips files
that already exist. Rate-limits at 0.5 s/request.

Usage:
    python3 download_corpus.py corpus_papers.csv
    python3 download_corpus.py corpus_papers.csv --output ./papers/
    python3 download_corpus.py corpus_papers.csv --only core
    python3 download_corpus.py corpus_papers.csv --only noise

Requires: requests (`python3 -m pip install requests`).
"""
from __future__ import annotations

import argparse
import csv
import sys
import time
from pathlib import Path

try:
    import requests
except ImportError:
    sys.stderr.write(
        "ERROR: this script requires the 'requests' package.\n"
        "       Install with: python3 -m pip install requests\n"
    )
    sys.exit(2)

USER_AGENT = (
    "Mozilla/5.0 (corpus_reproducer/1.0) "
    "https://github.com/vbianchi/Mentori"
)
TIMEOUT = 30
SLEEP_BETWEEN = 0.5  # seconds


def download_one(source_url: str, dest: Path) -> tuple[str, str]:
    """Download a single PDF. Returns (status, message)."""
    if dest.exists() and dest.stat().st_size > 1024:
        return "SKIP", f"already exists ({dest.stat().st_size // 1024} KB)"

    dest.parent.mkdir(parents=True, exist_ok=True)

    try:
        resp = requests.get(
            source_url,
            headers={"User-Agent": USER_AGENT},
            timeout=TIMEOUT,
            allow_redirects=True,
            stream=True,
        )
    except requests.RequestException as e:
        return "FAIL", f"network error: {type(e).__name__}: {str(e)[:60]}"

    if resp.status_code != 200:
        return "FAIL", f"HTTP {resp.status_code}"

    content = resp.content
    if not content.startswith(b"%PDF"):
        return "FAIL", "response is not a PDF (likely a paywall or HTML page)"

    dest.write_bytes(content)
    return "OK", f"{len(content) // 1024} KB"


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Download the 200-paper corpus from corpus_papers.csv"
    )
    parser.add_argument(
        "csv_file", help="Path to corpus_papers.csv (the manifest)"
    )
    parser.add_argument(
        "--output", default="./papers", help="Output directory (default: ./papers)"
    )
    parser.add_argument(
        "--only",
        choices=["core", "noise"],
        help="Download only one bucket",
    )
    parser.add_argument(
        "--limit",
        type=int,
        default=None,
        help="Stop after N papers (useful for smoke testing)",
    )
    args = parser.parse_args()

    csv_path = Path(args.csv_file)
    output_dir = Path(args.output)

    if not csv_path.exists():
        print(f"ERROR: manifest not found: {csv_path}", file=sys.stderr)
        return 1

    with csv_path.open() as f:
        rows = list(csv.DictReader(f))

    if args.only:
        rows = [r for r in rows if r["bucket"] == args.only]
    if args.limit:
        rows = rows[: args.limit]

    print(f"Manifest: {csv_path}")
    print(f"Output:   {output_dir}")
    print(f"Papers:   {len(rows)}")
    print("=" * 78)

    counts = {"OK": 0, "SKIP": 0, "FAIL": 0}
    failures: list[tuple[str, str, str, str]] = []

    for i, row in enumerate(rows, 1):
        pid = row["paper_id"]
        bucket = row["bucket"]
        filename = row["filename"]
        url = row["source_url"]
        dest = output_dir / bucket / filename

        status, message = download_one(url, dest)
        counts[status] += 1
        marker = {"OK": "✓", "SKIP": "·", "FAIL": "✗"}[status]
        print(f"[{i:3d}/{len(rows)}] {marker} [{status:4s}] {filename:38s} {message}")

        if status == "FAIL":
            failures.append((pid, bucket, filename, message))

        if status != "SKIP":
            time.sleep(SLEEP_BETWEEN)

    print("=" * 78)
    print(f"Summary:  OK {counts['OK']}   SKIP {counts['SKIP']}   FAIL {counts['FAIL']}")

    if failures:
        print()
        print("Failed downloads (try manually):")
        for pid, bucket, fname, reason in failures:
            print(f"  paper_id={pid:>3} {bucket}/{fname}  ({reason})")
            # Show URL for manual retrieval
            for r in rows:
                if r["paper_id"] == pid:
                    print(f"      {r['source_url']}")
                    break

    return 0 if counts["FAIL"] == 0 else 1


if __name__ == "__main__":
    sys.exit(main())
