"""
11_hathifiles_to_csv.py

Downloads HathiTrust's monthly "Hathifiles" metadata snapshot and
filters it down to public-domain items with no download restriction,
writing the result to a small local CSV -- a genuine bulk metadata
door, separate from the restricted Data API described in 6.2.

This is metadata only. It never fetches a book's actual text or
page images -- it produces a list of titles, identifiers, and links,
the same shape as the museum CC0 CSV exports in Chapter 2.
"""

import csv
import gzip
import io
import sys
import requests

# Hathifiles use no CSV quote character at all -- a literal " in a
# title or author field (curly quotes get flattened to it, inch
# marks, nested quotations, etc.) is just a character, not a
# field-quoting delimiter. Reading with the csv module's default
# QUOTE_MINIMAL makes it treat that stray " as "start of a quoted
# field" and keep consuming subsequent lines looking for a closing
# ", which is what produced the "field larger than field limit"
# crash. QUOTE_NONE turns that behavior off. We also raise the
# field-size limit defensively, since some description/title fields
# are legitimately long.
csv.field_size_limit(min(sys.maxsize, 2**31 - 1))

# HathiTrust publishes a new monthly snapshot on the 1st of each
# month, and doesn't keep a fixed, guessable URL -- so rather than
# hardcoding a date, we ask HathiTrust's own file index which one is
# current. They publish exactly this as a small JSON listing of
# every hathifile (monthly "full" snapshots and daily "upd" deltas
# alike), each with a "full" flag and a "created" timestamp.
HATHI_FILE_LIST_URL = "https://www.hathitrust.org/files/hathifiles/hathi_file_list.json"


def get_latest_full_hathifile_url():
    """
    Ask HathiTrust's own file index for the most recently created
    monthly "full" snapshot, instead of hardcoding a filename/date
    that goes stale the moment the next monthly file is published.
    """
    response = requests.get(HATHI_FILE_LIST_URL, timeout=30)
    response.raise_for_status()
    files = response.json()

    full_snapshots = [f for f in files if f.get("full")]
    if not full_snapshots:
        raise RuntimeError(
            "No full snapshot found in HathiTrust's file list -- "
            "the index format may have changed."
        )

    # "created" looks like "2026-08-01 08:37:44 -0400" -- it sorts
    # correctly as a plain string since it's zero-padded and in
    # year-month-day order, so no date parsing needed.
    latest = max(full_snapshots, key=lambda f: f["created"])
    print(f"Latest full Hathifile snapshot: {latest['filename']} "
          f"({latest['created']})")
    return latest["url"]

# The field order is fixed and documented at HathiTrust's own
# "Hathifiles Description" page -- confirmed directly against their
# published field list rather than guessed from a sample row.
FIELDNAMES = [
    "htid", "access", "rights", "ht_bib_key", "description", "source",
    "source_bib_num", "oclc_num", "isbn", "issn", "lccn", "title",
    "imprint", "rights_reason_code", "rights_timestamp", "us_gov_doc_flag",
    "rights_date_used", "pub_place", "lang", "bib_fmt", "collection_code",
    "content_provider_code", "responsible_entity_code",
    "digitization_agent_code", "access_profile_code", "author",
]

# rights codes that mean "public domain" -- see Section 6.2's note on
# why this is a rolling calculation, not a fixed year: HathiTrust does
# this calculation for you, per item, which is exactly why the
# Hathifiles are worth using instead of hardcoding a cutoff year.
PUBLIC_DOMAIN_RIGHTS_CODES = {"pd", "pdus"}


def download_and_filter_hathifile(output_csv="hathitrust_public_domain.csv"):
    """
    Stream the monthly Hathifile, keep only rows that are both
    public domain AND unrestricted for download, and write a small,
    genuinely useful local CSV.
    """
    hathifile_url = get_latest_full_hathifile_url()
    print("Downloading HathiTrust's monthly metadata snapshot...")
    response = requests.get(hathifile_url, timeout=300, stream=True)
    response.raise_for_status()

    kept, seen = 0, 0
    with gzip.GzipFile(fileobj=io.BytesIO(response.content)) as gz_file:
        text_stream = io.TextIOWrapper(gz_file, encoding="utf-8", errors="replace")
        reader = csv.DictReader(
            text_stream,
            fieldnames=FIELDNAMES,
            delimiter="\t",
            quoting=csv.QUOTE_NONE,
        )

        with open(output_csv, "w", newline="", encoding="utf-8") as out:
            # QUOTE_MINIMAL is fine for the *output* file, since this
            # is a normal comma-delimited CSV and we want it to quote
            # any field that happens to contain a comma or quote mark.
            writer = csv.writer(out)
            writer.writerow(["htid", "title", "author", "rights",
                              "access_profile_code", "oclc_num", "reader_url"])

            for row in reader:
                seen += 1
                # DictReader stuffs any extra tab-separated values into
                # row[None] as a list, and fills missing trailing
                # columns with None -- both signal a malformed line.
                # Skip those rather than crashing on a bad row deep
                # into a multi-gigabyte file.
                if row.get(None) is not None or row.get("access_profile_code") is None:
                    continue
                if row["rights"] not in PUBLIC_DOMAIN_RIGHTS_CODES:
                    continue
                if row["access_profile_code"] != "open":
                    # Public domain, but still page-by-page-only or
                    # otherwise gated -- not what this script is for.
                    continue
                kept += 1
                reader_url = f"https://babel.hathitrust.org/cgi/pt?id={row['htid']}"
                writer.writerow([row["htid"], row["title"], row["author"],
                                  row["rights"], row["access_profile_code"],
                                  row["oclc_num"], reader_url])

    print(f"Scanned {seen:,} items -- kept {kept:,} public domain, "
          f"unrestricted-download items -> {output_csv}")


if __name__ == "__main__":
    download_and_filter_hathifile()