"""
GLAMWalk -- Volume 2: The Gauntlet
07_aic_completeness_audit_and_recrawl.py

A cheap sanity check for whether a 'bulk dataset' is actually complete, plus a real rate-limited, paginated crawl against the Art Institute of Chicago's live API.

Source: Volume Two, Sections 3.1-3.2
Targets Python 3.11.4 -- see Volume One, Chapter 1.5/1.6.

NOTE: This script makes real network requests to a live public API.
It's safe to run as-is, but requires an internet connection.
"""

import os

def audit_dataset_completeness(directory, expected_min_files=1000):
    """
    A cheap sanity check worth running against ANY 'bulk dataset'
    before building a pipeline around it: does the file count even
    plausibly match what a real full export of this size should have?
    """
    files = os.listdir(directory)
    print(f"{len(files)} files found in {directory}")
    if len(files) < expected_min_files:
        print(
            f"⚠ WARNING: expected at least {expected_min_files} files for "
            f"a real full export — this may be a sample set, not the full data."
        )
    return files


import requests
import time

BASE_URL = "https://api.artic.edu/api/v1/artworks"

def crawl_aic_page(page, fields="id,title,artist_display,date_display,image_id,is_public_domain"):
    params = {
        "page": page,
        "limit": 100,
        "fields": fields,
    }
    response = requests.get(BASE_URL, params=params, timeout=20)
    response.raise_for_status()
    return response.json()

def crawl_all_aic(max_pages=None, delay_seconds=1.0):
    """
    A full, resumable, rate-limited crawl of AIC's live API.
    Politeness (delay_seconds) matters as much as correctness here —
    this is a shared public resource, not a private database.
    """
    page = 1
    all_objects = []
    while True:
        data = crawl_aic_page(page)
        objects = data.get("data", [])
        if not objects:
            break

        all_objects.extend(objects)
        print(f"Page {page}: {len(objects)} objects (running total: {len(all_objects)})")

        total_pages = data.get("pagination", {}).get("total_pages", page)
        if page >= total_pages or (max_pages and page >= max_pages):
            break

        page += 1
        time.sleep(delay_seconds)   # be a polite, sustainable citizen of a shared API

    return all_objects