"""
GLAMWalk -- Volume 2: The Gauntlet
08_checkpointed_crawl.py

Making a multi-hour crawl survivable: saving progress to a checkpoint file after every page, so an interrupted run resumes instead of starting over.

Source: Volume Two, Section 3.3 -- Making a Multi-Day Crawl Survivable
Targets Python 3.11.4 -- see Volume One, Chapter 1.5/1.6.

NOTE: This script makes real network requests to a live public API.
Targets Python 3.11.4 -- see Volume One, Chapter 1.5/1.6.
"""

import json
import os
import time
import requests

BASE_URL = "https://api.artic.edu/api/v1/artworks"


def crawl_aic_page(page, fields="id,title,artist_display,date_display,image_id,is_public_domain"):
    """Same function as in 07_aic_completeness_audit_and_recrawl.py —
    included here too so this file runs standalone."""
    params = {"page": page, "limit": 100, "fields": fields}
    response = requests.get(BASE_URL, params=params, timeout=20)
    response.raise_for_status()
    return response.json()


CHECKPOINT_FILE = "aic_crawl_checkpoint.json"

def load_checkpoint():
    if os.path.exists(CHECKPOINT_FILE):
        with open(CHECKPOINT_FILE) as f:
            return json.load(f)
    return {"last_completed_page": 0, "objects": []}

def save_checkpoint(page, objects):
    with open(CHECKPOINT_FILE, "w") as f:
        json.dump({"last_completed_page": page, "objects": objects}, f)

def crawl_all_aic_resumable(delay_seconds=1.0):
    checkpoint = load_checkpoint()
    page = checkpoint["last_completed_page"] + 1
    all_objects = checkpoint["objects"]

    print(f"Resuming from page {page} ({len(all_objects)} objects already saved)")

    while True:
        data = crawl_aic_page(page)
        objects = data.get("data", [])
        if not objects:
            break

        all_objects.extend(objects)
        save_checkpoint(page, all_objects)   # save progress EVERY page, not just at the end

        total_pages = data.get("pagination", {}).get("total_pages", page)
        if page >= total_pages:
            break

        page += 1
        time.sleep(delay_seconds)

    return all_objects