"""
GLAMWalk -- Volume 1: Foundations
09_robots_txt_checker.py

Checking a site's robots.txt before writing any code against it -- using Python's own standard library, no extra installs needed.

Source: Volume One, Section 5.3 -- How to Actually Check, Before You Write a Single Line
Targets Python 3.11.4 -- see Volume One, Chapter 1.5/1.6.

NOTE: This script makes real network requests to a live public API.
It's safe to run as-is, but requires an internet connection.

UPDATE: this version adds two things past the original book listing:

1. A fallback path using the third-party `certifi` package if the
   machine's own trust store rejects a certificate that is, in fact,
   genuinely valid. This is a real, recurring failure mode -- see the
   explanation below -- not a hypothetical edge case.
2. A diagnostic that decodes the actual certificate a site presents,
   so you can see for yourself whether a rejected connection is a real
   problem with the site, or a local trust-store problem.

WHY THIS HAPPENS: `ssl.create_default_context()` (what urllib uses by
default) checks a certificate against your machine's own trust store.
On Windows, that's normally the operating system's own certificate
store, not something bundled with Python itself. That store can go
stale -- most commonly because Windows' automatic root-certificate
update mechanism is disabled (by group policy, on a locked-down or
corporate machine) or because the machine has no network path to fetch
a missing root/intermediate certificate on demand. When that happens,
Python correctly reports that IT can't verify the chain -- even though
the certificate itself, and the site serving it, are both fine.

`certifi` sidesteps this: it's a small, independent bundle of trusted
root certificates that ships as its own pip package and gets updated
on its own schedule, completely independent of the OS. Pointing an SSL
context at certifi's bundle instead of the OS store is a standard,
well-established fix for exactly this situation -- not a workaround
for something wrong with the target site.
"""

import os
import socket
import ssl
import tempfile
import urllib.request
import urllib.robotparser
from datetime import datetime, timezone

try:
    import certifi
    HAVE_CERTIFI = True
except ImportError:
    HAVE_CERTIFI = False


def diagnose_cert(host: str, port: int = 443) -> None:
    """Connect without verifying, and print what certificate we actually
    got back and when it expires. This tells you whether the problem is
    a genuinely bad cert, or something local (clock / trust store / AV / proxy).

    Note: on an unverified connection, ssl's getpeercert() normally
    returns an empty dict -- the parsed fields are only populated when
    the handshake was actually verified. So instead we pull the raw
    certificate bytes (which are always available) and decode those
    directly with _test_decode_cert, which works no matter how the
    verification turned out."""
    ctx = ssl._create_unverified_context()  # diagnostic only -- never use this to fetch real data
    with socket.create_connection((host, port), timeout=10) as sock:
        with ctx.wrap_socket(sock, server_hostname=host) as ssock:
            der_cert = ssock.getpeercert(binary_form=True)

    pem_cert = ssl.DER_cert_to_PEM_cert(der_cert)
    with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f:
        f.write(pem_cert)
        temp_path = f.name
    try:
        cert = ssl._ssl._test_decode_cert(temp_path)
    finally:
        os.unlink(temp_path)

    not_after = datetime.strptime(cert["notAfter"], "%b %d %H:%M:%S %Y %Z").replace(tzinfo=timezone.utc)
    issuer = dict(x[0] for x in cert["issuer"])
    now = datetime.now(timezone.utc)

    print(f"  Certificate presented by {host}:")
    print(f"    Issuer:     {issuer.get('organizationName', issuer.get('commonName', 'unknown'))}")
    print(f"    Expires:    {not_after.isoformat()}")
    print(f"    Local time: {now.isoformat()}")

    if not_after < now:
        print("  -> The certificate really is expired. That's genuinely unusual for a major site;")
        print("     double check the hostname is right, then try again in a few minutes.")
    else:
        print("  -> This certificate is genuinely valid. If the plain, verifying request still")
        print("     fails, your machine's own trust store doesn't currently trust the chain --")
        print("     see 'certifi' below for the fix.")


def check_robots(user_agent: str, target_url: str) -> None:
    host = target_url.split("/")[2] if target_url.startswith("http") else target_url
    robots_url = f"https://{host}/robots.txt"
    rp = urllib.robotparser.RobotFileParser()
    rp.set_url(robots_url)

    try:
        rp.read()
    except Exception as e:
        print(f"Default SSL context failed: {e}\n")

        if not HAVE_CERTIFI:
            print("`certifi` isn't installed, so there's no independent bundle to fall back to.")
            print("Install it with:  pip install certifi\n")
            print("Running a certificate diagnostic against the same host in the meantime:")
            try:
                diagnose_cert(host)
            except Exception as diag_error:
                print(f"  Diagnostic itself failed too: {diag_error}")
                print("  That points at something more fundamental -- no internet connection,")
                print("  or a firewall/proxy blocking the connection entirely.")
            return

        print("Retrying using certifi's independent CA bundle instead of the local trust store...")
        try:
            ctx = ssl.create_default_context(cafile=certifi.where())
            req = urllib.request.Request(robots_url, headers={"User-Agent": user_agent})
            with urllib.request.urlopen(req, context=ctx, timeout=10) as response:
                lines = response.read().decode("utf-8").splitlines()
            rp.parse(lines)
            print("Succeeded using certifi's bundle -- this confirms the machine's own trust")
            print("store (not the target site) was the actual problem.\n")
        except Exception as e2:
            print(f"certifi's bundle also failed: {e2}\n")
            print("Running a certificate diagnostic against the same host:")
            try:
                diagnose_cert(host)
            except Exception as diag_error:
                print(f"  Diagnostic itself failed too: {diag_error}")
                print("  That points at something more fundamental -- no internet connection,")
                print("  or a firewall/proxy blocking the connection entirely.")
            return

    can_fetch = rp.can_fetch(user_agent, target_url)
    print(f"{user_agent} allowed to fetch {target_url}: {can_fetch}")


if __name__ == "__main__":
    check_robots("MyResearchBot/1.0", "https://en.wikipedia.org/wiki/Vincent_van_Gogh")