import urllib.request
import urllib.robotparser

rp = urllib.robotparser.RobotFileParser()
rp.set_url("https://en.wikipedia.org/robots.txt")

try:
    rp.read()
    can_fetch = rp.can_fetch("MyResearchBot/1.0", "https://en.wikipedia.org/wiki/Vincent_van_Gogh")
    print(f"Can fetch: {can_fetch}")   # True or False

except Exception as e:
    print(f"Error reading robots.txt: {e}")
    # See 'When the certificate itself is fine and it still fails' below.