import ssl
import certifi
import urllib.request
import urllib.robotparser

ctx = ssl.create_default_context(cafile=certifi.where())
req = urllib.request.Request(
    "https://en.wikipedia.org/robots.txt",
    headers={"User-Agent": "MyResearchBot/1.0"},
)

rp = urllib.robotparser.RobotFileParser()
with urllib.request.urlopen(req, context=ctx) as response:
    rp.parse(response.read().decode("utf-8").splitlines())

can_fetch = rp.can_fetch("MyResearchBot/1.0", "https://en.wikipedia.org/wiki/Vincent_van_Gogh")
print(f"Can fetch: {can_fetch}")