ウェブサイト変更監視：公開ページをクリーンなMarkdownとして差分比較

import difflib
import json
import os
import pathlib
import time

import requests

API = "https://webreaderapi.com/api/v1"
HEADERS = {"Authorization": "Bearer " + os.environ["WEBREADER_API_KEY"]}
STORE = pathlib.Path("snapshots.json")
URLS = [
    "https://example.com/pricing",
    "https://example.com/legal/terms",
    "https://example.com/changelog",
]

old = json.loads(STORE.read_text(encoding="utf-8")) if STORE.exists() else {}
job = requests.post(API + "/batch", headers=HEADERS, json={"urls": URLS}, timeout=60).json()
results, cursor, delay = [], None, 5
while True:
    params = {"cursor": cursor} if cursor else {}
    page = requests.get(API + "/jobs/" + job["job_id"], headers=HEADERS, params=params, timeout=60).json()
    results += page.get("results", [])
    cursor = page.get("next_cursor")
    if page["status"] == "done" and not cursor:
        break
    if not cursor:
        time.sleep(delay)
        delay = min(delay * 2, 60)

new = dict(old)
for item in results:
    if "markdown" not in item:
        print("not read:", item["url"], item.get("error", {}).get("code"))
        continue
    before = old.get(item["url"], "")
    if before and before != item["markdown"]:
        diff = difflib.unified_diff(before.splitlines(), item["markdown"].splitlines(), lineterm="", n=0)
        print("CHANGED:", item["url"])
        print("\n".join(list(diff)[:40]))
    new[item["url"]] = item["markdown"]
STORE.write_text(json.dumps(new, ensure_ascii=False), encoding="utf-8")