llms.txtジェネレーター：自社サイトのクロールからファイルを作成

import os
import time
from collections import defaultdict
from urllib.parse import urlparse

import requests

API = "https://webreaderapi.com/api/v1"
HEADERS = {"Authorization": "Bearer " + os.environ["WEBREADER_API_KEY"]}
SITE = "https://docs.example.com/"


def first_paragraph(markdown):
    for block in markdown.split("\n\n"):
        block = block.strip()
        if block and not block.startswith(("#", "-", "|", "!", ">")):
            return " ".join(block.split())[:160]
    return ""


job = requests.post(API + "/crawl", headers=HEADERS, json={"url": SITE, "limit": 300}, timeout=60).json()
pages, cursor, delay = [], None, 5
while True:
    params = {"cursor": cursor} if cursor else {}
    batch = requests.get(API + "/jobs/" + job["job_id"], headers=HEADERS, params=params, timeout=60).json()
    pages += [p for p in batch.get("results", []) if "markdown" in p]
    cursor = batch.get("next_cursor")
    if batch["status"] == "done" and not cursor:
        break
    if not cursor:
        time.sleep(delay)
        delay = min(delay * 2, 60)

sections = defaultdict(list)
for p in pages:
    path = urlparse(p["final_url"]).path.strip("/")
    section = path.split("/")[0].replace("-", " ").title() if path else "Home"
    summary = p.get("description") or first_paragraph(p["markdown"])
    sections[section].append("- [" + p["title"] + "](" + p["final_url"] + "): " + summary)

lines = ["# Example Docs", "", "> Documentation for the Example product.", ""]
for name in sorted(sections):
    lines += ["## " + name, ""] + sorted(sections[name]) + [""]
with open("llms.txt", "w", encoding="utf-8") as fh:
    fh.write("\n".join(lines))
print(len(pages), "pages written to llms.txt")