Python에서 웹페이지를 Markdown으로 변환

pip install requests httpx
export WEBREADER_API_KEY="your-key"

import os

import requests

API = "https://webreaderapi.com/api/v1/read"
HEADERS = {"Authorization": "Bearer " + os.environ["WEBREADER_API_KEY"]}


def read(url, **options):
    resp = requests.get(API, headers=HEADERS, params={"url": url, **options}, timeout=75)
    if resp.status_code != 200:
        error = resp.json()["error"]
        raise RuntimeError(error["code"] + ": " + error["message"])
    return resp.json()


page = read("https://example.com/docs/getting-started", max_chars=20000)
print(page["title"], page["word_count"], "words", page["credits_used"], "credit(s)")
print(page["markdown"][:500])

import asyncio
import os

import httpx

API = "https://webreaderapi.com/api/v1/read"
HEADERS = {"Authorization": "Bearer " + os.environ["WEBREADER_API_KEY"]}


async def read_one(client, sem, url):
    async with sem:
        resp = await client.get(API, params={"url": url, "max_chars": 50000})
    body = resp.json()
    if resp.status_code == 200:
        return url, body, None
    return url, None, body["error"]["code"]


async def read_many(urls, limit=10):
    sem = asyncio.Semaphore(limit)
    async with httpx.AsyncClient(headers=HEADERS, timeout=75) as client:
        return await asyncio.gather(*(read_one(client, sem, u) for u in urls))


if __name__ == "__main__":
    urls = [f"https://example.com/blog/post-{n}" for n in range(1, 21)]
    for url, page, error in asyncio.run(read_many(urls)):
        print(url, "->", error or page["title"])

import os

import requests

urls = [f"https://example.com/changelog/{year}" for year in range(2020, 2027)]
resp = requests.post(
    "https://webreaderapi.com/api/v1/batch",
    headers={"Authorization": "Bearer " + os.environ["WEBREADER_API_KEY"]},
    json={"urls": urls, "options": {"format": "markdown", "main_content": True}},
    timeout=30,
)
resp.raise_for_status()
print("job:", resp.json()["job_id"])

import re


def chunk_markdown(page, max_chars=2000):
    """Split a /read response into chunks with a heading path and the source URL."""
    chunks, path, buf = [], [], []

    def flush():
        text = "\n".join(buf).strip()
        while text:
            part = text[:max_chars]
            cut = part.rfind("\n\n")
            if len(text) > max_chars and cut > max_chars // 2:
                part = part[:cut]
            chunks.append({"text": part.strip(), "section": " > ".join(path), "title": page.get("title"), "source": page["final_url"]})
            text = text[len(part):].strip()
        buf.clear()

    for line in page["markdown"].splitlines():
        m = re.match(r"^(#{1,3}) (.+)", line)
        if m:
            flush()
            level = len(m.group(1))
            path[:] = path[: level - 1] + [m.group(2).strip()]
        buf.append(line)
    flush()
    return chunks