오류, robots.txt 거부 및 재시도

import os
import random
import time

import requests

API = "https://webreaderapi.com/api/v1/read"
HEADERS = {"Authorization": "Bearer " + os.environ["WEBREADER_API_KEY"]}
RETRY = {"concurrency_limit", "fetch_timeout", "fetch_failed"}
FINAL = {"blocked_by_robots", "blocked_by_site_policy", "target_refused", "unsupported_content"}


def read_page(url, attempts=5, **options):
    wait = 1.0
    for attempt in range(1, attempts + 1):
        resp = requests.get(API, headers=HEADERS, params={"url": url, **options}, timeout=75)
        if resp.status_code == 200:
            return resp.json()
        error = resp.json().get("error", {})
        code = error.get("code", "unknown")
        if code in FINAL:
            return {"url": url, "skipped": code, "target_status": error.get("target_status")}
        if code not in RETRY or attempt == attempts or (code == "fetch_failed" and attempt >= 3):
            raise RuntimeError(f"{code}: {error.get('message')}")
        time.sleep(wait + random.uniform(0, wait / 2))
        wait = min(wait * 2, 30)
    raise RuntimeError("unreachable")


if __name__ == "__main__":
    print(read_page("https://example.com/blog/first-post", max_chars=20000).get("title"))

const RETRY = new Set(['concurrency_limit', 'fetch_timeout', 'fetch_failed']);
const FINAL = new Set(['blocked_by_robots', 'blocked_by_site_policy', 'target_refused', 'unsupported_content']);
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

export async function readPage(url, options = {}, attempts = 5) {
  let wait = 1000;
  for (let attempt = 1; attempt <= attempts; attempt++) {
    const params = new URLSearchParams({ url, ...options });
    const res = await fetch(`https://webreaderapi.com/api/v1/read?${params}`, {
      headers: { Authorization: `Bearer ${process.env.WEBREADER_API_KEY}` },
    });
    const body = await res.json();
    if (res.ok) return body;
    const code = body.error?.code ?? 'unknown';
    if (FINAL.has(code)) return { url, skipped: code, target_status: body.error.target_status };
    if (!RETRY.has(code) || attempt === attempts) throw new Error(`${code}: ${body.error?.message}`);
    await sleep(wait + Math.random() * (wait / 2));
    wait = Math.min(wait * 2, 30000);
  }
}