From b129ff1ef9534385249643b41f7fd5d998fceae3 Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 6 Aug 2026 11:55:02 -0400 Subject: [PATCH] fix(scrape): retry connection drops; tolerate single-page failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clean refresh (#15483) aborted the whole scrape on a transient upstream blip: requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed ...')) _get() retried HTTP error *statuses* (429/500/502/503/504) but not connection-level *exceptions*, so a single dropped connection — which the HPE portal does intermittently under the --force re-scrape of 2300+ pages at concurrency 6 — killed the entire weekly run. Prior runs just got lucky. - _get(): wrap the request in try/except requests.exceptions.RequestException and retry with the existing exponential backoff (covers RemoteDisconnected, timeouts, chunked-encoding errors). - scrape_toc_bundle(): a page that still fails after all retries is logged and skipped (its previously-committed .md is kept) instead of aborting the bundle. A broad outage still surfaces as a high failed-count in the log. Verified: unit test confirms _get retries a RemoteDisconnected then succeeds; happy-path scrape unaffected. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LFowQzJu7k97QLCRDSAeh1 --- scrape/runner.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/scrape/runner.py b/scrape/runner.py index 8742a5c..bc17dd0 100644 --- a/scrape/runner.py +++ b/scrape/runner.py @@ -68,8 +68,19 @@ def _session() -> requests.Session: def _get(s: requests.Session, url: str, expect_json: bool = False, retries: int = 4) -> Any: delay = 1.0 + last_exc: Exception | None = None for attempt in range(retries): - r = s.get(url, timeout=30) + try: + r = s.get(url, timeout=30) + except requests.exceptions.RequestException as e: + # Connection dropped / timed out mid-request. The HPE portal + # intermittently closes connections under the --force re-scrape + # (RemoteDisconnected). These are exceptions, not HTTP statuses, + # so retry them with the same backoff instead of aborting the run. + last_exc = e + time.sleep(delay) + delay *= 2 + continue if r.status_code == 200: return r.json() if expect_json else r.text if r.status_code == 404: @@ -79,7 +90,7 @@ def _get(s: requests.Session, url: str, expect_json: bool = False, retries: int delay *= 2 continue r.raise_for_status() - raise RuntimeError(f"GET failed after {retries} retries: {url}") + raise RuntimeError(f"GET failed after {retries} retries: {url}") from last_exc def _flatten_toc(toc: list[dict]) -> list[TocEntry]: @@ -230,10 +241,21 @@ def scrape_toc_bundle(s: requests.Session, bundle: dict, force: bool, concurrenc } return write_page(bundle_dir, entry.page_id, body_md, sidecar, force) + failed = 0 with ThreadPoolExecutor(max_workers=concurrency) as pool: - for fut in as_completed(pool.submit(do_one, e) for e in entries): - if fut.result(): - written += 1 + futs = {pool.submit(do_one, e): e for e in entries} + for fut in as_completed(futs): + try: + if fut.result(): + written += 1 + except Exception as e: + # One page exhausting its retries shouldn't abort a 2000+ page + # scrape. Skip it (its previously-committed .md is kept) and + # keep going; a broad outage still shows up as a high count. + failed += 1 + print(f" ! {slug}: page {futs[fut].page_id} failed: {e}", file=sys.stderr) + if failed: + print(f" ! {slug}: {failed}/{len(entries)} pages failed — kept prior content", file=sys.stderr) return written -- 2.54.0