diff --git a/scrape/runner.py b/scrape/runner.py index 8742a5c..bc17dd0 100644 --- a/scrape/runner.py +++ b/scrape/runner.py @@ -68,8 +68,19 @@ def _session() -> requests.Session: def _get(s: requests.Session, url: str, expect_json: bool = False, retries: int = 4) -> Any: delay = 1.0 + last_exc: Exception | None = None for attempt in range(retries): - r = s.get(url, timeout=30) + try: + r = s.get(url, timeout=30) + except requests.exceptions.RequestException as e: + # Connection dropped / timed out mid-request. The HPE portal + # intermittently closes connections under the --force re-scrape + # (RemoteDisconnected). These are exceptions, not HTTP statuses, + # so retry them with the same backoff instead of aborting the run. + last_exc = e + time.sleep(delay) + delay *= 2 + continue if r.status_code == 200: return r.json() if expect_json else r.text if r.status_code == 404: @@ -79,7 +90,7 @@ def _get(s: requests.Session, url: str, expect_json: bool = False, retries: int delay *= 2 continue r.raise_for_status() - raise RuntimeError(f"GET failed after {retries} retries: {url}") + raise RuntimeError(f"GET failed after {retries} retries: {url}") from last_exc def _flatten_toc(toc: list[dict]) -> list[TocEntry]: @@ -230,10 +241,21 @@ def scrape_toc_bundle(s: requests.Session, bundle: dict, force: bool, concurrenc } return write_page(bundle_dir, entry.page_id, body_md, sidecar, force) + failed = 0 with ThreadPoolExecutor(max_workers=concurrency) as pool: - for fut in as_completed(pool.submit(do_one, e) for e in entries): - if fut.result(): - written += 1 + futs = {pool.submit(do_one, e): e for e in entries} + for fut in as_completed(futs): + try: + if fut.result(): + written += 1 + except Exception as e: + # One page exhausting its retries shouldn't abort a 2000+ page + # scrape. Skip it (its previously-committed .md is kept) and + # keep going; a broad outage still shows up as a high count. + failed += 1 + print(f" ! {slug}: page {futs[fut].page_id} failed: {e}", file=sys.stderr) + if failed: + print(f" ! {slug}: {failed}/{len(entries)} pages failed — kept prior content", file=sys.stderr) return written