scrape/runner.py

@@ -68,8 +68,19 @@ def _session() -> requests.Session:
 
 def _get(s: requests.Session, url: str, expect_json: bool = False, retries: int = 4) -> Any:
     delay = 1.0
+    last_exc: Exception | None = None
     for attempt in range(retries):
-        r = s.get(url, timeout=30)
+        try:
+            r = s.get(url, timeout=30)
+        except requests.exceptions.RequestException as e:
+            # Connection dropped / timed out mid-request. The HPE portal
+            # intermittently closes connections under the --force re-scrape
+            # (RemoteDisconnected). These are exceptions, not HTTP statuses,
+            # so retry them with the same backoff instead of aborting the run.
+            last_exc = e
+            time.sleep(delay)
+            delay *= 2
+            continue
         if r.status_code == 200:
             return r.json() if expect_json else r.text
         if r.status_code == 404:
@@ -79,7 +90,7 @@ def _get(s: requests.Session, url: str, expect_json: bool = False, retries: int
             delay *= 2
             continue
         r.raise_for_status()
-    raise RuntimeError(f"GET failed after {retries} retries: {url}")
+    raise RuntimeError(f"GET failed after {retries} retries: {url}") from last_exc
 
 
 def _flatten_toc(toc: list[dict]) -> list[TocEntry]:
@@ -230,10 +241,21 @@ def scrape_toc_bundle(s: requests.Session, bundle: dict, force: bool, concurrenc
         }
         return write_page(bundle_dir, entry.page_id, body_md, sidecar, force)
 
+    failed = 0
     with ThreadPoolExecutor(max_workers=concurrency) as pool:
-        for fut in as_completed(pool.submit(do_one, e) for e in entries):
+        futs = {pool.submit(do_one, e): e for e in entries}
-            if fut.result():
+        for fut in as_completed(futs):
-                written += 1
+            try:
+                if fut.result():
+                    written += 1
+            except Exception as e:
+                # One page exhausting its retries shouldn't abort a 2000+ page
+                # scrape. Skip it (its previously-committed .md is kept) and
+                # keep going; a broad outage still shows up as a high count.
+                failed += 1
+                print(f"  ! {slug}: page {futs[fut].page_id} failed: {e}", file=sys.stderr)
+    if failed:
+        print(f"  ! {slug}: {failed}/{len(entries)} pages failed — kept prior content", file=sys.stderr)
     return written