fix(scrape): retry connection drops; tolerate single-page failures

The clean refresh (#15483) aborted the whole scrape on a transient upstream
blip:
  requests.exceptions.ConnectionError:
    ('Connection aborted.', RemoteDisconnected('Remote end closed ...'))

_get() retried HTTP error *statuses* (429/500/502/503/504) but not
connection-level *exceptions*, so a single dropped connection — which the
HPE portal does intermittently under the --force re-scrape of 2300+ pages
at concurrency 6 — killed the entire weekly run. Prior runs just got lucky.

- _get(): wrap the request in try/except requests.exceptions.RequestException
  and retry with the existing exponential backoff (covers RemoteDisconnected,
  timeouts, chunked-encoding errors).
- scrape_toc_bundle(): a page that still fails after all retries is logged
  and skipped (its previously-committed .md is kept) instead of aborting the
  bundle. A broad outage still surfaces as a high failed-count in the log.

Verified: unit test confirms _get retries a RemoteDisconnected then succeeds;
happy-path scrape unaffected.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
Claude-Session: https://claude.ai/code/session_01LFowQzJu7k97QLCRDSAeh1

scrape/runner.py

@@ -68,8 +68,19 @@ def _session() -> requests.Session:
 
 def _get(s: requests.Session, url: str, expect_json: bool = False, retries: int = 4) -> Any:
     delay = 1.0
+    last_exc: Exception | None = None
     for attempt in range(retries):
+        try:
             r = s.get(url, timeout=30)
+        except requests.exceptions.RequestException as e:
+            # Connection dropped / timed out mid-request. The HPE portal
+            # intermittently closes connections under the --force re-scrape
+            # (RemoteDisconnected). These are exceptions, not HTTP statuses,
+            # so retry them with the same backoff instead of aborting the run.
+            last_exc = e
+            time.sleep(delay)
+            delay *= 2
+            continue
         if r.status_code == 200:
             return r.json() if expect_json else r.text
         if r.status_code == 404:
@@ -79,7 +90,7 @@ def _get(s: requests.Session, url: str, expect_json: bool = False, retries: int
             delay *= 2
             continue
         r.raise_for_status()
-    raise RuntimeError(f"GET failed after {retries} retries: {url}")
+    raise RuntimeError(f"GET failed after {retries} retries: {url}") from last_exc
 
 
 def _flatten_toc(toc: list[dict]) -> list[TocEntry]:
@@ -230,10 +241,21 @@ def scrape_toc_bundle(s: requests.Session, bundle: dict, force: bool, concurrenc
         }
         return write_page(bundle_dir, entry.page_id, body_md, sidecar, force)
 
+    failed = 0
     with ThreadPoolExecutor(max_workers=concurrency) as pool:
-        for fut in as_completed(pool.submit(do_one, e) for e in entries):
+        futs = {pool.submit(do_one, e): e for e in entries}
+        for fut in as_completed(futs):
+            try:
                 if fut.result():
                     written += 1
+            except Exception as e:
+                # One page exhausting its retries shouldn't abort a 2000+ page
+                # scrape. Skip it (its previously-committed .md is kept) and
+                # keep going; a broad outage still shows up as a high count.
+                failed += 1
+                print(f"  ! {slug}: page {futs[fut].page_id} failed: {e}", file=sys.stderr)
+    if failed:
+        print(f"  ! {slug}: {failed}/{len(entries)} pages failed — kept prior content", file=sys.stderr)
     return written