feat(scrape): vision OCR of image-only doc pages (support matrices) (#5)

Co-authored-by: claude <[email protected]>

scrape/runner.py

@@ -29,11 +29,18 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
+from urllib.parse import urljoin
 
 import requests
 from bs4 import BeautifulSoup
 from markdownify import markdownify as md
 
+from scrape import vision
+
+# Images on the portal carry relative src (e.g. /hpesc/public/api/...); resolve
+# against the portal host before fetching them for OCR.
+IMG_BASE = "https://support.hpe.com"
+
 API = "https://support.hpe.com/hpesc/public/api/document"
 DOC_URL = "https://support.hpe.com/hpesc/public/docDisplay?docId={doc_id}&page={page_id}.html"
 DOC_URL_SINGLE = "https://support.hpe.com/hpesc/public/docDisplay?docId={doc_id}"
@@ -124,12 +131,39 @@ def _strip_dita_wrappers(html: str) -> str:
     return str(main) if main else str(soup)
 
 
-def html_to_md(page_html: str) -> str:
+def _transcribe_images(cleaned_html: str, session: requests.Session) -> list[str]:
+    """OCR every large image in the (boilerplate-stripped) page and return the
+    labeled transcription blocks. No-ops unless VISION_OCR is set. Deduped by
+    URL within a page; vision.transcribe() handles size/budget/cache/errors."""
+    soup = BeautifulSoup(cleaned_html, "html.parser")
+    blocks: list[str] = []
+    seen: set[str] = set()
+    for img in soup.find_all("img"):
+        src = (img.get("src") or "").strip()
+        if not src:
+            continue
+        url = urljoin(IMG_BASE, src)
+        if url in seen:
+            continue
+        seen.add(url)
+        block = vision.transcribe(url, session)
+        if block:
+            blocks.append(block)
+    return blocks
+
+
+def html_to_md(page_html: str, session: requests.Session | None = None) -> str:
     cleaned = _strip_dita_wrappers(page_html)
     text = md(cleaned, heading_style="ATX", bullets="-")
     # collapse runs of blank lines
-    text = re.sub(r"\n{3,}", "\n\n", text).strip()
-    return text + "\n"
+    text = re.sub(r"\n{3,}", "\n\n", text).strip() + "\n"
+    # Append vision OCR of any image-only content (support matrices, etc.) so
+    # the text is chunked + embedded instead of lost. Opt-in via VISION_OCR.
+    if session is not None and vision.enabled():
+        blocks = _transcribe_images(cleaned, session)
+        if blocks:
+            text += "\n## Image transcriptions\n" + "".join(blocks)
+    return text
 
 
 def fetch_toc_page(s: requests.Session, doc_id: str, page_id: str) -> str:
@@ -146,7 +180,10 @@ def fetch_single_doc(s: requests.Session, doc_id: str) -> tuple[str, str]:
         return "", ""
     soup = BeautifulSoup(html, "html.parser")
     h1 = soup.select_one("h1.title.topictitle1")
-    title = h1.get_text(" ", strip=True) if h1 else doc_id
+    # Empty (not doc_id) when there's no heading, so scrape_single_bundle falls
+    # back to the human bundle title. `sf` solution articles (e.g. the release
+    # schedule) have no topictitle1; without this they'd be titled by docId.
+    title = h1.get_text(" ", strip=True) if h1 else ""
     return html, title
 
 
@@ -175,7 +212,7 @@ def scrape_toc_bundle(s: requests.Session, bundle: dict, force: bool, concurrenc
         page_html = fetch_toc_page(s, doc_id, entry.page_id)
         if not page_html:
             return False
-        body_md = html_to_md(page_html)
+        body_md = html_to_md(page_html, s)
         sidecar = {
             "bundle_id": slug,
             "page_id": entry.page_id,
@@ -206,7 +243,7 @@ def scrape_single_bundle(s: requests.Session, bundle: dict, force: bool) -> int:
     if not html:
         print(f"  ! {slug}: empty body", file=sys.stderr)
         return 0
-    body_md = html_to_md(html)
+    body_md = html_to_md(html, s)
     sidecar = {
         "bundle_id": slug,
         "page_id": doc_id,
@@ -326,6 +363,8 @@ def main() -> int:
         else:
             total += scrape_toc_bundle(s, b, args.force, args.concurrency)
     print(f"scraped {total} new/updated pages", file=sys.stderr)
+    if vision.enabled():
+        print(vision.summary(), file=sys.stderr)
 
     # Always finalize after a scrape so sidecars are consistent.
     all_bundles = json.loads(BUNDLES_JSON.read_text())