feat(scrape): vision OCR of image-only doc pages (support matrices)

Some HPE pages carry information only inside an image. The Morpheus
release schedule / support-lifecycle matrix (sf000111242en_us) is the
case that surfaced this: the entire "which version ships when, when a
stream hits Maintenance/EOL" table is a JPEG, so html_to_md dropped
every value and retrieval could never answer "when does 8.x.x reach
end of life".

New scrape/vision.py (opt-in via VISION_OCR) transcribes qualifying
images with a local Ollama vision model and appends a labeled markdown
block to the page so the text is chunked, embedded and retrieved.

Model/prompt chosen against ground truth on the matrix (14x4 cells):
- qwen2.5vl:7b + a bare "transcribe the table" prompt: 55-56/56, ~18s,
  GPU-resident, and prompt-ROBUST (accurate without hand-tuning).
- gemma3:12b needs an exact "read row by row, keep cells aligned"
  prompt or it shifts a column by one row; qwen2.5vl:32b is no more
  accurate and 5x slower (CPU spill). Both documented in the code.
- Note: self-consistency (2 samples must agree) only catches RANDOM
  flakiness — a wrong read is stable across seeds — so every block also
  ships a "verify against the source image" caveat.

Reliability/cost:
- content-hash cache in corpus/.vision-cache/ (committed) — each unique
  image OCR'd once ever, shared across version bundles.
- VISION_MAX_NEW bounds NEW OCRs per run so the first pass can't balloon
  into hours; the cache fills incrementally. Deferred count is logged.
- every failure path degrades to the pre-vision behavior; never blocks
  the scrape.

Also:
- add the morpheus_release_schedule bundle (sf single-doc) + 2 eval
  golden queries for it.
- fetch_single_doc: title falls back to the bundle title (not docId)
  when a page has no <h1> (sf solution articles).
- Pillow in requirements-vision.txt (scrape-only; kept out of the
  server image), installed + VISION_* wired into refresh.yml.

Verified locally: matrix transcribes to the exact 14-row table; second
run is a cache hit (ocr=0).

Co-Authored-By: Claude Opus 4.8 <[email protected]>
Claude-Session: https://claude.ai/code/session_01LFowQzJu7k97QLCRDSAeh1

scrape/runner.py

@@ -29,11 +29,18 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
+from urllib.parse import urljoin
 
 import requests
 from bs4 import BeautifulSoup
 from markdownify import markdownify as md
 
+from scrape import vision
+
+# Images on the portal carry relative src (e.g. /hpesc/public/api/...); resolve
+# against the portal host before fetching them for OCR.
+IMG_BASE = "https://support.hpe.com"
+
 API = "https://support.hpe.com/hpesc/public/api/document"
 DOC_URL = "https://support.hpe.com/hpesc/public/docDisplay?docId={doc_id}&page={page_id}.html"
 DOC_URL_SINGLE = "https://support.hpe.com/hpesc/public/docDisplay?docId={doc_id}"
@@ -124,12 +131,39 @@ def _strip_dita_wrappers(html: str) -> str:
     return str(main) if main else str(soup)
 
 
-def html_to_md(page_html: str) -> str:
+def _transcribe_images(cleaned_html: str, session: requests.Session) -> list[str]:
+    """OCR every large image in the (boilerplate-stripped) page and return the
+    labeled transcription blocks. No-ops unless VISION_OCR is set. Deduped by
+    URL within a page; vision.transcribe() handles size/budget/cache/errors."""
+    soup = BeautifulSoup(cleaned_html, "html.parser")
+    blocks: list[str] = []
+    seen: set[str] = set()
+    for img in soup.find_all("img"):
+        src = (img.get("src") or "").strip()
+        if not src:
+            continue
+        url = urljoin(IMG_BASE, src)
+        if url in seen:
+            continue
+        seen.add(url)
+        block = vision.transcribe(url, session)
+        if block:
+            blocks.append(block)
+    return blocks
+
+
+def html_to_md(page_html: str, session: requests.Session | None = None) -> str:
     cleaned = _strip_dita_wrappers(page_html)
     text = md(cleaned, heading_style="ATX", bullets="-")
     # collapse runs of blank lines
-    text = re.sub(r"\n{3,}", "\n\n", text).strip()
-    return text + "\n"
+    text = re.sub(r"\n{3,}", "\n\n", text).strip() + "\n"
+    # Append vision OCR of any image-only content (support matrices, etc.) so
+    # the text is chunked + embedded instead of lost. Opt-in via VISION_OCR.
+    if session is not None and vision.enabled():
+        blocks = _transcribe_images(cleaned, session)
+        if blocks:
+            text += "\n## Image transcriptions\n" + "".join(blocks)
+    return text
 
 
 def fetch_toc_page(s: requests.Session, doc_id: str, page_id: str) -> str:
@@ -146,7 +180,10 @@ def fetch_single_doc(s: requests.Session, doc_id: str) -> tuple[str, str]:
         return "", ""
     soup = BeautifulSoup(html, "html.parser")
     h1 = soup.select_one("h1.title.topictitle1")
-    title = h1.get_text(" ", strip=True) if h1 else doc_id
+    # Empty (not doc_id) when there's no heading, so scrape_single_bundle falls
+    # back to the human bundle title. `sf` solution articles (e.g. the release
+    # schedule) have no topictitle1; without this they'd be titled by docId.
+    title = h1.get_text(" ", strip=True) if h1 else ""
     return html, title
 
 
@@ -175,7 +212,7 @@ def scrape_toc_bundle(s: requests.Session, bundle: dict, force: bool, concurrenc
         page_html = fetch_toc_page(s, doc_id, entry.page_id)
         if not page_html:
             return False
-        body_md = html_to_md(page_html)
+        body_md = html_to_md(page_html, s)
         sidecar = {
             "bundle_id": slug,
             "page_id": entry.page_id,
@@ -206,7 +243,7 @@ def scrape_single_bundle(s: requests.Session, bundle: dict, force: bool) -> int:
     if not html:
         print(f"  ! {slug}: empty body", file=sys.stderr)
         return 0
-    body_md = html_to_md(html)
+    body_md = html_to_md(html, s)
     sidecar = {
         "bundle_id": slug,
         "page_id": doc_id,
@@ -326,6 +363,8 @@ def main() -> int:
         else:
             total += scrape_toc_bundle(s, b, args.force, args.concurrency)
     print(f"scraped {total} new/updated pages", file=sys.stderr)
+    if vision.enabled():
+        print(vision.summary(), file=sys.stderr)
 
     # Always finalize after a scrape so sidecars are consistent.
     all_bundles = json.loads(BUNDLES_JSON.read_text())