fix(scrape): make vision OCR a per-bundle allowlist, not a corpus sweep (#6)

Co-authored-by: claude <[email protected]>
This commit was merged in pull request #6.
This commit is contained in:
2026-08-06 11:39:47 -04:00
committed by claude
parent 43d891968e
commit 08c4b4b71d
66 changed files with 223 additions and 447 deletions
+8 -1
View File
@@ -62,6 +62,11 @@ class BundleSpec:
platform: str | None = None
language: str = "en-US"
source_url: str | None = None # overrides the default support.hpe.com URL
# Vision-OCR this bundle's images? Default False. Image-only data lives in
# only a handful of docs (the release-schedule matrix); every other image
# in the corpus is a product UI screenshot that OCR would just add noise
# for. So OCR is an explicit per-bundle opt-in, not a corpus-wide sweep.
ocr: bool = False
# Pinned baseline bundles. docIds confirmed by probing the portal for
@@ -101,7 +106,7 @@ BUNDLES: list[BundleSpec] = [
# the matrix text comes from the VISION_OCR pass (see scrape/vision.py).
# version=None — it spans all major streams, not one release.
BundleSpec("morpheus_release_schedule", "sf000111242en_us", "HPE Morpheus Software Release Schedule",
None, "Release Schedule", "single"),
None, "Release Schedule", "single", ocr=True),
]
@@ -180,6 +185,7 @@ def discover_bundle(s: requests.Session, spec: BundleSpec) -> dict[str, Any]:
"dates": {},
"landing_page": spec.doc_id,
"source_url": spec.source_url or f"https://www.hpe.com/psnow/doc/{spec.doc_id}",
"ocr": spec.ocr,
}
abstract_html = _get(s, f"{API}/{spec.doc_id}", expect_json=False)
@@ -211,6 +217,7 @@ def discover_bundle(s: requests.Session, spec: BundleSpec) -> dict[str, Any]:
"dates": {"Published": meta.get("published", "")},
"landing_page": landing,
"source_url": spec.source_url or DOC_URL.format(doc_id=spec.doc_id),
"ocr": spec.ocr,
}