fix(scrape): make vision OCR a per-bundle allowlist, not a corpus sweep (#6)

Co-authored-by: claude <[email protected]>
This commit was merged in pull request #6.
This commit is contained in:
2026-08-06 11:39:47 -04:00
committed by claude
parent 43d891968e
commit 08c4b4b71d
66 changed files with 223 additions and 447 deletions
+5 -2
View File
@@ -212,7 +212,10 @@ def scrape_toc_bundle(s: requests.Session, bundle: dict, force: bool, concurrenc
page_html = fetch_toc_page(s, doc_id, entry.page_id)
if not page_html:
return False
body_md = html_to_md(page_html, s)
# Only pass the session (which enables image OCR) for bundles that
# opt in — image-only data lives in a few docs; the rest are UI
# screenshots that OCR would just turn into noise.
body_md = html_to_md(page_html, s if bundle.get("ocr") else None)
sidecar = {
"bundle_id": slug,
"page_id": entry.page_id,
@@ -243,7 +246,7 @@ def scrape_single_bundle(s: requests.Session, bundle: dict, force: bool) -> int:
if not html:
print(f" ! {slug}: empty body", file=sys.stderr)
return 0
body_md = html_to_md(html, s)
body_md = html_to_md(html, s if bundle.get("ocr") else None)
sidecar = {
"bundle_id": slug,
"page_id": doc_id,