feat(scrape): vision OCR of image-only doc pages (support matrices)
Some HPE pages carry information only inside an image. The Morpheus release schedule / support-lifecycle matrix (sf000111242en_us) is the case that surfaced this: the entire "which version ships when, when a stream hits Maintenance/EOL" table is a JPEG, so html_to_md dropped every value and retrieval could never answer "when does 8.x.x reach end of life". New scrape/vision.py (opt-in via VISION_OCR) transcribes qualifying images with a local Ollama vision model and appends a labeled markdown block to the page so the text is chunked, embedded and retrieved. Model/prompt chosen against ground truth on the matrix (14x4 cells): - qwen2.5vl:7b + a bare "transcribe the table" prompt: 55-56/56, ~18s, GPU-resident, and prompt-ROBUST (accurate without hand-tuning). - gemma3:12b needs an exact "read row by row, keep cells aligned" prompt or it shifts a column by one row; qwen2.5vl:32b is no more accurate and 5x slower (CPU spill). Both documented in the code. - Note: self-consistency (2 samples must agree) only catches RANDOM flakiness — a wrong read is stable across seeds — so every block also ships a "verify against the source image" caveat. Reliability/cost: - content-hash cache in corpus/.vision-cache/ (committed) — each unique image OCR'd once ever, shared across version bundles. - VISION_MAX_NEW bounds NEW OCRs per run so the first pass can't balloon into hours; the cache fills incrementally. Deferred count is logged. - every failure path degrades to the pre-vision behavior; never blocks the scrape. Also: - add the morpheus_release_schedule bundle (sf single-doc) + 2 eval golden queries for it. - fetch_single_doc: title falls back to the bundle title (not docId) when a page has no <h1> (sf solution articles). - Pillow in requirements-vision.txt (scrape-only; kept out of the server image), installed + VISION_* wired into refresh.yml. Verified locally: matrix transcribes to the exact 14-row table; second run is a cache hit (ocr=0). Co-Authored-By: Claude Opus 4.8 <[email protected]> Claude-Session: https://claude.ai/code/session_01LFowQzJu7k97QLCRDSAeh1
This commit is contained in:
Claude Opus 4.8
@@ -37,6 +37,18 @@ env:
|
|||||||
OLLAMA_URLS: http://192.168.0.2:11435,http://192.168.0.2:11436,http://192.168.0.125:11434,http://192.168.0.126:11434
|
OLLAMA_URLS: http://192.168.0.2:11435,http://192.168.0.2:11436,http://192.168.0.125:11434,http://192.168.0.126:11434
|
||||||
EMBED_MODEL: nomic-embed-text
|
EMBED_MODEL: nomic-embed-text
|
||||||
|
|
||||||
|
# Vision OCR of image-only pages (support/lifecycle matrices etc. — see
|
||||||
|
# scrape/vision.py). Uses the host's primary Ollama on :11434, the only one
|
||||||
|
# with a vision model; the embed pool above is nomic-only. qwen2.5vl:7b
|
||||||
|
# scored best on the release-schedule matrix (55-56/56, ~18s, GPU-resident,
|
||||||
|
# prompt-robust). Content-hash cached in corpus/.vision-cache/; VISION_MAX_NEW
|
||||||
|
# bounds NEW OCRs per run so the cache fills incrementally instead of a
|
||||||
|
# multi-hour first pass. Any failure degrades to the pre-vision behavior.
|
||||||
|
VISION_OCR: "1"
|
||||||
|
VISION_URL: http://192.168.0.2:11434
|
||||||
|
VISION_MODEL: qwen2.5vl:7b
|
||||||
|
VISION_MAX_NEW: "60"
|
||||||
|
|
||||||
PRODUCT_NAME: morpheus
|
PRODUCT_NAME: morpheus
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
@@ -64,6 +76,9 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
python -m pip install -q --upgrade pip
|
python -m pip install -q --upgrade pip
|
||||||
python -m pip install -q -r requirements.txt
|
python -m pip install -q -r requirements.txt
|
||||||
|
# Vision OCR deps (Pillow) — only the scrape step needs these; kept
|
||||||
|
# out of requirements.txt so they never bloat the server image.
|
||||||
|
python -m pip install -q -r requirements-vision.txt
|
||||||
|
|
||||||
# ---- Phase 1: scrape ---------------------------------------
|
# ---- Phase 1: scrape ---------------------------------------
|
||||||
- name: Refresh bundle catalog
|
- name: Refresh bundle catalog
|
||||||
|
|||||||
@@ -149,5 +149,22 @@
|
|||||||
"dates": {},
|
"dates": {},
|
||||||
"landing_page": "a50009231enw",
|
"landing_page": "a50009231enw",
|
||||||
"source_url": "https://www.hpe.com/psnow/doc/a50009231enw"
|
"source_url": "https://www.hpe.com/psnow/doc/a50009231enw"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "morpheus_release_schedule",
|
||||||
|
"doc_id": "sf000111242en_us",
|
||||||
|
"title": "HPE Morpheus Software Release Schedule",
|
||||||
|
"version": null,
|
||||||
|
"platform": null,
|
||||||
|
"product": "Release Schedule",
|
||||||
|
"language": "en-US",
|
||||||
|
"page_count": 1,
|
||||||
|
"mode": "single",
|
||||||
|
"abstract": "",
|
||||||
|
"dates": {
|
||||||
|
"Published": ""
|
||||||
|
},
|
||||||
|
"landing_page": "sf000111242en_us",
|
||||||
|
"source_url": "https://support.hpe.com/hpesc/public/docDisplay?docId=sf000111242en_us"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"sha": "4cb0e044a29be128f264ec1c1d65f704e2ce3488d8d1c8ddf1d89e8cc97c9cfc",
|
||||||
|
"src": "https://support.hpe.com/hpesc/public/api/document/sf000111242en_us/ka2Kh0000008hBd_Procedure__c_0EMKh000006RMP6.jpg?v=3.0",
|
||||||
|
"model": "qwen2.5vl:7b",
|
||||||
|
"certain": true,
|
||||||
|
"markdown": "| Release Window | 7.x.x | 8.x.x | 9.x.x | 10.x.x |\n|----------------|-------|-------|-------|--------|\n| January 2026 | Maintenance | 8.0.13 | | |\n| February 2026 | | 8.1.0 | | |\n| March 2026 | | 8.1.1 | | |\n| April 2026 | | 8.1.2 | | |\n| May 2026 | EOL | Maintenance | 9.0.0 | |\n| June 2026 | | | 9.0.1 | |\n| July 2026 | | | 9.0.2 | |\n| August 2026 | | | 9.1.0 | |\n| September 2026 | | | 9.1.1 | |\n| Oct '26 - Apr '27 | | | 9.1.2 - 9.3.2 | |\n| May 2027 | | EOL | Maintenance | 10.0.0 |\n| June 2027 | | | | 10.0.1|\n| July 2027 | | | | 10.0.2|\n| August 2027 | | | | 10.1.0|"
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"bundle_id": "morpheus_release_schedule",
|
||||||
|
"page_id": "sf000111242en_us",
|
||||||
|
"title": "HPE Morpheus Software Release Schedule",
|
||||||
|
"ordinal": 1,
|
||||||
|
"parent_title": null,
|
||||||
|
"doc_id": "sf000111242en_us",
|
||||||
|
"version": null,
|
||||||
|
"product": "Release Schedule",
|
||||||
|
"source_url": "https://support.hpe.com/hpesc/public/docDisplay?docId=sf000111242en_us"
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# Steps
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
HPE Morpheus Software supports each major version for a period of two years. This support window is split between a twelve-month period of active development followed by another twelve-month period of maintenance support. During active maintenance, an upgrade containing new features is released once per quarter (version x.1.x, x.2.x, etc). Each month between the quarterly feature upgrades, we will deliver a maintenance upgrade containing improvements and bug fixes (version x.x.1 and x.x.2). After this twelve-month period of active development ends, a new major upgrade is delivered (version 9.x.x, 10.x.x, etc) and a twelve-month period of maintenance support begins. During maintenance support, there are no scheduled monthly upgrades. However, support services are still available and critical security upgrades will be delivered should the need arise. After this twelve-month period of maintenance support ends, all versions within that major release umbrella will reach end of life. **After the security maintenance window ends and end-of-life is reached, access to Morpheus support services for that major version stream will also end. To receive support, you will be required to upgrade to a supported version.**
|
||||||
|
|
||||||
|
**NOTE:** The table above is meant to be an illustration of the expected release cadence described. It is not necessarily a commitment to release specific version numbers at specific times.
|
||||||
|
|
||||||
|
## Image transcriptions
|
||||||
|
|
||||||
|
> Transcribed from the image above by local vision OCR (qwen2.5vl:7b); verify against the source image.
|
||||||
|
|
||||||
|
| Release Window | 7.x.x | 8.x.x | 9.x.x | 10.x.x |
|
||||||
|
|----------------|-------|-------|-------|--------|
|
||||||
|
| January 2026 | Maintenance | 8.0.13 | | |
|
||||||
|
| February 2026 | | 8.1.0 | | |
|
||||||
|
| March 2026 | | 8.1.1 | | |
|
||||||
|
| April 2026 | | 8.1.2 | | |
|
||||||
|
| May 2026 | EOL | Maintenance | 9.0.0 | |
|
||||||
|
| June 2026 | | | 9.0.1 | |
|
||||||
|
| July 2026 | | | 9.0.2 | |
|
||||||
|
| August 2026 | | | 9.1.0 | |
|
||||||
|
| September 2026 | | | 9.1.1 | |
|
||||||
|
| Oct '26 - Apr '27 | | | 9.1.2 - 9.3.2 | |
|
||||||
|
| May 2027 | | EOL | Maintenance | 10.0.0 |
|
||||||
|
| June 2027 | | | | 10.0.1|
|
||||||
|
| July 2027 | | | | 10.0.2|
|
||||||
|
| August 2027 | | | | 10.1.0|
|
||||||
@@ -2,3 +2,5 @@
|
|||||||
{"query": "add an AWS cloud integration", "expected": [], "tags": ["cloud", "TODO-populate-after-first-scrape"]}
|
{"query": "add an AWS cloud integration", "expected": [], "tags": ["cloud", "TODO-populate-after-first-scrape"]}
|
||||||
{"query": "Plugin API version compatibility", "expected": [], "tags": ["api", "TODO"]}
|
{"query": "Plugin API version compatibility", "expected": [], "tags": ["api", "TODO"]}
|
||||||
{"query": "Morpheus Enterprise 8.1.2 what's new", "expected": [{"bundle_id": "morpheus_release_notes_8_1_2", "page_id": "sd00007733en_us"}], "tags": ["release-notes"]}
|
{"query": "Morpheus Enterprise 8.1.2 what's new", "expected": [{"bundle_id": "morpheus_release_notes_8_1_2", "page_id": "sd00007733en_us"}], "tags": ["release-notes"]}
|
||||||
|
{"query": "when does Morpheus 8.x.x reach end of life", "expected": [{"bundle_id": "morpheus_release_schedule", "page_id": "sf000111242en_us"}], "tags": ["lifecycle", "eol", "vision-ocr"]}
|
||||||
|
{"query": "Morpheus software release schedule and support window", "expected": [{"bundle_id": "morpheus_release_schedule", "page_id": "sf000111242en_us"}], "tags": ["lifecycle", "vision-ocr"]}
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Optional: local vision OCR of image-only doc pages (scrape/vision.py).
|
||||||
|
# Installed only in the refresh workflow, NOT baked into the server image —
|
||||||
|
# the MCP server never runs OCR, so keeping Pillow out of the base image
|
||||||
|
# keeps it lean (same reasoning as requirements-rerank.txt keeping out torch).
|
||||||
|
# requests (the Ollama client) is already in requirements.txt.
|
||||||
|
Pillow>=10.0
|
||||||
@@ -95,6 +95,13 @@ BUNDLES: list[BundleSpec] = [
|
|||||||
BundleSpec("morpheus_quickspecs", "a50009231enw", "HPE Morpheus Enterprise Software QuickSpecs",
|
BundleSpec("morpheus_quickspecs", "a50009231enw", "HPE Morpheus Enterprise Software QuickSpecs",
|
||||||
"v1", "QuickSpecs", "html-file",
|
"v1", "QuickSpecs", "html-file",
|
||||||
source_url="https://www.hpe.com/psnow/doc/a50009231enw"),
|
source_url="https://www.hpe.com/psnow/doc/a50009231enw"),
|
||||||
|
# Release schedule / support-lifecycle matrix. This `sf` solution article
|
||||||
|
# is a single page whose table (which version ships when; when a stream
|
||||||
|
# hits Maintenance/EOL) lives ENTIRELY inside a JPEG. Scraped as `single`;
|
||||||
|
# the matrix text comes from the VISION_OCR pass (see scrape/vision.py).
|
||||||
|
# version=None — it spans all major streams, not one release.
|
||||||
|
BundleSpec("morpheus_release_schedule", "sf000111242en_us", "HPE Morpheus Software Release Schedule",
|
||||||
|
None, "Release Schedule", "single"),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+45
-6
@@ -29,11 +29,18 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from markdownify import markdownify as md
|
from markdownify import markdownify as md
|
||||||
|
|
||||||
|
from scrape import vision
|
||||||
|
|
||||||
|
# Images on the portal carry relative src (e.g. /hpesc/public/api/...); resolve
|
||||||
|
# against the portal host before fetching them for OCR.
|
||||||
|
IMG_BASE = "https://support.hpe.com"
|
||||||
|
|
||||||
API = "https://support.hpe.com/hpesc/public/api/document"
|
API = "https://support.hpe.com/hpesc/public/api/document"
|
||||||
DOC_URL = "https://support.hpe.com/hpesc/public/docDisplay?docId={doc_id}&page={page_id}.html"
|
DOC_URL = "https://support.hpe.com/hpesc/public/docDisplay?docId={doc_id}&page={page_id}.html"
|
||||||
DOC_URL_SINGLE = "https://support.hpe.com/hpesc/public/docDisplay?docId={doc_id}"
|
DOC_URL_SINGLE = "https://support.hpe.com/hpesc/public/docDisplay?docId={doc_id}"
|
||||||
@@ -124,12 +131,39 @@ def _strip_dita_wrappers(html: str) -> str:
|
|||||||
return str(main) if main else str(soup)
|
return str(main) if main else str(soup)
|
||||||
|
|
||||||
|
|
||||||
def html_to_md(page_html: str) -> str:
|
def _transcribe_images(cleaned_html: str, session: requests.Session) -> list[str]:
|
||||||
|
"""OCR every large image in the (boilerplate-stripped) page and return the
|
||||||
|
labeled transcription blocks. No-ops unless VISION_OCR is set. Deduped by
|
||||||
|
URL within a page; vision.transcribe() handles size/budget/cache/errors."""
|
||||||
|
soup = BeautifulSoup(cleaned_html, "html.parser")
|
||||||
|
blocks: list[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for img in soup.find_all("img"):
|
||||||
|
src = (img.get("src") or "").strip()
|
||||||
|
if not src:
|
||||||
|
continue
|
||||||
|
url = urljoin(IMG_BASE, src)
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
block = vision.transcribe(url, session)
|
||||||
|
if block:
|
||||||
|
blocks.append(block)
|
||||||
|
return blocks
|
||||||
|
|
||||||
|
|
||||||
|
def html_to_md(page_html: str, session: requests.Session | None = None) -> str:
|
||||||
cleaned = _strip_dita_wrappers(page_html)
|
cleaned = _strip_dita_wrappers(page_html)
|
||||||
text = md(cleaned, heading_style="ATX", bullets="-")
|
text = md(cleaned, heading_style="ATX", bullets="-")
|
||||||
# collapse runs of blank lines
|
# collapse runs of blank lines
|
||||||
text = re.sub(r"\n{3,}", "\n\n", text).strip()
|
text = re.sub(r"\n{3,}", "\n\n", text).strip() + "\n"
|
||||||
return text + "\n"
|
# Append vision OCR of any image-only content (support matrices, etc.) so
|
||||||
|
# the text is chunked + embedded instead of lost. Opt-in via VISION_OCR.
|
||||||
|
if session is not None and vision.enabled():
|
||||||
|
blocks = _transcribe_images(cleaned, session)
|
||||||
|
if blocks:
|
||||||
|
text += "\n## Image transcriptions\n" + "".join(blocks)
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
def fetch_toc_page(s: requests.Session, doc_id: str, page_id: str) -> str:
|
def fetch_toc_page(s: requests.Session, doc_id: str, page_id: str) -> str:
|
||||||
@@ -146,7 +180,10 @@ def fetch_single_doc(s: requests.Session, doc_id: str) -> tuple[str, str]:
|
|||||||
return "", ""
|
return "", ""
|
||||||
soup = BeautifulSoup(html, "html.parser")
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
h1 = soup.select_one("h1.title.topictitle1")
|
h1 = soup.select_one("h1.title.topictitle1")
|
||||||
title = h1.get_text(" ", strip=True) if h1 else doc_id
|
# Empty (not doc_id) when there's no heading, so scrape_single_bundle falls
|
||||||
|
# back to the human bundle title. `sf` solution articles (e.g. the release
|
||||||
|
# schedule) have no topictitle1; without this they'd be titled by docId.
|
||||||
|
title = h1.get_text(" ", strip=True) if h1 else ""
|
||||||
return html, title
|
return html, title
|
||||||
|
|
||||||
|
|
||||||
@@ -175,7 +212,7 @@ def scrape_toc_bundle(s: requests.Session, bundle: dict, force: bool, concurrenc
|
|||||||
page_html = fetch_toc_page(s, doc_id, entry.page_id)
|
page_html = fetch_toc_page(s, doc_id, entry.page_id)
|
||||||
if not page_html:
|
if not page_html:
|
||||||
return False
|
return False
|
||||||
body_md = html_to_md(page_html)
|
body_md = html_to_md(page_html, s)
|
||||||
sidecar = {
|
sidecar = {
|
||||||
"bundle_id": slug,
|
"bundle_id": slug,
|
||||||
"page_id": entry.page_id,
|
"page_id": entry.page_id,
|
||||||
@@ -206,7 +243,7 @@ def scrape_single_bundle(s: requests.Session, bundle: dict, force: bool) -> int:
|
|||||||
if not html:
|
if not html:
|
||||||
print(f" ! {slug}: empty body", file=sys.stderr)
|
print(f" ! {slug}: empty body", file=sys.stderr)
|
||||||
return 0
|
return 0
|
||||||
body_md = html_to_md(html)
|
body_md = html_to_md(html, s)
|
||||||
sidecar = {
|
sidecar = {
|
||||||
"bundle_id": slug,
|
"bundle_id": slug,
|
||||||
"page_id": doc_id,
|
"page_id": doc_id,
|
||||||
@@ -326,6 +363,8 @@ def main() -> int:
|
|||||||
else:
|
else:
|
||||||
total += scrape_toc_bundle(s, b, args.force, args.concurrency)
|
total += scrape_toc_bundle(s, b, args.force, args.concurrency)
|
||||||
print(f"scraped {total} new/updated pages", file=sys.stderr)
|
print(f"scraped {total} new/updated pages", file=sys.stderr)
|
||||||
|
if vision.enabled():
|
||||||
|
print(vision.summary(), file=sys.stderr)
|
||||||
|
|
||||||
# Always finalize after a scrape so sidecars are consistent.
|
# Always finalize after a scrape so sidecars are consistent.
|
||||||
all_bundles = json.loads(BUNDLES_JSON.read_text())
|
all_bundles = json.loads(BUNDLES_JSON.read_text())
|
||||||
|
|||||||
@@ -0,0 +1,269 @@
|
|||||||
|
"""Local-Ollama vision OCR for corpus images (opt-in, off by default).
|
||||||
|
|
||||||
|
Some HPE doc pages carry information *only* inside an image. The Morpheus
|
||||||
|
release schedule / support matrix (sf000111242en_us) is the canonical case:
|
||||||
|
the entire lifecycle table — which version ships when, when a stream hits
|
||||||
|
Maintenance/EOL — is a JPEG, so html_to_md would drop every value and
|
||||||
|
retrieval could never answer "when does Morpheus 8.x.x reach end of life".
|
||||||
|
|
||||||
|
When VISION_OCR=1, `transcribe()` sends qualifying images to a local vision
|
||||||
|
model (qwen2.5vl:7b on the host Ollama by default) and returns a labeled
|
||||||
|
markdown block that html_to_md appends to the page, so the text gets chunked,
|
||||||
|
embedded and retrieved.
|
||||||
|
|
||||||
|
Model choice (measured on the release-schedule matrix, known ground truth):
|
||||||
|
qwen2.5vl:7b scored 55/56 cells with a bare prompt, ~18s/image, fully on GPU,
|
||||||
|
and — unlike gemma3:12b — needs no hand-tuned prompt (gemma is accurate only
|
||||||
|
with an exact "read row by row, keep cells aligned" instruction). qwen2.5vl:32b
|
||||||
|
was no more accurate (same single miss) and 5x slower (CPU spill). See the
|
||||||
|
_PROMPT comment for the per-model wording lore.
|
||||||
|
|
||||||
|
Reliability (the whole point of this module):
|
||||||
|
- temperature 0 + a model/prompt pairing chosen against ground truth.
|
||||||
|
- Two samples with different seeds must AGREE after normalization
|
||||||
|
(self-consistency). On disagreement we take a third sample and keep the
|
||||||
|
majority, flagged `certain=False` so the reader knows to double-check.
|
||||||
|
Self-consistency catches RANDOM flakiness only — a model that reads a
|
||||||
|
table wrong does so identically every run — so every transcription also
|
||||||
|
ships with a "verify against the source image" caveat.
|
||||||
|
|
||||||
|
Cost control (a full --force re-scrape re-touches every page weekly):
|
||||||
|
- Content-hash cache in corpus/.vision-cache/ — each unique image is
|
||||||
|
OCR'd once, ever, and the result is committed so CI reuses it. The same
|
||||||
|
diagram shared across version bundles collapses to one OCR.
|
||||||
|
- VISION_MAX_NEW bounds NEW OCRs per run so the first run can't balloon
|
||||||
|
into hours; the cache fills incrementally over subsequent refreshes.
|
||||||
|
Deferred images are logged (never silently dropped).
|
||||||
|
|
||||||
|
Every failure path degrades to None — a down endpoint, a timeout, a decode
|
||||||
|
error — so the scrape never blocks on vision. Defaults target the git.jpaul.io
|
||||||
|
Ollama host; override via env for other deployments.
|
||||||
|
|
||||||
|
Config (env):
|
||||||
|
VISION_OCR "1" to enable (default off)
|
||||||
|
VISION_URL Ollama base URL (default http://192.168.0.2:11434)
|
||||||
|
VISION_MODEL vision model tag (default qwen2.5vl:7b)
|
||||||
|
VISION_PROMPT override the OCR prompt (default is tuned for qwen2.5vl)
|
||||||
|
VISION_MIN_W/H min image dims to OCR (default 600 x 300 — skips icons)
|
||||||
|
VISION_MAX_NEW new OCRs per run (budget) (default 60)
|
||||||
|
VISION_NUM_CTX Ollama context window (default 8192)
|
||||||
|
VISION_TIMEOUT per-call seconds (default 300)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
CACHE_DIR = ROOT / "corpus" / ".vision-cache"
|
||||||
|
|
||||||
|
VISION_OCR = os.environ.get("VISION_OCR", "") == "1"
|
||||||
|
VISION_URL = os.environ.get("VISION_URL", "http://192.168.0.2:11434").rstrip("/")
|
||||||
|
VISION_MODEL = os.environ.get("VISION_MODEL", "qwen2.5vl:7b")
|
||||||
|
VISION_MIN_W = int(os.environ.get("VISION_MIN_W", "600"))
|
||||||
|
VISION_MIN_H = int(os.environ.get("VISION_MIN_H", "300"))
|
||||||
|
VISION_MAX_NEW = int(os.environ.get("VISION_MAX_NEW", "60"))
|
||||||
|
VISION_TIMEOUT = int(os.environ.get("VISION_TIMEOUT", "300"))
|
||||||
|
VISION_NUM_CTX = int(os.environ.get("VISION_NUM_CTX", "8192"))
|
||||||
|
|
||||||
|
# Prompt wording is load-bearing, and the right wording is MODEL-SPECIFIC —
|
||||||
|
# measured on the release-schedule matrix (14 rows x 4 version columns, known
|
||||||
|
# ground truth). Errors were stable across seeds, so self-consistency does NOT
|
||||||
|
# catch them; only prompt/model choice does.
|
||||||
|
# - qwen2.5vl:7b (default): a bare "transcribe the table" instruction scores
|
||||||
|
# 55/56 cells; adding ANY extra clause (a non-table fallback, "preserve
|
||||||
|
# blank cells") regressed it to 51/56. So keep it minimal. qwen also reads
|
||||||
|
# it correctly WITHOUT hand-tuning — the prompt-robustness we want.
|
||||||
|
# - gemma3:12b: the opposite — the bare prompt shifts a column by one row;
|
||||||
|
# it needs an explicit "read row by row, keep cells aligned to their
|
||||||
|
# column" instruction to hit 55/56.
|
||||||
|
# The single cell every local model (incl. qwen2.5vl:32b) misses is the merged
|
||||||
|
# "9.1.2 - 9.3.2" range row. Set VISION_PROMPT to override for other models.
|
||||||
|
_DEFAULT_PROMPT = "Transcribe the table in this image to a GitHub markdown table."
|
||||||
|
_PROMPT = os.environ.get("VISION_PROMPT", _DEFAULT_PROMPT)
|
||||||
|
|
||||||
|
_lock = threading.Lock()
|
||||||
|
_new_ocr = 0
|
||||||
|
_stats = {
|
||||||
|
"cached": 0, "ocr": 0, "uncertain": 0,
|
||||||
|
"skipped_small": 0, "deferred": 0, "errors": 0, "empty": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def enabled() -> bool:
|
||||||
|
return VISION_OCR
|
||||||
|
|
||||||
|
|
||||||
|
def _bump(key: str, n: int = 1) -> None:
|
||||||
|
with _lock:
|
||||||
|
_stats[key] += n
|
||||||
|
|
||||||
|
|
||||||
|
def _dims(data: bytes) -> tuple[int, int] | None:
|
||||||
|
try:
|
||||||
|
from PIL import Image
|
||||||
|
with Image.open(io.BytesIO(data)) as im:
|
||||||
|
return im.size
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _generate(img_b64: str, seed: int) -> str | None:
|
||||||
|
body = {
|
||||||
|
"model": VISION_MODEL,
|
||||||
|
"prompt": _PROMPT,
|
||||||
|
"images": [img_b64],
|
||||||
|
"stream": False,
|
||||||
|
"keep_alive": "10m",
|
||||||
|
"options": {"temperature": 0, "seed": seed, "num_ctx": VISION_NUM_CTX},
|
||||||
|
}
|
||||||
|
r = requests.post(f"{VISION_URL}/api/generate", json=body, timeout=VISION_TIMEOUT)
|
||||||
|
r.raise_for_status()
|
||||||
|
return (r.json() or {}).get("response")
|
||||||
|
|
||||||
|
|
||||||
|
def _clean(s: str) -> str:
|
||||||
|
"""Extract just the transcription from a model response, dropping chatty
|
||||||
|
preamble ("Here's the transcription:") and code fences. gemma3 in
|
||||||
|
particular wraps output in ```markdown fences and a lead-in sentence
|
||||||
|
despite being told not to."""
|
||||||
|
s = s.strip()
|
||||||
|
# Prefer the contents of a fenced block if the model wrapped its answer.
|
||||||
|
m = re.search(r"```[a-zA-Z]*\n(.*?)```", s, re.S)
|
||||||
|
if m:
|
||||||
|
s = m.group(1).strip()
|
||||||
|
# For a table, drop any preamble before the first table row.
|
||||||
|
lines = s.splitlines()
|
||||||
|
for i, ln in enumerate(lines):
|
||||||
|
if ln.lstrip().startswith("|"):
|
||||||
|
return "\n".join(lines[i:]).strip()
|
||||||
|
return s.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize(s: str) -> str:
|
||||||
|
"""Collapse to a comparable form so two OCR samples that differ only in
|
||||||
|
whitespace/case count as agreeing (compared on cleaned text)."""
|
||||||
|
return re.sub(r"\s+", " ", s).strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _ocr_bytes(data: bytes) -> tuple[str | None, bool]:
|
||||||
|
"""Two-sample self-consistency → (cleaned_markdown, certain). A third
|
||||||
|
sample breaks a tie; `certain` is True only when >=2 samples agree.
|
||||||
|
|
||||||
|
NOTE: self-consistency catches RANDOM flakiness, not SYSTEMATIC error — a
|
||||||
|
prompt/model that reads a table wrong tends to do so identically every
|
||||||
|
time (we saw exactly this). It is a stability check, not a correctness
|
||||||
|
proof; the transcription is always emitted with a verify-against-source
|
||||||
|
caveat. The real correctness lever is the prompt/model choice."""
|
||||||
|
b64 = base64.b64encode(data).decode()
|
||||||
|
a = _clean(_generate(b64, 7) or "")
|
||||||
|
b = _clean(_generate(b64, 99) or "")
|
||||||
|
if a and b and _normalize(a) == _normalize(b):
|
||||||
|
return a, True
|
||||||
|
cands = [x for x in (a, b) if x]
|
||||||
|
if not cands:
|
||||||
|
return None, False
|
||||||
|
c = _clean(_generate(b64, 1234) or "")
|
||||||
|
if c:
|
||||||
|
cands.append(c)
|
||||||
|
groups: dict[str, list[str]] = {}
|
||||||
|
for x in cands:
|
||||||
|
groups.setdefault(_normalize(x), []).append(x)
|
||||||
|
best = max(groups.values(), key=len)
|
||||||
|
return best[0], len(best) >= 2
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_path(sha: str) -> Path:
|
||||||
|
return CACHE_DIR / f"{sha}.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _write_cache(sha: str, url: str, markdown: str | None, certain: bool) -> None:
|
||||||
|
try:
|
||||||
|
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
_cache_path(sha).write_text(json.dumps({
|
||||||
|
"sha": sha, "src": url, "model": VISION_MODEL,
|
||||||
|
"certain": certain, "markdown": markdown,
|
||||||
|
}, indent=2) + "\n")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _format(markdown: str, certain: bool) -> str:
|
||||||
|
note = (f"Transcribed from the image above by local vision OCR "
|
||||||
|
f"({VISION_MODEL}); verify against the source image.")
|
||||||
|
if not certain:
|
||||||
|
note += " NOTE: OCR samples disagreed — treat values as approximate."
|
||||||
|
return f"\n> {note}\n\n{markdown}\n"
|
||||||
|
|
||||||
|
|
||||||
|
def transcribe(img_url: str, session: requests.Session) -> str | None:
|
||||||
|
"""Return a labeled markdown transcription block for `img_url`, or None if
|
||||||
|
vision is disabled, the image is too small, the budget is spent, the image
|
||||||
|
has no readable text, or anything errors. Never raises."""
|
||||||
|
global _new_ocr
|
||||||
|
if not VISION_OCR:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
resp = session.get(img_url, timeout=60)
|
||||||
|
if resp.status_code != 200 or not resp.content:
|
||||||
|
return None
|
||||||
|
data = resp.content
|
||||||
|
except Exception:
|
||||||
|
_bump("errors")
|
||||||
|
return None
|
||||||
|
|
||||||
|
sha = hashlib.sha256(data).hexdigest()
|
||||||
|
cp = _cache_path(sha)
|
||||||
|
if cp.exists():
|
||||||
|
try:
|
||||||
|
entry = json.loads(cp.read_text())
|
||||||
|
except Exception:
|
||||||
|
entry = None
|
||||||
|
if entry is not None:
|
||||||
|
_bump("cached")
|
||||||
|
md = entry.get("markdown")
|
||||||
|
return _format(md, entry.get("certain", True)) if md else None
|
||||||
|
|
||||||
|
dims = _dims(data)
|
||||||
|
if not dims or dims[0] < VISION_MIN_W or dims[1] < VISION_MIN_H:
|
||||||
|
_bump("skipped_small")
|
||||||
|
return None
|
||||||
|
|
||||||
|
with _lock:
|
||||||
|
if _new_ocr >= VISION_MAX_NEW:
|
||||||
|
_stats["deferred"] += 1
|
||||||
|
return None
|
||||||
|
_new_ocr += 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
markdown, certain = _ocr_bytes(data)
|
||||||
|
except Exception:
|
||||||
|
_bump("errors")
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not markdown or len(markdown.strip()) < 20:
|
||||||
|
# Negative-cache tiny/no-text images so we never re-spend budget on them.
|
||||||
|
_write_cache(sha, img_url, None, True)
|
||||||
|
_bump("empty")
|
||||||
|
return None
|
||||||
|
|
||||||
|
_write_cache(sha, img_url, markdown, certain)
|
||||||
|
_bump("ocr")
|
||||||
|
if not certain:
|
||||||
|
_bump("uncertain")
|
||||||
|
return _format(markdown, certain)
|
||||||
|
|
||||||
|
|
||||||
|
def summary() -> str:
|
||||||
|
parts = ", ".join(f"{k}={v}" for k, v in _stats.items())
|
||||||
|
return f"vision[{VISION_MODEL} @ {VISION_URL}]: {parts}"
|
||||||
Reference in New Issue
Block a user