274 lines
10 KiB
Python
274 lines
10 KiB
Python
"""Local-Ollama vision OCR for corpus images (opt-in, off by default).
|
|
|
|
Some HPE doc pages carry information *only* inside an image. The Morpheus
|
|
release schedule / support matrix (sf000111242en_us) is the canonical case:
|
|
the entire lifecycle table — which version ships when, when a stream hits
|
|
Maintenance/EOL — is a JPEG, so html_to_md would drop every value and
|
|
retrieval could never answer "when does Morpheus 8.x.x reach end of life".
|
|
|
|
When VISION_OCR=1, `transcribe()` sends qualifying images to a local vision
|
|
model (qwen2.5vl:7b on the host Ollama by default) and returns a labeled
|
|
markdown block that html_to_md appends to the page, so the text gets chunked,
|
|
embedded and retrieved.
|
|
|
|
Model choice (measured on the release-schedule matrix, known ground truth):
|
|
qwen2.5vl:7b scored 55/56 cells with a bare prompt, ~18s/image, fully on GPU,
|
|
and — unlike gemma3:12b — needs no hand-tuned prompt (gemma is accurate only
|
|
with an exact "read row by row, keep cells aligned" instruction). qwen2.5vl:32b
|
|
was no more accurate (same single miss) and 5x slower (CPU spill). See the
|
|
_PROMPT comment for the per-model wording lore.
|
|
|
|
Reliability (the whole point of this module):
|
|
- temperature 0 + a model/prompt pairing chosen against ground truth.
|
|
- Two samples with different seeds must AGREE after normalization
|
|
(self-consistency). On disagreement we take a third sample and keep the
|
|
majority, flagged `certain=False` so the reader knows to double-check.
|
|
Self-consistency catches RANDOM flakiness only — a model that reads a
|
|
table wrong does so identically every run — so every transcription also
|
|
ships with a "verify against the source image" caveat.
|
|
|
|
Scope: OCR is a per-bundle opt-in (BundleSpec.ocr in scrape/bundles.py), NOT
|
|
a corpus-wide sweep. Image-only data is rare — only the release-schedule
|
|
matrix in this corpus; every other image is a product UI screenshot that OCR
|
|
would just turn into a noise "table". So only allowlisted bundles reach this
|
|
module; the runner passes no session for the rest.
|
|
|
|
Cost control (secondary, now that scope is an allowlist):
|
|
- Content-hash cache in corpus/.vision-cache/ — each unique image is
|
|
OCR'd once, ever, and the result is committed so CI reuses it.
|
|
- VISION_MAX_NEW still bounds NEW OCRs per run as a safety net; with the
|
|
allowlist it rarely binds. Deferred images are logged, never dropped.
|
|
|
|
Every failure path degrades to None — a down endpoint, a timeout, a decode
|
|
error — so the scrape never blocks on vision. Defaults target the git.jpaul.io
|
|
Ollama host; override via env for other deployments.
|
|
|
|
Config (env):
|
|
VISION_OCR "1" to enable (default off)
|
|
VISION_URL Ollama base URL (default http://192.168.0.2:11434)
|
|
VISION_MODEL vision model tag (default qwen2.5vl:7b)
|
|
VISION_PROMPT override the OCR prompt (default is tuned for qwen2.5vl)
|
|
VISION_MIN_W/H min image dims to OCR (default 600 x 300 — skips icons)
|
|
VISION_MAX_NEW new OCRs per run (budget) (default 60)
|
|
VISION_NUM_CTX Ollama context window (default 8192)
|
|
VISION_TIMEOUT per-call seconds (default 300)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import threading
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
CACHE_DIR = ROOT / "corpus" / ".vision-cache"
|
|
|
|
VISION_OCR = os.environ.get("VISION_OCR", "") == "1"
|
|
VISION_URL = os.environ.get("VISION_URL", "http://192.168.0.2:11434").rstrip("/")
|
|
VISION_MODEL = os.environ.get("VISION_MODEL", "qwen2.5vl:7b")
|
|
VISION_MIN_W = int(os.environ.get("VISION_MIN_W", "600"))
|
|
VISION_MIN_H = int(os.environ.get("VISION_MIN_H", "300"))
|
|
VISION_MAX_NEW = int(os.environ.get("VISION_MAX_NEW", "60"))
|
|
VISION_TIMEOUT = int(os.environ.get("VISION_TIMEOUT", "300"))
|
|
VISION_NUM_CTX = int(os.environ.get("VISION_NUM_CTX", "8192"))
|
|
|
|
# Prompt wording is load-bearing, and the right wording is MODEL-SPECIFIC —
|
|
# measured on the release-schedule matrix (14 rows x 4 version columns, known
|
|
# ground truth). Errors were stable across seeds, so self-consistency does NOT
|
|
# catch them; only prompt/model choice does.
|
|
# - qwen2.5vl:7b (default): a bare "transcribe the table" instruction scores
|
|
# 55/56 cells; adding ANY extra clause (a non-table fallback, "preserve
|
|
# blank cells") regressed it to 51/56. So keep it minimal. qwen also reads
|
|
# it correctly WITHOUT hand-tuning — the prompt-robustness we want.
|
|
# - gemma3:12b: the opposite — the bare prompt shifts a column by one row;
|
|
# it needs an explicit "read row by row, keep cells aligned to their
|
|
# column" instruction to hit 55/56.
|
|
# The single cell every local model (incl. qwen2.5vl:32b) misses is the merged
|
|
# "9.1.2 - 9.3.2" range row. Set VISION_PROMPT to override for other models.
|
|
_DEFAULT_PROMPT = "Transcribe the table in this image to a GitHub markdown table."
|
|
_PROMPT = os.environ.get("VISION_PROMPT", _DEFAULT_PROMPT)
|
|
|
|
_lock = threading.Lock()
|
|
_new_ocr = 0
|
|
_stats = {
|
|
"cached": 0, "ocr": 0, "uncertain": 0,
|
|
"skipped_small": 0, "deferred": 0, "errors": 0, "empty": 0,
|
|
}
|
|
|
|
|
|
def enabled() -> bool:
|
|
return VISION_OCR
|
|
|
|
|
|
def _bump(key: str, n: int = 1) -> None:
|
|
with _lock:
|
|
_stats[key] += n
|
|
|
|
|
|
def _dims(data: bytes) -> tuple[int, int] | None:
|
|
try:
|
|
from PIL import Image
|
|
with Image.open(io.BytesIO(data)) as im:
|
|
return im.size
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _generate(img_b64: str, seed: int) -> str | None:
|
|
body = {
|
|
"model": VISION_MODEL,
|
|
"prompt": _PROMPT,
|
|
"images": [img_b64],
|
|
"stream": False,
|
|
"keep_alive": "10m",
|
|
"options": {"temperature": 0, "seed": seed, "num_ctx": VISION_NUM_CTX},
|
|
}
|
|
r = requests.post(f"{VISION_URL}/api/generate", json=body, timeout=VISION_TIMEOUT)
|
|
r.raise_for_status()
|
|
return (r.json() or {}).get("response")
|
|
|
|
|
|
def _clean(s: str) -> str:
|
|
"""Extract just the transcription from a model response, dropping chatty
|
|
preamble ("Here's the transcription:") and code fences. gemma3 in
|
|
particular wraps output in ```markdown fences and a lead-in sentence
|
|
despite being told not to."""
|
|
s = s.strip()
|
|
# Prefer the contents of a fenced block if the model wrapped its answer.
|
|
m = re.search(r"```[a-zA-Z]*\n(.*?)```", s, re.S)
|
|
if m:
|
|
s = m.group(1).strip()
|
|
# For a table, drop any preamble before the first table row.
|
|
lines = s.splitlines()
|
|
for i, ln in enumerate(lines):
|
|
if ln.lstrip().startswith("|"):
|
|
return "\n".join(lines[i:]).strip()
|
|
return s.strip()
|
|
|
|
|
|
def _normalize(s: str) -> str:
|
|
"""Collapse to a comparable form so two OCR samples that differ only in
|
|
whitespace/case count as agreeing (compared on cleaned text)."""
|
|
return re.sub(r"\s+", " ", s).strip().lower()
|
|
|
|
|
|
def _ocr_bytes(data: bytes) -> tuple[str | None, bool]:
|
|
"""Two-sample self-consistency → (cleaned_markdown, certain). A third
|
|
sample breaks a tie; `certain` is True only when >=2 samples agree.
|
|
|
|
NOTE: self-consistency catches RANDOM flakiness, not SYSTEMATIC error — a
|
|
prompt/model that reads a table wrong tends to do so identically every
|
|
time (we saw exactly this). It is a stability check, not a correctness
|
|
proof; the transcription is always emitted with a verify-against-source
|
|
caveat. The real correctness lever is the prompt/model choice."""
|
|
b64 = base64.b64encode(data).decode()
|
|
a = _clean(_generate(b64, 7) or "")
|
|
b = _clean(_generate(b64, 99) or "")
|
|
if a and b and _normalize(a) == _normalize(b):
|
|
return a, True
|
|
cands = [x for x in (a, b) if x]
|
|
if not cands:
|
|
return None, False
|
|
c = _clean(_generate(b64, 1234) or "")
|
|
if c:
|
|
cands.append(c)
|
|
groups: dict[str, list[str]] = {}
|
|
for x in cands:
|
|
groups.setdefault(_normalize(x), []).append(x)
|
|
best = max(groups.values(), key=len)
|
|
return best[0], len(best) >= 2
|
|
|
|
|
|
def _cache_path(sha: str) -> Path:
|
|
return CACHE_DIR / f"{sha}.json"
|
|
|
|
|
|
def _write_cache(sha: str, url: str, markdown: str | None, certain: bool) -> None:
|
|
try:
|
|
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
|
_cache_path(sha).write_text(json.dumps({
|
|
"sha": sha, "src": url, "model": VISION_MODEL,
|
|
"certain": certain, "markdown": markdown,
|
|
}, indent=2) + "\n")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _format(markdown: str, certain: bool) -> str:
|
|
note = (f"Transcribed from the image above by local vision OCR "
|
|
f"({VISION_MODEL}); verify against the source image.")
|
|
if not certain:
|
|
note += " NOTE: OCR samples disagreed — treat values as approximate."
|
|
return f"\n> {note}\n\n{markdown}\n"
|
|
|
|
|
|
def transcribe(img_url: str, session: requests.Session) -> str | None:
|
|
"""Return a labeled markdown transcription block for `img_url`, or None if
|
|
vision is disabled, the image is too small, the budget is spent, the image
|
|
has no readable text, or anything errors. Never raises."""
|
|
global _new_ocr
|
|
if not VISION_OCR:
|
|
return None
|
|
|
|
try:
|
|
resp = session.get(img_url, timeout=60)
|
|
if resp.status_code != 200 or not resp.content:
|
|
return None
|
|
data = resp.content
|
|
except Exception:
|
|
_bump("errors")
|
|
return None
|
|
|
|
sha = hashlib.sha256(data).hexdigest()
|
|
cp = _cache_path(sha)
|
|
if cp.exists():
|
|
try:
|
|
entry = json.loads(cp.read_text())
|
|
except Exception:
|
|
entry = None
|
|
if entry is not None:
|
|
_bump("cached")
|
|
md = entry.get("markdown")
|
|
return _format(md, entry.get("certain", True)) if md else None
|
|
|
|
dims = _dims(data)
|
|
if not dims or dims[0] < VISION_MIN_W or dims[1] < VISION_MIN_H:
|
|
_bump("skipped_small")
|
|
return None
|
|
|
|
with _lock:
|
|
if _new_ocr >= VISION_MAX_NEW:
|
|
_stats["deferred"] += 1
|
|
return None
|
|
_new_ocr += 1
|
|
|
|
try:
|
|
markdown, certain = _ocr_bytes(data)
|
|
except Exception:
|
|
_bump("errors")
|
|
return None
|
|
|
|
if not markdown or len(markdown.strip()) < 20:
|
|
# Negative-cache tiny/no-text images so we never re-spend budget on them.
|
|
_write_cache(sha, img_url, None, True)
|
|
_bump("empty")
|
|
return None
|
|
|
|
_write_cache(sha, img_url, markdown, certain)
|
|
_bump("ocr")
|
|
if not certain:
|
|
_bump("uncertain")
|
|
return _format(markdown, certain)
|
|
|
|
|
|
def summary() -> str:
|
|
parts = ", ".join(f"{k}={v}" for k, v in _stats.items())
|
|
return f"vision[{VISION_MODEL} @ {VISION_URL}]: {parts}"
|