build out morpheus-docs MCP stack, mirroring hvm-docs through Phases 1-13

Initial scaffold: the docs-mcp-template clone with all the HVM-validated stack ported across, customized for Morpheus Enterprise (PRODUCT_NAME=morpheus, server name morpheus-docs). Bundles (live-discovered 2026-05-22; 1710 cataloged pages total): * morpheus_user_manual_8_1_0 sd00007510en_us 568 pages (Feb 2026) * morpheus_user_manual_8_1_1 sd00007621en_us 569 pages (Mar 2026) * morpheus_user_manual_8_1_2 sd00007732en_us 569 pages (Apr 2026) * morpheus_release_notes_8_1_0 sd00007496en_us single-doc * morpheus_release_notes_8_1_1 sd00007610en_us single-doc * morpheus_release_notes_8_1_2 sd00007733en_us single-doc * morpheus_quickspecs a50009231enw html-file (live curl_cffi against www.hpe.com; all 12+ Enterprise SKUs captured — S6E64..S6E73AAE for new/renewal/upgrade × 1/3/5-yr terms, plus services SKUs HA124A1#V38/V39 and H46SBA1). No Deployment Guide or Qualification Matrix on HPE Support for Morpheus Enterprise specifically — the only QM (sd00006551en_us) covers HVM clusters managed by Morpheus and lives in hvm-docs. Stack carried forward from hvm-docs: * rag/{index,chunk,embeddings,bm25}.py — including the MAX_CHARS=4000 chunk-cap fix for table-dense content * docs_mcp/{server,usage}.py — 11 MCP tools, BM25-default search, cross-encoder rerank, hybrid behind HYBRID_SEARCH=true, morpheus_api_lessons (renamed from hvm_api_lessons), env-gated submit_doc_bug * docs_mcp/api_lessons.md — Morpheus-specific scaffold covering licensing model, HVM elevation path, REST vs Plugin API, with TODO markers for sections to flesh out from real ops experience * scrape/{runner,quickspecs,changelog,bundles}.py — TOC + single-doc + html-file modes, curl_cffi Chrome120 for www.hpe.com edge bypass * eval/{retrievers,run_eval}.py + queries.jsonl scaffold (4 placeholder queries; populate after first scrape) * scripts/{rerank_server,usage_report,registry_gc}.py * .gitea/workflows/{refresh,image-only}.yml — same Gitea Actions setup zerto-docs uses (push LAN, pull public-URL, GPU Ollama pool) * deploy/docker-compose.yml — morpheus-docs-mcp service definition, shared jina-rerank sidecar, Watchtower-labeled * Dockerfile, requirements.txt, requirements-rerank.txt Verified locally: scrape produced 1599 .md pages (some TOC entries are parent-only and yield no body), 6353 chunks all under the 4 KB cap, MCP server boots and lists 11 tools cleanly. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-22 15:26:24 -04:00
parent 43728320bf
commit fa448f94e1
22 changed files with 2822 additions and 247 deletions
@@ -31,6 +31,31 @@ from typing import Iterator
 CHARS_PER_TOKEN = 4
 TARGET_TOKENS = 500
 TARGET_CHARS = TARGET_TOKENS * CHARS_PER_TOKEN
+# Hard cap: nomic-embed-text's context is 2048 tokens. Anything larger
+# 400s the entire embed batch. 6000 chars works for prose but markdown
+# tables with lots of `|` separators tokenize ~1.4× denser; a 5839-char
+# table chunk from the HVM qualification matrix tokenized past 2048 and
+# crashed the rebuild. 4000 chars stays under 2048 tokens even for
+# dense table content while leaving headroom for the query side.
+MAX_CHARS = 4000
+
+
+def _hard_split(text: str) -> list[str]:
+    """Split an oversized block on line boundaries into MAX_CHARS pieces."""
+    if len(text) <= MAX_CHARS:
+        return [text]
+    out: list[str] = []
+    buf: list[str] = []
+    buf_chars = 0
+    for line in text.splitlines(keepends=True):
+        if buf_chars + len(line) > MAX_CHARS and buf:
+            out.append("".join(buf).rstrip())
+            buf, buf_chars = [], 0
+        buf.append(line)
+        buf_chars += len(line)
+    if buf:
+        out.append("".join(buf).rstrip())
+    return out


 def estimate_tokens(text: str) -> int:
@@ -104,23 +129,26 @@ def chunks_from_page(

    # ----- Body chunks: pack paragraphs up to TARGET_CHARS -------
    ordinal = 1
+
+    def emit(buf: list[str]) -> Iterator[dict]:
+        nonlocal ordinal
+        merged = "\n\n".join(buf)
+        for piece in _hard_split(merged):
+            yield {
+                "id":       f"{metadata['bundle_id']}::{page_id}::{ordinal}",
+                "text":     piece,
+                "metadata": {**metadata, "ordinal": ordinal},
+            }
+            ordinal += 1
+
    buf: list[str] = []
    buf_chars = 0
    for p in paragraphs:
        if buf_chars + len(p) > TARGET_CHARS and buf:
-            yield {
-                "id":       f"{metadata['bundle_id']}::{page_id}::{ordinal}",
-                "text":     "\n\n".join(buf),
-                "metadata": {**metadata, "ordinal": ordinal},
-            }
-            ordinal += 1
+            yield from emit(buf)
            buf = []
            buf_chars = 0
        buf.append(p)
        buf_chars += len(p)
    if buf:
-        yield {
-            "id":       f"{metadata['bundle_id']}::{page_id}::{ordinal}",
-            "text":     "\n\n".join(buf),
-            "metadata": {**metadata, "ordinal": ordinal},
-        }
+        yield from emit(buf)
@@ -3,8 +3,15 @@
 Swappable: implement the same `embedding_function()` interface returning
 a Chroma `EmbeddingFunction` and the rest of the pipeline doesn't care.

-Defaults (override via env):
-  OLLAMA_URL    one or more comma-separated URLs (load-balanced)
+Env-configurable (matches the zerto-docs-rag pattern so the same Gitea
+runner + GPU-pinned Ollama containers can serve every docs MCP build):
+
+  OLLAMA_URLS   comma-separated list, load-balanced round-robin per batch.
+                Preferred — set in the CI workflow to fan out across two
+                GPU-pinned Ollama containers on the Gitea host.
+  OLLAMA_URL    single endpoint, fallback when OLLAMA_URLS is unset.
+                Default http://192.168.0.2:11434 (the host where the GPUs
+                live in Justin's lab).
  EMBED_MODEL   model name; default 'nomic-embed-text'
  EMBED_DIM     expected embedding dim; default 768 (nomic-embed-text)
 """
@@ -19,8 +26,18 @@ from chromadb import EmbeddingFunction, Documents, Embeddings

 log = logging.getLogger(__name__)

-OLLAMA_URLS = [u.strip() for u in os.environ.get("OLLAMA_URL",
-               "http://localhost:11434").split(",") if u.strip()]
+DEFAULT_OLLAMA_URL = "http://192.168.0.2:11434"
+
+
+def _resolve_urls() -> list[str]:
+    raw = os.environ.get("OLLAMA_URLS", "").strip()
+    if raw:
+        return [u.strip().rstrip("/") for u in raw.split(",") if u.strip()]
+    single = os.environ.get("OLLAMA_URL", DEFAULT_OLLAMA_URL).strip().rstrip("/")
+    return [single]
+
+
+OLLAMA_URLS = _resolve_urls()
 EMBED_MODEL = os.environ.get("EMBED_MODEL", "nomic-embed-text")
 EMBED_DIM = int(os.environ.get("EMBED_DIM", "768"))

@@ -29,7 +29,7 @@ CHROMA_DIR = ROOT / "chroma"

 # Collection name — convention: <product>_docs. Override via env if needed.
 import os
-PRODUCT_NAME = os.environ.get("PRODUCT_NAME", "myproduct")
+PRODUCT_NAME = os.environ.get("PRODUCT_NAME", "morpheus")
 COLLECTION = f"{PRODUCT_NAME}_docs"