build out morpheus-docs MCP stack, mirroring hvm-docs through Phases 1-13
Initial scaffold: the docs-mcp-template clone with all the
HVM-validated stack ported across, customized for Morpheus
Enterprise (PRODUCT_NAME=morpheus, server name morpheus-docs).
Bundles (live-discovered 2026-05-22; 1710 cataloged pages total):
* morpheus_user_manual_8_1_0 sd00007510en_us 568 pages (Feb 2026)
* morpheus_user_manual_8_1_1 sd00007621en_us 569 pages (Mar 2026)
* morpheus_user_manual_8_1_2 sd00007732en_us 569 pages (Apr 2026)
* morpheus_release_notes_8_1_0 sd00007496en_us single-doc
* morpheus_release_notes_8_1_1 sd00007610en_us single-doc
* morpheus_release_notes_8_1_2 sd00007733en_us single-doc
* morpheus_quickspecs a50009231enw html-file (live
curl_cffi against www.hpe.com; all 12+ Enterprise SKUs captured —
S6E64..S6E73AAE for new/renewal/upgrade × 1/3/5-yr terms, plus
services SKUs HA124A1#V38/V39 and H46SBA1).
No Deployment Guide or Qualification Matrix on HPE Support for
Morpheus Enterprise specifically — the only QM (sd00006551en_us)
covers HVM clusters managed by Morpheus and lives in hvm-docs.
Stack carried forward from hvm-docs:
* rag/{index,chunk,embeddings,bm25}.py — including the
MAX_CHARS=4000 chunk-cap fix for table-dense content
* docs_mcp/{server,usage}.py — 11 MCP tools, BM25-default search,
cross-encoder rerank, hybrid behind HYBRID_SEARCH=true,
morpheus_api_lessons (renamed from hvm_api_lessons), env-gated
submit_doc_bug
* docs_mcp/api_lessons.md — Morpheus-specific scaffold covering
licensing model, HVM elevation path, REST vs Plugin API, with
TODO markers for sections to flesh out from real ops experience
* scrape/{runner,quickspecs,changelog,bundles}.py — TOC + single-doc
+ html-file modes, curl_cffi Chrome120 for www.hpe.com edge bypass
* eval/{retrievers,run_eval}.py + queries.jsonl scaffold (4 placeholder
queries; populate after first scrape)
* scripts/{rerank_server,usage_report,registry_gc}.py
* .gitea/workflows/{refresh,image-only}.yml — same Gitea Actions
setup zerto-docs uses (push LAN, pull public-URL, GPU Ollama pool)
* deploy/docker-compose.yml — morpheus-docs-mcp service definition,
shared jina-rerank sidecar, Watchtower-labeled
* Dockerfile, requirements.txt, requirements-rerank.txt
Verified locally: scrape produced 1599 .md pages (some TOC entries
are parent-only and yield no body), 6353 chunks all under the 4 KB
cap, MCP server boots and lists 11 tools cleanly.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,120 @@
|
||||
"""Minimal HTTP reranker — `/v1/rerank` endpoint over a sentence-transformers CrossEncoder.
|
||||
|
||||
Matches the Cohere `/v1/rerank` request/response shape, which is what the
|
||||
server's `_rerank()` helper expects. This is the dev-friendly fallback;
|
||||
production replaces this with the llama.cpp + jina-reranker-v2-base GGUF
|
||||
sidecar (see deploy/docker-compose.yml) without changing the client.
|
||||
|
||||
Request:
|
||||
POST /v1/rerank
|
||||
{"model": "...", "query": "...", "documents": ["text", ...], "top_n": 10}
|
||||
|
||||
Response:
|
||||
{"model": "...", "results": [{"index": 0, "relevance_score": 0.93}, ...]}
|
||||
|
||||
Usage:
|
||||
python -m scripts.rerank_server # localhost:8001
|
||||
RERANK_MODEL=cross-encoder/ms-marco-MiniLM-L-12-v2 \\
|
||||
RERANK_PORT=8001 python -m scripts.rerank_server
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
||||
|
||||
MODEL_NAME = os.environ.get("RERANK_MODEL", "cross-encoder/ms-marco-MiniLM-L-6-v2")
|
||||
PORT = int(os.environ.get("RERANK_PORT", "8001"))
|
||||
HOST = os.environ.get("RERANK_HOST", "127.0.0.1")
|
||||
# Truncate docs to this many chars before scoring. jina-reranker GGUF has a
|
||||
# 1024-token per-pair cap that 400s the entire batch; ms-marco is more
|
||||
# forgiving but we still cap to keep latency predictable.
|
||||
MAX_DOC_CHARS = int(os.environ.get("RERANK_MAX_DOC_CHARS", "2000"))
|
||||
|
||||
_model = None
|
||||
|
||||
|
||||
def _get_model():
|
||||
global _model
|
||||
if _model is None:
|
||||
from sentence_transformers import CrossEncoder
|
||||
log.info("loading %s", MODEL_NAME)
|
||||
_model = CrossEncoder(MODEL_NAME)
|
||||
log.info("loaded")
|
||||
return _model
|
||||
|
||||
|
||||
def _rerank(query: str, documents: list[str], top_n: int | None) -> list[dict]:
|
||||
model = _get_model()
|
||||
pairs = [[query, (d or "")[:MAX_DOC_CHARS]] for d in documents]
|
||||
scores = model.predict(pairs)
|
||||
ranked = sorted(
|
||||
({"index": i, "relevance_score": float(s)} for i, s in enumerate(scores)),
|
||||
key=lambda r: -r["relevance_score"],
|
||||
)
|
||||
if top_n is not None:
|
||||
ranked = ranked[:top_n]
|
||||
return ranked
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args):
|
||||
log.info("%s - %s", self.address_string(), fmt % args)
|
||||
|
||||
def _send_json(self, status: int, payload: dict) -> None:
|
||||
body = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self): # noqa: N802
|
||||
if self.path in ("/", "/health"):
|
||||
self._send_json(200, {"status": "ok", "model": MODEL_NAME})
|
||||
return
|
||||
self._send_json(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self): # noqa: N802
|
||||
if self.path not in ("/v1/rerank", "/rerank"):
|
||||
self._send_json(404, {"error": "not found"})
|
||||
return
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
try:
|
||||
req = json.loads(self.rfile.read(length).decode())
|
||||
except Exception as e:
|
||||
self._send_json(400, {"error": f"bad json: {e}"})
|
||||
return
|
||||
query = req.get("query")
|
||||
documents = req.get("documents")
|
||||
if not isinstance(query, str) or not isinstance(documents, list):
|
||||
self._send_json(400, {"error": "expected {query: str, documents: list[str]}"})
|
||||
return
|
||||
top_n = req.get("top_n")
|
||||
try:
|
||||
results = _rerank(query, documents, top_n if isinstance(top_n, int) else None)
|
||||
except Exception as e:
|
||||
log.exception("rerank failed")
|
||||
self._send_json(500, {"error": str(e)})
|
||||
return
|
||||
self._send_json(200, {"model": MODEL_NAME, "results": results})
|
||||
|
||||
|
||||
def main() -> int:
|
||||
_get_model() # warm-load before accepting traffic
|
||||
server = ThreadingHTTPServer((HOST, PORT), Handler)
|
||||
log.info("listening on http://%s:%d", HOST, PORT)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
log.info("shutting down")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user