build out morpheus-docs MCP stack, mirroring hvm-docs through Phases 1-13
Initial scaffold: the docs-mcp-template clone with all the
HVM-validated stack ported across, customized for Morpheus
Enterprise (PRODUCT_NAME=morpheus, server name morpheus-docs).
Bundles (live-discovered 2026-05-22; 1710 cataloged pages total):
* morpheus_user_manual_8_1_0 sd00007510en_us 568 pages (Feb 2026)
* morpheus_user_manual_8_1_1 sd00007621en_us 569 pages (Mar 2026)
* morpheus_user_manual_8_1_2 sd00007732en_us 569 pages (Apr 2026)
* morpheus_release_notes_8_1_0 sd00007496en_us single-doc
* morpheus_release_notes_8_1_1 sd00007610en_us single-doc
* morpheus_release_notes_8_1_2 sd00007733en_us single-doc
* morpheus_quickspecs a50009231enw html-file (live
curl_cffi against www.hpe.com; all 12+ Enterprise SKUs captured —
S6E64..S6E73AAE for new/renewal/upgrade × 1/3/5-yr terms, plus
services SKUs HA124A1#V38/V39 and H46SBA1).
No Deployment Guide or Qualification Matrix on HPE Support for
Morpheus Enterprise specifically — the only QM (sd00006551en_us)
covers HVM clusters managed by Morpheus and lives in hvm-docs.
Stack carried forward from hvm-docs:
* rag/{index,chunk,embeddings,bm25}.py — including the
MAX_CHARS=4000 chunk-cap fix for table-dense content
* docs_mcp/{server,usage}.py — 11 MCP tools, BM25-default search,
cross-encoder rerank, hybrid behind HYBRID_SEARCH=true,
morpheus_api_lessons (renamed from hvm_api_lessons), env-gated
submit_doc_bug
* docs_mcp/api_lessons.md — Morpheus-specific scaffold covering
licensing model, HVM elevation path, REST vs Plugin API, with
TODO markers for sections to flesh out from real ops experience
* scrape/{runner,quickspecs,changelog,bundles}.py — TOC + single-doc
+ html-file modes, curl_cffi Chrome120 for www.hpe.com edge bypass
* eval/{retrievers,run_eval}.py + queries.jsonl scaffold (4 placeholder
queries; populate after first scrape)
* scripts/{rerank_server,usage_report,registry_gc}.py
* .gitea/workflows/{refresh,image-only}.yml — same Gitea Actions
setup zerto-docs uses (push LAN, pull public-URL, GPU Ollama pool)
* deploy/docker-compose.yml — morpheus-docs-mcp service definition,
shared jina-rerank sidecar, Watchtower-labeled
* Dockerfile, requirements.txt, requirements-rerank.txt
Verified locally: scrape produced 1599 .md pages (some TOC entries
are parent-only and yield no body), 6353 chunks all under the 4 KB
cap, MCP server boots and lists 11 tools cleanly.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+70
-41
@@ -1,42 +1,58 @@
|
||||
"""Gitea container-registry garbage collection.
|
||||
|
||||
Lists package versions for one container package and deletes versions
|
||||
older than --keep-days. Always preserves:
|
||||
Lists tagged versions of one container package and deletes old ones.
|
||||
Always preserves:
|
||||
|
||||
- the :latest tag
|
||||
- the --keep-latest most-recent date-tagged versions
|
||||
- anything pushed in the last --keep-days days
|
||||
- the `latest` tag (Watchtower's auto-deploy target)
|
||||
- the `--keep-latest` most-recent date-tagged versions (YYYY.MM.DD)
|
||||
- the `--keep-latest` most-recent short-SHA tags (rollback pins)
|
||||
- anything pushed within `--keep-days` days
|
||||
|
||||
The actual disk reclaim happens on Gitea's next package GC cron (admin
|
||||
site settings). This script just marks the versions for deletion.
|
||||
OCI blob-level versions (`sha256:...`) are never touched directly — those
|
||||
are managed by Gitea's internal package GC cron when their last tag
|
||||
goes away.
|
||||
|
||||
Usage:
|
||||
|
||||
python scripts/registry_gc.py \\
|
||||
--owner <user> \\
|
||||
--package <product>-docs-mcp \\
|
||||
GITEA_TOKEN=... python scripts/registry_gc.py \\
|
||||
--owner justin \\
|
||||
--package hvm-docs \\
|
||||
--keep-days 90 \\
|
||||
--keep-latest 5
|
||||
|
||||
Auth: reads GITEA_TOKEN from env (set in the workflow as a secret).
|
||||
The Gitea endpoint shape (confirmed 2026-05-22 against git.jpaul.io):
|
||||
|
||||
GET /api/v1/packages/{owner}/container/{package}
|
||||
-> [{id, version, created_at, ...}, ...]
|
||||
DELETE /api/v1/packages/{owner}/container/{package}/{version}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import HTTPError
|
||||
import json
|
||||
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
GITEA_HOST = os.environ.get("GITEA_HOST", "https://git.jpaul.io")
|
||||
DATE_TAG = re.compile(r"^\d{4}\.\d{2}\.\d{2}$")
|
||||
SHA_TAG = re.compile(r"^[0-9a-f]{7,40}$") # short or full git SHA
|
||||
BLOB_VER = re.compile(r"^sha256:") # OCI blob versions — skip
|
||||
|
||||
|
||||
def api(token: str, method: str, path: str) -> object:
|
||||
# Explicit User-Agent: git.jpaul.io is behind Cloudflare, whose default
|
||||
# Bot Fight Mode 403s `Python-urllib/X.Y` with error 1010. Any
|
||||
# recognizable browser/curl-style UA passes.
|
||||
req = Request(f"{GITEA_HOST}{path}",
|
||||
headers={"Authorization": f"token {token}"},
|
||||
headers={
|
||||
"Authorization": f"token {token}",
|
||||
"User-Agent": "hvm-docs-registry-gc/1.0",
|
||||
},
|
||||
method=method)
|
||||
try:
|
||||
with urlopen(req, timeout=30) as r:
|
||||
@@ -63,44 +79,57 @@ def main() -> int:
|
||||
return 1
|
||||
|
||||
versions = api(token, "GET",
|
||||
f"/api/v1/packages/{args.owner}/container/{args.package}/versions") or []
|
||||
f"/api/v1/packages/{args.owner}/container/{args.package}") or []
|
||||
if not versions:
|
||||
print(f"no versions found for {args.owner}/{args.package}")
|
||||
print(f"no versions found for {args.owner}/container/{args.package}")
|
||||
return 0
|
||||
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=args.keep_days)
|
||||
print(f" {len(versions)} version(s); cutoff={cutoff.isoformat()} "
|
||||
f"keep_days={args.keep_days} keep_latest={args.keep_latest}")
|
||||
|
||||
# Date-tagged versions (YYYY.MM.DD), newest first
|
||||
date_tagged = []
|
||||
for v in versions:
|
||||
tags = v.get("tags") or []
|
||||
for t in tags:
|
||||
if len(t) == 10 and t[4] == "." and t[7] == ".":
|
||||
date_tagged.append((t, v))
|
||||
break
|
||||
date_tagged.sort(key=lambda kv: kv[0], reverse=True)
|
||||
keep_date_tags = {t for t, _ in date_tagged[:args.keep_latest]}
|
||||
|
||||
deleted = 0
|
||||
for v in versions:
|
||||
tags = v.get("tags") or []
|
||||
if "latest" in tags:
|
||||
continue
|
||||
if any(t in keep_date_tags for t in tags):
|
||||
continue
|
||||
# Sort newest first by created_at.
|
||||
def parsed_ts(v: dict) -> datetime:
|
||||
try:
|
||||
created = datetime.fromisoformat(v["created_at"].replace("Z", "+00:00"))
|
||||
return datetime.fromisoformat(v["created_at"].replace("Z", "+00:00"))
|
||||
except (KeyError, ValueError):
|
||||
return datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
versions.sort(key=parsed_ts, reverse=True)
|
||||
|
||||
# Compute the keep-set: top-N date tags + top-N sha tags + always latest.
|
||||
keep_dates: list[str] = []
|
||||
keep_shas: list[str] = []
|
||||
for v in versions:
|
||||
ver = v.get("version") or ""
|
||||
if DATE_TAG.match(ver) and len(keep_dates) < args.keep_latest:
|
||||
keep_dates.append(ver)
|
||||
elif SHA_TAG.match(ver) and len(keep_shas) < args.keep_latest:
|
||||
keep_shas.append(ver)
|
||||
keep = {"latest", *keep_dates, *keep_shas}
|
||||
print(f" keep tags: {sorted(keep)}")
|
||||
|
||||
deleted = skipped_blob = skipped_age = skipped_keep = 0
|
||||
for v in versions:
|
||||
ver = v.get("version") or ""
|
||||
ts = parsed_ts(v)
|
||||
if BLOB_VER.match(ver):
|
||||
skipped_blob += 1
|
||||
continue
|
||||
if created >= cutoff:
|
||||
if ver in keep:
|
||||
skipped_keep += 1
|
||||
continue
|
||||
version_id = v.get("id")
|
||||
print(f" deleting v{version_id} tags={tags} created={v['created_at']}")
|
||||
if ts >= cutoff:
|
||||
skipped_age += 1
|
||||
continue
|
||||
print(f" deleting {ver!r} id={v.get('id')} created={v.get('created_at')}")
|
||||
if not args.dry_run:
|
||||
api(token, "DELETE",
|
||||
f"/api/v1/packages/{args.owner}/container/{args.package}/versions/{version_id}")
|
||||
f"/api/v1/packages/{args.owner}/container/{args.package}/{quote(ver, safe='')}")
|
||||
deleted += 1
|
||||
print(f"done: {deleted} version(s) deleted")
|
||||
|
||||
print(f"done: deleted={deleted} kept_named={skipped_keep} "
|
||||
f"kept_recent={skipped_age} skipped_blobs={skipped_blob}")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Minimal HTTP reranker — `/v1/rerank` endpoint over a sentence-transformers CrossEncoder.
|
||||
|
||||
Matches the Cohere `/v1/rerank` request/response shape, which is what the
|
||||
server's `_rerank()` helper expects. This is the dev-friendly fallback;
|
||||
production replaces this with the llama.cpp + jina-reranker-v2-base GGUF
|
||||
sidecar (see deploy/docker-compose.yml) without changing the client.
|
||||
|
||||
Request:
|
||||
POST /v1/rerank
|
||||
{"model": "...", "query": "...", "documents": ["text", ...], "top_n": 10}
|
||||
|
||||
Response:
|
||||
{"model": "...", "results": [{"index": 0, "relevance_score": 0.93}, ...]}
|
||||
|
||||
Usage:
|
||||
python -m scripts.rerank_server # localhost:8001
|
||||
RERANK_MODEL=cross-encoder/ms-marco-MiniLM-L-12-v2 \\
|
||||
RERANK_PORT=8001 python -m scripts.rerank_server
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
||||
|
||||
MODEL_NAME = os.environ.get("RERANK_MODEL", "cross-encoder/ms-marco-MiniLM-L-6-v2")
|
||||
PORT = int(os.environ.get("RERANK_PORT", "8001"))
|
||||
HOST = os.environ.get("RERANK_HOST", "127.0.0.1")
|
||||
# Truncate docs to this many chars before scoring. jina-reranker GGUF has a
|
||||
# 1024-token per-pair cap that 400s the entire batch; ms-marco is more
|
||||
# forgiving but we still cap to keep latency predictable.
|
||||
MAX_DOC_CHARS = int(os.environ.get("RERANK_MAX_DOC_CHARS", "2000"))
|
||||
|
||||
_model = None
|
||||
|
||||
|
||||
def _get_model():
|
||||
global _model
|
||||
if _model is None:
|
||||
from sentence_transformers import CrossEncoder
|
||||
log.info("loading %s", MODEL_NAME)
|
||||
_model = CrossEncoder(MODEL_NAME)
|
||||
log.info("loaded")
|
||||
return _model
|
||||
|
||||
|
||||
def _rerank(query: str, documents: list[str], top_n: int | None) -> list[dict]:
|
||||
model = _get_model()
|
||||
pairs = [[query, (d or "")[:MAX_DOC_CHARS]] for d in documents]
|
||||
scores = model.predict(pairs)
|
||||
ranked = sorted(
|
||||
({"index": i, "relevance_score": float(s)} for i, s in enumerate(scores)),
|
||||
key=lambda r: -r["relevance_score"],
|
||||
)
|
||||
if top_n is not None:
|
||||
ranked = ranked[:top_n]
|
||||
return ranked
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args):
|
||||
log.info("%s - %s", self.address_string(), fmt % args)
|
||||
|
||||
def _send_json(self, status: int, payload: dict) -> None:
|
||||
body = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self): # noqa: N802
|
||||
if self.path in ("/", "/health"):
|
||||
self._send_json(200, {"status": "ok", "model": MODEL_NAME})
|
||||
return
|
||||
self._send_json(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self): # noqa: N802
|
||||
if self.path not in ("/v1/rerank", "/rerank"):
|
||||
self._send_json(404, {"error": "not found"})
|
||||
return
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
try:
|
||||
req = json.loads(self.rfile.read(length).decode())
|
||||
except Exception as e:
|
||||
self._send_json(400, {"error": f"bad json: {e}"})
|
||||
return
|
||||
query = req.get("query")
|
||||
documents = req.get("documents")
|
||||
if not isinstance(query, str) or not isinstance(documents, list):
|
||||
self._send_json(400, {"error": "expected {query: str, documents: list[str]}"})
|
||||
return
|
||||
top_n = req.get("top_n")
|
||||
try:
|
||||
results = _rerank(query, documents, top_n if isinstance(top_n, int) else None)
|
||||
except Exception as e:
|
||||
log.exception("rerank failed")
|
||||
self._send_json(500, {"error": str(e)})
|
||||
return
|
||||
self._send_json(200, {"model": MODEL_NAME, "results": results})
|
||||
|
||||
|
||||
def main() -> int:
|
||||
_get_model() # warm-load before accepting traffic
|
||||
server = ThreadingHTTPServer((HOST, PORT), Handler)
|
||||
log.info("listening on http://%s:%d", HOST, PORT)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
log.info("shutting down")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user