"""Discover HVM doc bundles on HPE Support DocPortal and write bundles.json.

HPE mints a NEW docId per product version rather than versioning a single
doc (see ~/.claude/.../reference_hpe_docs_portal_api.md). This script no
longer requires a human to hand-add each new version's docId: it
AUTO-DISCOVERS new HPE Morpheus VM Essentials versions by scanning the
docId numeric range above the highest known bundle and classifying each
hit by its abstract body.

The static BUNDLES list below is the evergreen floor — the unversioned /
special bundles (Deployment Guide, Qualification Matrix, QuickSpecs) plus
the versioned bundles known when this file was last hand-edited. Discovery
appends anything newer on top. So a new HVM release (e.g. 9.0) flows into
corpus/ through the weekly refresh with no code change.

For each bundle this script:
  1. GETs /hpesc/public/api/document/{docId}        → abstract HTML
  2. GETs /hpesc/public/api/document/{docId}/toc    → page tree (or 404 for single-doc)
  3. Writes bundles.json at repo root with the schema PLAN.md Phase 1 documents.

Discovery (see discover_specs) is on by default; pass --no-discover to
build only from the static + prior-bundles.json set. It is FAILSOFT: a
network error during the range scan logs a warning and falls back to the
known bundles rather than breaking the refresh.
"""
from __future__ import annotations

import argparse
import json
import re
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

import requests
from bs4 import BeautifulSoup

API = "https://support.hpe.com/hpesc/public/api/document"
DOC_URL = "https://support.hpe.com/hpesc/public/docDisplay?docId={doc_id}"
UA = "hvm-docs-mcp/0.1 (+https://git.jpaul.io/justin/hvm-docs; admin@jpaul.io)"
ROOT = Path(__file__).resolve().parent.parent
BUNDLES_JSON = ROOT / "bundles.json"


@dataclass
class BundleSpec:
    slug: str
    doc_id: str
    title: str
    version: str | None
    product: str  # e.g. "User Manual", "Release Notes", "Deployment Guide"
    mode: str    # "toc", "single", or "html-file" (committed fixture under scrape/quickspecs/)
    platform: str | None = None
    language: str = "en-US"
    source_url: str | None = None   # overrides the default support.hpe.com URL


# Static floor. The unversioned/special bundles are authoritative here
# (Deployment Guide, Qualification Matrix, QuickSpecs); the versioned
# entries are the seed set. Newer versions are added automatically by
# discover_specs() — you should NOT need to hand-add a docId per release
# anymore. Versions confirmed 2026-05-22 by probing sd00007400..7740.
BUNDLES: list[BundleSpec] = [
    BundleSpec("hvm_user_manual_8_1_0",   "sd00007520en_us", "HPE Morpheus VM Essentials Software Documentation", "8.1.0", "User Manual",      "toc"),
    BundleSpec("hvm_user_manual_8_1_1",   "sd00007620en_us", "HPE Morpheus VM Essentials Software Documentation", "8.1.1", "User Manual",      "toc"),
    BundleSpec("hvm_user_manual_8_1_2",   "sd00007735en_us", "HPE Morpheus VM Essentials Software Documentation", "8.1.2", "User Manual",      "toc"),
    BundleSpec("hvm_release_notes_8_1_0", "sd00007497en_us", "HPE Morpheus VM Essentials Software Release Notes",  "8.1.0", "Release Notes",   "single"),
    BundleSpec("hvm_release_notes_8_1_1", "sd00007609en_us", "HPE Morpheus VM Essentials Software Release Notes",  "8.1.1", "Release Notes",   "single"),
    BundleSpec("hvm_release_notes_8_1_2", "sd00007734en_us", "HPE Morpheus VM Essentials Software Release Notes",  "8.1.2", "Release Notes",   "single"),
    BundleSpec("hvm_deployment_guide",    "sd00007332en_us", "HPE Morpheus VM Essentials Deployment Guide",        None,    "Deployment Guide","toc"),
    BundleSpec("hvm_qualification_matrix","sd00006551en_us", "Qualification Matrix for HVM Clusters Managed by HPE Morpheus Software", None, "Qualification Matrix", "toc"),
    # QuickSpecs is a static-HTML fixture (www.hpe.com edge drops automated
    # connections — see scrape/quickspecs/README.md). doc_id = the QuickSpecs
    # PSNow ref (a50004260enw). page_count is 1; source_url points at the
    # public PSNow URL.
    BundleSpec("hvm_quickspecs", "a50004260enw", "HPE Morpheus VM Essentials Software QuickSpecs",
               "v4-2026-02-02", "QuickSpecs", "html-file",
               source_url="https://www.hpe.com/psnow/doc/a50004260enw"),
]


def _session() -> requests.Session:
    s = requests.Session()
    s.headers.update({"User-Agent": UA, "Accept": "application/json, text/html"})
    return s


def _get(s: requests.Session, url: str, expect_json: bool = False, retries: int = 4) -> Any:
    delay = 1.0
    for attempt in range(retries):
        r = s.get(url, timeout=30)
        if r.status_code == 200:
            return r.json() if expect_json else r.text
        if r.status_code == 404:
            return None
        if r.status_code in (429, 500, 502, 503, 504):
            time.sleep(delay)
            delay *= 2
            continue
        r.raise_for_status()
    raise RuntimeError(f"GET failed after {retries} retries: {url}")


def _count_toc(toc: list[dict] | None) -> tuple[int, str | None]:
    """Returns (page_count, landing_page_guid)."""
    if not toc:
        return 0, None
    landing = None
    n = 0

    def walk(nodes: list[dict] | None, depth: int) -> None:
        nonlocal n, landing
        for node in nodes or []:
            link = node.get("topicLink")
            if link:
                n += 1
                m = re.search(r"page=(GUID-[A-F0-9-]+)\.html", link)
                if m and landing is None:
                    landing = m.group(1)
            walk(node.get("children"), depth + 1)

    walk(toc, 0)
    return n, landing


def _parse_abstract(html: str) -> dict[str, str]:
    """Pull title / abstract text / published date out of the DITA abstract HTML."""
    soup = BeautifulSoup(html, "html.parser")
    out: dict[str, str] = {}
    h1 = soup.select_one("h1.title.topictitle1")
    if h1:
        out["title"] = h1.get_text(" ", strip=True)
    desc = soup.select_one("div.desc")
    if desc:
        out["abstract"] = desc.get_text(" ", strip=True)
    pub = soup.select_one("div.publishedDate")
    if pub:
        out["published"] = pub.get_text(" ", strip=True).replace("Published:", "").strip()
    return out


# --- Auto-discovery -------------------------------------------------------
#
# HPE interleaves sibling products in the same docId range and reuses the
# SAME version string across them: HVM 9.0.0, Morpheus *Enterprise* 9.0.0,
# and Morpheus *Central* all sit within ~30 docIds of each other. Worse,
# a Release Notes doc's <h1> is often just "v9.0.0 Release Notes" with no
# product name — so we MUST classify on the abstract body, never the title
# alone. INCLUDE requires the full "Morpheus VM Essentials" string; EXCLUDE
# drops the neighbours and the Storage Integration Pack add-on.
_INCLUDE_RE = re.compile(r"Morpheus VM Essentials", re.I)
_EXCLUDE_RE = re.compile(r"Storage Integration Pack|Morpheus Enterprise|Morpheus Central|OneView", re.I)
_VER_RE = re.compile(r"v?(\d+\.\d+(?:\.\d+)?)")
_DOCID_NUM_RE = re.compile(r"^sd0*(\d+)en_us$")

# Default forward scan window above the high-water docId. 9.0 landed ~344
# ids above the 8.1.2 seed; 600 leaves comfortable headroom and self-heals
# from the static floor if bundles.json is ever lost. Bump if a future
# release outruns it (a one-line change, still no per-version docId edits).
DISCOVER_WINDOW = 600


def _docid_num(doc_id: str) -> int | None:
    m = _DOCID_NUM_RE.match(doc_id)
    return int(m.group(1)) if m else None


def _slug_for(kind: str, version: str) -> str:
    v = version.replace(".", "_").replace("-", "_")
    base = "hvm_user_manual" if kind == "doc" else "hvm_release_notes"
    return f"{base}_{v}"


def _classify_abstract(html: str) -> tuple[str, str, str] | None:
    """Return (kind, version, title) for an HVM bundle, else None.

    kind is 'doc' (paginated User Manual / Software Documentation) or 'rn'
    (single-doc Release Notes). Only these two well-understood shapes are
    auto-added; anything else that mentions VM Essentials is logged for
    manual review by the caller and skipped."""
    meta = _parse_abstract(html)
    title = meta.get("title", "")
    blob = f"{title} {meta.get('abstract', '')}"
    if not _INCLUDE_RE.search(blob) or _EXCLUDE_RE.search(blob):
        return None
    vm = _VER_RE.search(blob)
    if not vm:
        return None
    version = vm.group(1)
    if re.search(r"release note", blob, re.I):
        return ("rn", version, title)
    if re.search(r"software documentation", blob, re.I):
        return ("doc", version, title)
    return None  # VM-Essentials-ish but not a UM/RN shape → caller logs it


def discover_specs(s: requests.Session, known: list[BundleSpec],
                   window: int = DISCOVER_WINDOW,
                   scan_start: int | None = None) -> list[BundleSpec]:
    """Scan the docId range above the known high-water mark for new HVM
    versions and return BundleSpecs for any not already covered.

    FAILSOFT: any per-docId fetch error is swallowed (the range is sparse
    and full of auth-gated stubs); a wholesale failure is logged and returns
    [] so the caller proceeds with the known bundles."""
    known_ids = {b.doc_id for b in known}
    known_slugs = {b.slug for b in known}
    anchor = scan_start
    if anchor is None:
        nums = [n for n in (_docid_num(b.doc_id) for b in known) if n is not None]
        anchor = max(nums) if nums else 7400
    lo, hi = anchor + 1, anchor + window
    print(f"  discovery: scanning sd[{lo}..{hi}] for new VM Essentials versions ...",
          file=sys.stderr)

    def probe(n: int) -> tuple[int, str, str, str] | None:
        doc_id = f"sd{n:08d}en_us"
        try:
            html = _get(s, f"{API}/{doc_id}", expect_json=False)
        except Exception:
            return None
        if not html or len(html) < 200:
            return None
        # Log VM-Essentials docs that don't classify as UM/RN for manual review.
        meta = _parse_abstract(html)
        blob = f"{meta.get('title','')} {meta.get('abstract','')}"
        res = _classify_abstract(html)
        if res is None:
            if _INCLUDE_RE.search(blob) and not _EXCLUDE_RE.search(blob):
                print(f"  discovery: {doc_id} mentions VM Essentials but is not a "
                      f"UM/RN shape — review manually: {meta.get('title','')!r}",
                      file=sys.stderr)
            return None
        kind, version, title = res
        return (n, kind, version, title)

    hits: list[tuple[int, str, str, str]] = []
    try:
        with ThreadPoolExecutor(max_workers=6) as pool:
            for fut in as_completed(pool.submit(probe, n) for n in range(lo, hi + 1)):
                r = fut.result()
                if r:
                    hits.append(r)
    except Exception as e:  # network/pool blew up wholesale
        print(f"  ! discovery scan failed ({e}) — proceeding with known bundles",
              file=sys.stderr)
        return []

    new: list[BundleSpec] = []
    for n, kind, version, title in sorted(hits):
        doc_id = f"sd{n:08d}en_us"
        slug = _slug_for(kind, version)
        if doc_id in known_ids or slug in known_slugs:
            continue
        known_ids.add(doc_id)
        known_slugs.add(slug)
        product = "User Manual" if kind == "doc" else "Release Notes"
        mode = "toc" if kind == "doc" else "single"
        new.append(BundleSpec(slug, doc_id, title or "", version, product, mode))
        print(f"  discovery: + {slug} ({doc_id}) v{version} [{product}]", file=sys.stderr)

    if not new:
        print("  discovery: no new versions found", file=sys.stderr)
    return new


def _spec_from_json(entry: dict[str, Any]) -> BundleSpec:
    """Reconstruct a BundleSpec from a prior bundles.json entry so previously
    discovered versions persist across runs (the scan anchor rolls forward)."""
    return BundleSpec(
        slug=entry["slug"], doc_id=entry["doc_id"], title=entry.get("title") or "",
        version=entry.get("version"), product=entry.get("product") or "",
        mode=entry.get("mode") or "toc", platform=entry.get("platform"),
        language=entry.get("language", "en-US"), source_url=entry.get("source_url"),
    )


def collect_specs(s: requests.Session, discover: bool = True,
                  window: int = DISCOVER_WINDOW,
                  scan_start: int | None = None) -> list[BundleSpec]:
    """Merge static floor + prior bundles.json + freshly discovered specs,
    deduped by doc_id (static wins)."""
    by_id: dict[str, BundleSpec] = {}
    for spec in BUNDLES:
        by_id.setdefault(spec.doc_id, spec)
    # Retain previously-discovered versions recorded in bundles.json.
    if BUNDLES_JSON.exists():
        try:
            for entry in json.loads(BUNDLES_JSON.read_text()):
                by_id.setdefault(entry["doc_id"], _spec_from_json(entry))
        except Exception as e:
            print(f"  ! could not read prior {BUNDLES_JSON.name} ({e})", file=sys.stderr)
    specs = list(by_id.values())
    if discover:
        for spec in discover_specs(s, specs, window=window, scan_start=scan_start):
            if spec.doc_id not in by_id:
                by_id[spec.doc_id] = spec
                specs.append(spec)
    return specs


def discover_bundle(s: requests.Session, spec: BundleSpec) -> dict[str, Any]:
    # html-file bundles are static fixtures — no upstream fetch.
    if spec.mode == "html-file":
        return {
            "slug": spec.slug,
            "doc_id": spec.doc_id,
            "title": spec.title,
            "version": spec.version,
            "platform": spec.platform,
            "product": spec.product,
            "language": spec.language,
            "page_count": 1,
            "mode": "html-file",
            "abstract": "",
            "dates": {},
            "landing_page": spec.doc_id,
            "source_url": spec.source_url or f"https://www.hpe.com/psnow/doc/{spec.doc_id}",
        }

    abstract_html = _get(s, f"{API}/{spec.doc_id}", expect_json=False)
    meta = _parse_abstract(abstract_html or "")

    page_count: int
    landing: str | None
    if spec.mode == "toc":
        toc = _get(s, f"{API}/{spec.doc_id}/toc", expect_json=True)
        page_count, landing = _count_toc(toc)
        if page_count == 0:
            print(f"  ! {spec.slug}: TOC empty — falling back to single-doc mode", file=sys.stderr)
            spec.mode = "single"
            page_count, landing = 1, spec.doc_id
    else:
        page_count, landing = 1, spec.doc_id

    return {
        "slug": spec.slug,
        "doc_id": spec.doc_id,
        "title": meta.get("title") or spec.title,
        "version": spec.version,
        "platform": spec.platform,
        "product": spec.product,
        "language": spec.language,
        "page_count": page_count,
        "mode": spec.mode,
        "abstract": meta.get("abstract", ""),
        "dates": {"Published": meta.get("published", "")},
        "landing_page": landing,
        "source_url": spec.source_url or DOC_URL.format(doc_id=spec.doc_id),
    }


def main() -> int:
    p = argparse.ArgumentParser(description="Build bundles.json (static floor + auto-discovered versions).")
    p.add_argument("--out", default=str(BUNDLES_JSON))
    p.add_argument("--no-discover", dest="discover", action="store_false",
                   help="skip the docId range scan; build only from static + prior bundles.json")
    p.add_argument("--window", type=int, default=DISCOVER_WINDOW,
                   help=f"how many docIds above the high-water mark to scan (default {DISCOVER_WINDOW})")
    p.add_argument("--scan-start", type=int, default=None,
                   help="override the scan anchor (numeric docId, e.g. 7735)")
    args = p.parse_args()

    s = _session()
    specs = collect_specs(s, discover=args.discover, window=args.window,
                          scan_start=args.scan_start)
    out: list[dict[str, Any]] = []
    for spec in specs:
        print(f"  • {spec.slug} ({spec.doc_id}) ...", file=sys.stderr)
        out.append(discover_bundle(s, spec))

    Path(args.out).write_text(json.dumps(out, indent=2) + "\n")
    print(f"wrote {args.out}: {len(out)} bundles, {sum(b['page_count'] for b in out)} pages total", file=sys.stderr)
    return 0


if __name__ == "__main__":
    sys.exit(main())
