Files
zerto-ai-rewind/src/zerto_rewind_mcp/server.py
T
justinandClaude Opus 5 8b973653d8 feat(flr): make FLR session lifecycle visible and reapable
zerto_recover_file already tore its session down in a finally block, but
three gaps meant a mount could stay up on the recovery site with nothing
tracking it. FLR cannot run during clone, test, live failover or EJC, so
a stuck session blocks the next recovery.

1. An unmount failure was swallowed (`except ZertoError: pass`). The
   caller got ok=true and never learned the mount was still up. The
   teardown result is now reported in the response as `unmount`, with a
   `warning` when it fails. ok stays true when the bytes did land -- the
   recovery genuinely succeeded -- but the caller is told.

2. If start_flr succeeded on the ZVM while its response failed to parse,
   session_id stayed None and the finally block did nothing, leaking a
   session the process never knew the id of. Teardown now snapshots live
   session ids before starting and reaps anything new that appeared,
   leaving other operators' sessions alone.

3. Nothing could see or clear an orphan left by a crashed process, since
   the finally block only runs if the process survives. Two new tools:

   - zerto_list_flr_sessions: every session the ZVM knows about.
     live_only (default true) keeps the ones still holding a mount;
     ended and failed sessions linger as history and hold nothing.
   - zerto_end_flr_session: unmount one. Gated on confirmed=true,
     matching the other destructive tools, because ending a session
     someone else is pulling files from will interrupt them.

Verified against ZVM 10.x: listing reports 0 live / 1 known after a clean
run, the confirm gate refuses without a human yes, a real recovery from
checkpoint 1368 returned 158 bytes and reported
unmount.ok=true with the session id it ended, and 0 live sessions
remained afterwards.

pytest 27 passed (5 new, including fakes covering the swallowed-failure
and orphan-reap paths).

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_016yVfC5nvZowoLFnEGWhLGn
2026-09-21 13:42:42 -04:00

571 lines
18 KiB
Python

"""Rewind-shaped MCP. One process for the demo. Not a second ZVM catalog."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from mcp.server.fastmcp import FastMCP
from zerto_rewind_mcp.catalog import CatalogEntry, MutatingCatalog
from zerto_rewind_mcp.checkpoints import checkpoint_id, checkpoint_tag, make_tag, tag_vpgs
from zerto_rewind_mcp.client import ZertoClient, ZertoError
from zerto_rewind_mcp.config import load_catalog, load_config
from zerto_rewind_mcp.protection import find_from_rows
from zerto_rewind_mcp.recover import (
download_token_from,
is_live_session,
resolve_flr_path,
session_id_from,
session_id_of,
session_rows,
session_summary,
wait_flr_ready,
)
from zerto_rewind_mcp.util import pick
mcp = FastMCP("zerto_rewind_mcp")
_client: ZertoClient | None = None
_catalog: MutatingCatalog | None = None
_settings: dict[str, Any] = {}
def _dump(payload: Any) -> str:
return json.dumps(payload, indent=2, default=str)
def _ensure_settings() -> dict[str, Any]:
global _settings, _catalog
if not _settings:
_settings = load_config()
_catalog = load_catalog(_settings)
return _settings
def get_client() -> ZertoClient:
global _client
settings = _ensure_settings()
if _client is None:
if not settings.get("zerto_url") or not settings.get("username"):
raise ZertoError(
"Missing zerto_url/username. Copy config.example.json to config.json "
"or set ZERTO_URL and ZERTO_USERNAME."
)
_client = ZertoClient(
base_url=str(settings["zerto_url"]),
username=str(settings["username"]),
password=str(settings.get("password") or ""),
client_id=str(settings.get("client_id") or "zerto-client"),
verify_tls=bool(settings.get("verify_tls")),
)
return _client
def get_catalog() -> MutatingCatalog:
_ensure_settings()
assert _catalog is not None
return _catalog
async def _find(query: str):
client = get_client()
q = query.strip()
looks_id = len(q) >= 32 and "-" in q
rows = await client.get_vms(vm_identifier=q) if looks_id else await client.get_vms(vm_name=q)
if not rows and looks_id:
rows = await client.get_vms(vm_name=q)
elif not rows:
rows = await client.get_vms(vm_identifier=q)
return find_from_rows(q, rows)
@mcp.tool(
name="zerto_find_protection",
annotations={
"title": "Find Zerto protection for a VM",
"readOnlyHint": True,
"destructiveHint": False,
"idempotentHint": True,
"openWorldHint": True,
},
)
async def zerto_find_protection(query: str) -> str:
"""Resolve a VM name, hostname, or Zerto vmIdentifier to exactly one VM and every VPG it is in.
Zero matches or two-plus VMs: stop. Do not mutate. A VM can be in up to three VPGs.
"""
try:
result = await _find(query)
except ZertoError as exc:
return _dump({"ok": False, "message": str(exc)})
payload = result.as_dict()
payload["ok"] = result.outcome == "ok" and bool(result.taggable_vpgs)
return _dump(payload)
@mcp.tool(
name="zerto_create_tagged_checkpoint",
annotations={
"title": "Insert tagged checkpoints on every protecting VPG",
"readOnlyHint": False,
"destructiveHint": False,
"idempotentHint": False,
"openWorldHint": True,
},
)
async def zerto_create_tagged_checkpoint(
query: str,
change_id: str,
agent: str = "agent",
action: str = "",
tag: str | None = None,
) -> str:
"""Insert the same tagged checkpoint on every protecting VPG, then wait.
Call this before every cataloged guest-mutating tool. If it fails, refuse the change.
action: say plainly what you are about to do to this VM, e.g.
"edit /etc/nginx/nginx.conf" or "apt upgrade". It is written into the
checkpoint name so an operator reading the journal in the Zerto UI can see
which agent made the checkpoint and why. CheckpointName is the only field
the Zerto API accepts, so this is the only place that context can live.
Name format: ai:<agent> | <action> | vm=<vm> | change=<change_id> | <utc>
Docs: tagged checkpoints are not supported when the protected site is Azure or AWS;
run this against the vSphere protected ZVM.
"""
try:
result = await _find(query)
if result.outcome != "ok" or not result.taggable_vpgs:
payload = result.as_dict()
payload["ok"] = False
return _dump(payload)
use_tag = tag or make_tag(
agent,
change_id,
action=action,
vm_name=result.vm.vm_name if result.vm else None,
)
out = await tag_vpgs(get_client(), result, use_tag)
return _dump(out)
except ZertoError as exc:
return _dump({"ok": False, "message": str(exc)})
@mcp.tool(
name="zerto_list_checkpoints",
annotations={
"title": "List journal checkpoints for a VPG",
"readOnlyHint": True,
"destructiveHint": False,
"idempotentHint": True,
"openWorldHint": True,
},
)
async def zerto_list_checkpoints(vpg_identifier: str, tag: str | None = None) -> str:
"""List checkpoints on a VPG. Pass tag to keep only matching tagged checkpoints."""
try:
rows = await get_client().list_checkpoints(vpg_identifier)
except ZertoError as exc:
return _dump({"ok": False, "message": str(exc)})
items = []
for row in rows:
item = {
"checkpoint_id": checkpoint_id(row),
"tag": checkpoint_tag(row),
"timestamp": pick(row, "Timestamp", "TimeStamp", "timestamp"),
}
if tag and item["tag"] != tag:
continue
items.append(item)
return _dump(
{
"ok": True,
"vpg_identifier": vpg_identifier,
"count": len(items),
"checkpoints": items,
}
)
@mcp.tool(
name="zerto_guard_before_mutate",
annotations={
"title": "Guard: find protection and tag before a mutating call",
"readOnlyHint": False,
"destructiveHint": False,
"idempotentHint": False,
"openWorldHint": True,
},
)
async def zerto_guard_before_mutate(
query: str,
change_id: str,
agent: str = "agent",
action: str = "",
) -> str:
"""Find protection and tag. If ok is not true, do not mutate.
The skill and any cataloged tool wrapper call this first. Capture-before-execute:
if this does not return ok=true, do not run the guest-mutating call.
Pass action describing the change you are about to make; it is recorded in
the checkpoint name so the journal says which agent did what.
"""
return await zerto_create_tagged_checkpoint(
query=query, change_id=change_id, agent=agent, action=action
)
@mcp.tool(
name="zerto_list_mutating_tools",
annotations={
"title": "List the mutating catalog",
"readOnlyHint": True,
"destructiveHint": False,
"idempotentHint": True,
"openWorldHint": False,
},
)
async def zerto_list_mutating_tools() -> str:
"""Tools that must be guarded. Unlisted tools pass through."""
entries = [e.as_dict() for e in get_catalog().list()]
return _dump({"ok": True, "count": len(entries), "mutating_tools": entries})
@mcp.tool(
name="zerto_add_mutating_tool",
annotations={
"title": "Add a tool to the mutating catalog",
"readOnlyHint": False,
"destructiveHint": False,
"idempotentHint": True,
"openWorldHint": False,
},
)
async def zerto_add_mutating_tool(
server: str,
tool: str,
vm_arg: str,
notes: str = "",
) -> str:
"""Register an MCP tool as guest-mutating. vm_arg holds VM name/hostname/id."""
try:
entry = CatalogEntry(server=server, tool=tool, vm_arg=vm_arg, notes=notes)
get_catalog().add(entry)
except ValueError as exc:
return _dump({"ok": False, "message": str(exc)})
return _dump({"ok": True, "entry": entry.as_dict()})
async def _live_session_ids(client: ZertoClient) -> set[str]:
try:
rows = session_rows(await client.list_flrs())
except ZertoError:
return set()
return {session_id_of(r) for r in rows if is_live_session(r) and session_id_of(r)}
async def _teardown_flr(
client: ZertoClient,
session_id: str | None,
before: set[str],
) -> dict[str, Any]:
"""Unmount the FLR session and say what happened.
A swallowed unmount failure is how a mount silently wedges the recovery
site: FLR cannot run during clone/test/live/EJC, so a stuck session blocks
the next recovery. Report it instead.
If session_id is None the start may still have succeeded on the ZVM while
the response failed to parse, so reap anything new that appeared.
"""
out: dict[str, Any] = {"attempted": False, "ok": True, "ended": [], "failed": []}
targets = [session_id] if session_id else []
if not targets:
orphans = sorted(await _live_session_ids(client) - before)
targets = orphans
out["orphans_reaped"] = orphans
for sid in targets:
out["attempted"] = True
try:
await client.end_flr(sid)
out["ended"].append(sid)
except ZertoError as exc:
out["ok"] = False
out["failed"].append({"session_id": sid, "message": str(exc)})
if not out["ok"]:
out["message"] = (
"FLR session may still be mounted on the recovery site. "
"List it with zerto_list_flr_sessions and end it with "
"zerto_end_flr_session; a stuck mount blocks the next FLR."
)
return out
@mcp.tool(
name="zerto_recover_file",
annotations={
"title": "FLR: pull a file from a tagged checkpoint",
"readOnlyHint": False,
"destructiveHint": True,
"idempotentHint": False,
"openWorldHint": True,
},
)
async def zerto_recover_file(
vpg_identifier: str,
vm_identifier: str,
checkpoint_identifier: str,
guest_path: str,
confirmed: bool = False,
dest_dir: str | None = None,
) -> str:
"""File-level recovery from a journal checkpoint. The VM stays up.
Requires confirmed=true (human yes). Cannot run during clone/test/live/EJC.
10.9 FLR Operator role fails; use an Administrator account.
"""
if not confirmed:
return _dump(
{
"ok": False,
"needs_confirm": True,
"message": (
"Set confirmed=true after a human yes. FLR mounts a disk on the recovery site."
),
}
)
client = get_client()
dest = Path(dest_dir or _settings.get("recovery_dir") or "./recovered")
dest.mkdir(parents=True, exist_ok=True)
session_id: str | None = None
before = await _live_session_ids(client)
payload: dict[str, Any]
try:
started = await client.start_flr(
vpg_identifier,
vm_identifier,
checkpoint_identifier,
str(dest.resolve()),
)
session_id = session_id_from(started)
await wait_flr_ready(client, session_id)
flr_path = await resolve_flr_path(client, session_id, guest_path)
token_payload = await client.download_flr(session_id, [flr_path])
token = download_token_from(token_payload)
blob = await client.fetch_download(token)
name = Path(guest_path.replace("\\", "/")).name or "recovered.bin"
out_path = dest / name
out_path.write_bytes(blob)
payload = {
"ok": True,
"path": str(out_path.resolve()),
"bytes": len(blob),
"session_id": session_id,
"flr_path": flr_path,
"message": f"Wrote {len(blob)} bytes to {out_path}",
}
except ZertoError as exc:
payload = {"ok": False, "session_id": session_id, "message": str(exc)}
finally:
unmount = await _teardown_flr(client, session_id, before)
payload["unmount"] = unmount
if not unmount["ok"]:
# The bytes are on disk, so ok stays true, but the caller must be told
# the mount is still up rather than finding out at the next recovery.
payload["warning"] = unmount["message"]
return _dump(payload)
@mcp.tool(
name="zerto_list_flr_sessions",
annotations={
"title": "List FLR sessions on the ZVM",
"readOnlyHint": True,
"destructiveHint": False,
"idempotentHint": True,
"openWorldHint": True,
},
)
async def zerto_list_flr_sessions(live_only: bool = True) -> str:
"""Every FLR session the ZVM knows about, so orphaned mounts are visible.
zerto_recover_file tears its own session down, but that cleanup only runs if
this process survives the call. A crash, disconnect or timeout mid-recovery
leaves the mount up with nothing tracking it. FLR cannot run during clone,
test, live failover or EJC, so a stuck mount blocks the next recovery.
live_only keeps sessions still holding a mount. Pass false to see ended and
failed sessions too, which the ZVM keeps as history.
"""
try:
rows = session_rows(await get_client().list_flrs())
except ZertoError as exc:
return _dump({"ok": False, "message": str(exc)})
kept = [r for r in rows if is_live_session(r)] if live_only else rows
return _dump(
{
"ok": True,
"live_only": live_only,
"count": len(kept),
"total_known": len(rows),
"sessions": [session_summary(r) for r in kept],
}
)
@mcp.tool(
name="zerto_end_flr_session",
annotations={
"title": "End an FLR session (unmount)",
"readOnlyHint": False,
"destructiveHint": True,
"idempotentHint": True,
"openWorldHint": True,
},
)
async def zerto_end_flr_session(session_id: str, confirmed: bool = False) -> str:
"""Unmount an FLR session. Use to reap an orphan left by a crashed recovery.
Requires confirmed=true: ending a session that another operator is actively
pulling files from will interrupt them. Find the id with
zerto_list_flr_sessions.
"""
if not confirmed:
return _dump(
{
"ok": False,
"needs_confirm": True,
"message": (
"Set confirmed=true after a human yes. Ending a session that "
"someone is actively recovering from will interrupt them."
),
}
)
try:
await get_client().end_flr(session_id)
except ZertoError as exc:
return _dump({"ok": False, "session_id": session_id, "message": str(exc)})
return _dump(
{
"ok": True,
"session_id": session_id,
"message": f"Ended FLR session {session_id}.",
}
)
@mcp.tool(
name="zerto_start_failover_test",
annotations={
"title": "Start a failover test (inspect only)",
"readOnlyHint": False,
"destructiveHint": True,
"idempotentHint": False,
"openWorldHint": True,
},
)
async def zerto_start_failover_test(
vpg_identifier: str,
confirmed: bool = False,
checkpoint_identifier: str | None = None,
) -> str:
"""Spin up test VMs from a checkpoint. Not a production failback. Requires confirmed=true."""
if not confirmed:
return _dump(
{
"ok": False,
"needs_confirm": True,
"message": (
"Set confirmed=true after a human yes. "
"Failover test creates VMs on the test network."
),
}
)
try:
result = await get_client().start_failover_test(vpg_identifier, checkpoint_identifier)
return _dump({"ok": True, "result": result})
except ZertoError as exc:
return _dump({"ok": False, "message": str(exc)})
@mcp.tool(
name="zerto_stop_failover_test",
annotations={
"title": "Stop a failover test",
"readOnlyHint": False,
"destructiveHint": True,
"idempotentHint": False,
"openWorldHint": True,
},
)
async def zerto_stop_failover_test(
vpg_identifier: str,
success: bool = True,
summary: str = "stopped by zerto rewind mcp",
confirmed: bool = False,
) -> str:
"""Stop an in-progress failover test. Requires confirmed=true."""
if not confirmed:
return _dump(
{
"ok": False,
"needs_confirm": True,
"message": "Set confirmed=true after a human yes.",
}
)
try:
result = await get_client().stop_failover_test(vpg_identifier, success, summary)
return _dump({"ok": True, "result": result})
except ZertoError as exc:
return _dump({"ok": False, "message": str(exc)})
@mcp.tool(
name="zerto_offsite_clone",
annotations={
"title": "Offsite clone from a checkpoint",
"readOnlyHint": False,
"destructiveHint": True,
"idempotentHint": False,
"openWorldHint": True,
},
)
async def zerto_offsite_clone(
vpg_identifier: str,
checkpoint_identifier: str,
confirmed: bool = False,
datastore_identifier: str | None = None,
) -> str:
"""Copy VMs at the recovery site from a checkpoint. Prod stays up. Requires confirmed=true."""
if not confirmed:
return _dump(
{
"ok": False,
"needs_confirm": True,
"message": (
"Set confirmed=true after a human yes. Clone consumes recovery datastore."
),
}
)
try:
result = await get_client().start_clone(
vpg_identifier,
checkpoint_identifier,
datastore_identifier=datastore_identifier,
)
return _dump({"ok": True, "result": result})
except ZertoError as exc:
return _dump({"ok": False, "message": str(exc)})
def main() -> None:
mcp.run()
if __name__ == "__main__":
main()