From e595dcd9e7e242ae2ed37e77f10e0d8b689e5906 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Tue, 8 Sep 2026 14:57:39 +0000 Subject: [PATCH] Fix code formatting - tox -e fmt --- AGENTS.md | 2 + openvixdisklib/nfc_auth.py | 209 +++++++++++++--------- openvixdisklib/nfc_open.py | 215 ++++++++++------------- openvixdisklib/openvixdisklib.py | 152 ++++++++-------- tests/integration/base.py | 117 ++++++------ tests/integration/test_crosscheck.py | 41 ++--- tests/integration/test_nfc_open.py | 15 +- tests/integration/test_nfc_read_write.py | 30 ++-- tests/integration/test_openvixdisklib.py | 67 ++++--- tests/integration/test_vddk.py | 9 +- tests/integration/vix_disklib_errors.py | 2 +- tests/integration/vixdisklib.py | 172 ++++++++++++------ tests/perf/test_compare.py | 76 ++++---- 13 files changed, 615 insertions(+), 492 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f4692f2..1875873 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,3 +73,5 @@ - When writing unit tests, mock decorators are preferred instead of context managers. - If a folder or file under this directory is inaccessible, ask for permissions. +- Use "tox -e fmt" to apply code formatting, "tox -e pep8" and "tox -e fmt" + and "tox -e mypy" for liniting / static code analysis. diff --git a/openvixdisklib/nfc_auth.py b/openvixdisklib/nfc_auth.py index d5af760..bd8d75c 100644 --- a/openvixdisklib/nfc_auth.py +++ b/openvixdisklib/nfc_auth.py @@ -24,7 +24,7 @@ from typing import Optional from pyVim.connect import Disconnect, SmartConnect from pyVmomi import vim -from pyVmomi.VmomiSupport import CreateManagedType, F_OPTIONAL, GetVmodlType +from pyVmomi.VmomiSupport import F_OPTIONAL, CreateManagedType, GetVmodlType NFC_SERVICE_MOID = "nfcService" AUTHD_DEFAULT_PORT = 902 @@ -59,28 +59,62 @@ def _register_nfc_types() -> None: "vim.version.version1", [], [ - ("getVmFiles", "NfcGetVmFiles", "vim.version.version1", - (("vm", "vim.VirtualMachine", "vim.version.version1", 0, None),), - (0, "vim.HostServiceTicket", "vim.HostServiceTicket"), None, None), - ("randomAccessOpen", "NfcRandomAccessOpenDisk", - "vim.version.version1", - (("vm", "vim.VirtualMachine", "vim.version.version1", 0, None), - ("diskDeviceKey", "int", "vim.version.version1", 0, None), - ("hostForAccess", "vim.HostSystem", "vim.version.version1", - F_OPTIONAL, None),), - (0, "vim.HostServiceTicket", "vim.HostServiceTicket"), None, None), - ("randomAccessOpenReadonly", "NfcRandomAccessOpenReadonly", - "vim.version.version1", - (("vm", "vim.VirtualMachine", "vim.version.version1", 0, None), - ("diskDeviceKey", "int", "vim.version.version1", 0, None), - ("hostForAccess", "vim.HostSystem", "vim.version.version1", - F_OPTIONAL, None),), - (0, "vim.HostServiceTicket", "vim.HostServiceTicket"), None, None), - ("getServerNfcLibVersion", "NfcGetServerNfcLibVersion", - "vim.version.version1", - (("hostForAccess", "vim.HostSystem", "vim.version.version1", - 0, None),), - (0, "int", "int"), None, None), + ( + "getVmFiles", + "NfcGetVmFiles", + "vim.version.version1", + (("vm", "vim.VirtualMachine", "vim.version.version1", 0, None),), + (0, "vim.HostServiceTicket", "vim.HostServiceTicket"), + None, + None, + ), + ( + "randomAccessOpen", + "NfcRandomAccessOpenDisk", + "vim.version.version1", + ( + ("vm", "vim.VirtualMachine", "vim.version.version1", 0, None), + ("diskDeviceKey", "int", "vim.version.version1", 0, None), + ( + "hostForAccess", + "vim.HostSystem", + "vim.version.version1", + F_OPTIONAL, + None, + ), + ), + (0, "vim.HostServiceTicket", "vim.HostServiceTicket"), + None, + None, + ), + ( + "randomAccessOpenReadonly", + "NfcRandomAccessOpenReadonly", + "vim.version.version1", + ( + ("vm", "vim.VirtualMachine", "vim.version.version1", 0, None), + ("diskDeviceKey", "int", "vim.version.version1", 0, None), + ( + "hostForAccess", + "vim.HostSystem", + "vim.version.version1", + F_OPTIONAL, + None, + ), + ), + (0, "vim.HostServiceTicket", "vim.HostServiceTicket"), + None, + None, + ), + ( + "getServerNfcLibVersion", + "NfcGetServerNfcLibVersion", + "vim.version.version1", + (("hostForAccess", "vim.HostSystem", "vim.version.version1", 0, None),), + (0, "int", "int"), + None, + None, + ), ], ) _NFC_TYPES_REGISTERED = True @@ -98,12 +132,13 @@ def nfc_service(si: vim.ServiceInstance) -> vim.NfcService: def connect_vim( - host: str, - username: str, - password: str, - port: int = 443, - thumbprint: Optional[str] = None, - allow_untrusted: bool = False) -> vim.ServiceInstance: + host: str, + username: str, + password: str, + port: int = 443, + thumbprint: Optional[str] = None, + allow_untrusted: bool = False, +) -> vim.ServiceInstance: """Login to vCenter or ESXi using pyVim.connect.SmartConnect. Args: @@ -124,7 +159,8 @@ def connect_vim( port=port, thumbprint=thumbprint, sslContext=ssl_context, - disableSslCertValidation=allow_untrusted) + disableSslCertValidation=allow_untrusted, + ) def _virtual_disk_key(vm: vim.VirtualMachine, disk_path: str) -> int: @@ -144,17 +180,17 @@ def _virtual_disk_key(vm: vim.VirtualMachine, disk_path: str) -> int: if getattr(backing, "fileName", None) == disk_path: return device.key backing = getattr(backing, "parent", None) - raise ValueError( - f"VMDK path {disk_path!r} is not attached to {vm._moId}") + raise ValueError(f"VMDK path {disk_path!r} is not attached to {vm._moId}") def get_nfc_ticket( - si: vim.ServiceInstance, - vm: vim.VirtualMachine, - disk_device_key: Optional[int] = None, - host_for_access: Optional[vim.HostSystem] = None, - read_only: bool = True, - disk_path: Optional[str] = None) -> vim.HostServiceTicket: + si: vim.ServiceInstance, + vm: vim.VirtualMachine, + disk_device_key: Optional[int] = None, + host_for_access: Optional[vim.HostSystem] = None, + read_only: bool = True, + disk_path: Optional[str] = None, +) -> vim.HostServiceTicket: """Return a one-time NFC HostServiceTicket for ``vm``. Matches VDDK: ``NfcGetVmFiles`` when only the VM is known (read-only), @@ -174,14 +210,12 @@ def get_nfc_ticket( return nfc.GetVmFiles(vm) if disk_device_key is None: if disk_path is None: - raise ValueError( - "writable NFC tickets need disk_path or disk_device_key") + raise ValueError("writable NFC tickets need disk_path or disk_device_key") disk_device_key = _virtual_disk_key(vm, disk_path) if host_for_access is None: host_for_access = vm.runtime.host if read_only: - return nfc.RandomAccessOpenReadonly( - vm, disk_device_key, host_for_access) + return nfc.RandomAccessOpenReadonly(vm, disk_device_key, host_for_access) return nfc.RandomAccessOpen(vm, disk_device_key, host_for_access) @@ -198,11 +232,12 @@ def _normalize_thumbprint(thumbprint: str) -> str: def get_ssl_cert_thumbprint( - host: str, - port: int = 443, - digest_algorithm: str = "sha1", - ssl_context: Optional[ssl.SSLContext] = None, - timeout: float = 30.0) -> str: + host: str, + port: int = 443, + digest_algorithm: str = "sha1", + ssl_context: Optional[ssl.SSLContext] = None, + timeout: float = 30.0, +) -> str: """Return the TLS certificate thumbprint of ``host``:``port``. Reads the peer certificate in DER form and hashes it with ``hashlib``. @@ -223,14 +258,11 @@ def get_ssl_cert_thumbprint( if ssl_context is None: ssl_context = _ssl_client_context(verify=False) with socket.create_connection((host, port), timeout=timeout) as sock: - with ssl_context.wrap_socket( - sock, server_hostname=host) as ssock: + with ssl_context.wrap_socket(sock, server_hostname=host) as ssock: cert = ssock.getpeercert(binary_form=True) if not cert: - raise ConnectionError( - f"no peer certificate from {host}:{port}") - return _format_thumbprint( - hashlib.new(digest_algorithm, cert).digest()) + raise ConnectionError(f"no peer certificate from {host}:{port}") + return _format_thumbprint(hashlib.new(digest_algorithm, cert).digest()) def _readline(sock: socket.socket) -> str: @@ -248,7 +280,7 @@ def _readline(sock: socket.socket) -> str: def _expect_code(line: str, code: str, what: str) -> str: if not line.startswith(code): raise ConnectionError(f"authd {what} failed: {line}") - return line[len(code):].lstrip() + return line[len(code) :].lstrip() def nfcssl_service_name(service: str) -> str: @@ -266,10 +298,11 @@ def nfcssl_service_name(service: str) -> str: def connect_authd( - ticket: vim.HostServiceTicket, - allow_untrusted: bool = False, - timeout: float = 30.0, - nfc_ssl: bool = True) -> ssl.SSLSocket: + ticket: vim.HostServiceTicket, + allow_untrusted: bool = False, + timeout: float = 30.0, + nfc_ssl: bool = True, +) -> ssl.SSLSocket: """Complete the ESXi authd handshake using an NFC HostServiceTicket. Wire sequence captured from VDDK against authd on TCP 902: @@ -310,10 +343,12 @@ def connect_authd( if not allow_untrusted and ticket.sslThumbprint: peer = _sha1_thumbprint(ssock.getpeercert(True)) if _normalize_thumbprint(peer) != _normalize_thumbprint( - ticket.sslThumbprint): + ticket.sslThumbprint + ): raise ConnectionError( f"ESXi SSL thumbprint mismatch: got {peer}, " - f"expected {ticket.sslThumbprint}") + f"expected {ticket.sslThumbprint}" + ) ssock.sendall(f"SESSION {ticket.sessionId}\r\n".encode("ascii")) # Trailing space is part of the BANNER command token used by authd. @@ -338,11 +373,12 @@ class NfcAuthSession: """Authenticated VIM session plus an authd/NFC TLS socket.""" def __init__( - self, - si: vim.ServiceInstance, - ticket: vim.HostServiceTicket, - authd_sock: ssl.SSLSocket, - nfc_ssl: bool = True) -> None: + self, + si: vim.ServiceInstance, + ticket: vim.HostServiceTicket, + authd_sock: ssl.SSLSocket, + nfc_ssl: bool = True, + ) -> None: self.si = si self.ticket = ticket self.authd_sock = authd_sock @@ -363,17 +399,18 @@ class NfcAuthSession: def authenticate( - host: str, - username: str, - password: str, - vm_moref: str, - port: int = 443, - thumbprint: Optional[str] = None, - allow_untrusted: bool = False, - disk_device_key: Optional[int] = None, - disk_path: Optional[str] = None, - read_only: bool = True, - nfc_ssl: bool = True) -> NfcAuthSession: + host: str, + username: str, + password: str, + vm_moref: str, + port: int = 443, + thumbprint: Optional[str] = None, + allow_untrusted: bool = False, + disk_device_key: Optional[int] = None, + disk_path: Optional[str] = None, + read_only: bool = True, + nfc_ssl: bool = True, +) -> NfcAuthSession: """Login to vSphere and complete NFC authd authentication for a VM. Args: @@ -393,15 +430,25 @@ def authenticate( PROXY service used by nbdssl. Pass False for nbd. """ si = connect_vim( - host, username, password, port=port, - thumbprint=thumbprint, allow_untrusted=allow_untrusted) + host, + username, + password, + port=port, + thumbprint=thumbprint, + allow_untrusted=allow_untrusted, + ) try: vm = vim.VirtualMachine(vm_moref, si._stub) ticket = get_nfc_ticket( - si, vm, disk_device_key=disk_device_key, - disk_path=disk_path, read_only=read_only) + si, + vm, + disk_device_key=disk_device_key, + disk_path=disk_path, + read_only=read_only, + ) authd_sock = connect_authd( - ticket, allow_untrusted=allow_untrusted, nfc_ssl=nfc_ssl) + ticket, allow_untrusted=allow_untrusted, nfc_ssl=nfc_ssl + ) except Exception: Disconnect(si) raise diff --git a/openvixdisklib/nfc_open.py b/openvixdisklib/nfc_open.py index e48a1a4..1ea5676 100644 --- a/openvixdisklib/nfc_open.py +++ b/openvixdisklib/nfc_open.py @@ -98,15 +98,14 @@ def takeover_authd_socket(ssock: ssl.SSLSocket) -> socket.socket: family=ssock.family, type=ssock.type, proto=ssock.proto, - fileno=os.dup(ssock.fileno())) + fileno=os.dup(ssock.fileno()), + ) raw.settimeout(timeout) _enable_tcp_nodelay(raw) return raw -def wrap_nfcssl_socket( - ssock: ssl.SSLSocket, - server_hostname: str) -> ssl.SSLSocket: +def wrap_nfcssl_socket(ssock: ssl.SSLSocket, server_hostname: str) -> ssl.SSLSocket: """Start the second TLS session used by NBDSSL after PROXY. After ``200 Connect ha-nfcssl``, authd TLS is finished and @@ -121,8 +120,7 @@ def wrap_nfcssl_socket( raw = takeover_authd_socket(ssock) ssl_context = _ssl_client_context(verify=False) try: - return ssl_context.wrap_socket( - raw, server_hostname=server_hostname) + return ssl_context.wrap_socket(raw, server_hostname=server_hostname) except Exception: raw.close() raise @@ -139,13 +137,13 @@ def _recvn(sock: socket.socket, size: int) -> bytes: chunk = sock.recv(size - len(buf)) if not chunk: raise NfcProtocolError( - f"NFC connection closed, needed {size} bytes, got {len(buf)}") + f"NFC connection closed, needed {size} bytes, got {len(buf)}" + ) buf.extend(chunk) return bytes(buf) -def _send_nfc_msg( - sock: socket.socket, msg_type: int, body: bytes = b"") -> None: +def _send_nfc_msg(sock: socket.socket, msg_type: int, body: bytes = b"") -> None: if len(body) > NFC_MSG_SIZE - 4: raise ValueError("NFC classic message body too large") frame = struct.pack(" tuple[int, bytes]: def _pack_aio_hdr(msg_type: int, payload_size: int, op_id: int) -> bytes: - return struct.pack( - " tuple[int, int, int]: magic, msg_type, size, op_id = struct.unpack_from(" None: + self, + sock: socket.socket, + path: str, + handle: int, + sector_size: int, + compression: int = NFC_COMPRESSION_NONE, + ) -> None: """Wrap an AIO session that already has ``path`` open. Args: @@ -207,15 +205,12 @@ class NfcDisk: self._op_id += 1 return op_id - def _aio_send( - self, - msg_type: int, - payload: bytes, - extra: bytes = b"") -> int: + def _aio_send(self, msg_type: int, payload: bytes, extra: bytes = b"") -> int: """Send one AIO request (header, payload, and extra in one write).""" op_id = self._next_op_id() self._sock.sendall( - _pack_aio_hdr(msg_type, len(payload), op_id) + payload + extra) + _pack_aio_hdr(msg_type, len(payload), op_id) + payload + extra + ) return op_id def _aio_recv_reply(self) -> tuple[int, int, bytes]: @@ -224,27 +219,24 @@ class NfcDisk: magic, rtype, rsize, rop = struct.unpack_from(" bytes: + self, msg_type: int, payload: bytes, extra: bytes = b"", extra_recv: int = 0 + ) -> bytes: """Send one AIO request and return the reply payload (+ extra).""" op_id = self._aio_send(msg_type, payload, extra) rtype, rop, body = self._aio_recv_reply() if rtype != msg_type or rop != op_id: raise NfcProtocolError( f"AIO reply type={rtype} opId={rop}, " - f"expected type={msg_type} opId={op_id}") + f"expected type={msg_type} opId={op_id}" + ) if extra_recv: body += _recvn(self._sock, extra_recv) return body @@ -269,17 +261,10 @@ class NfcDisk: offset = start_sector * self.sector_size opcode = NFC_AIO_IO_READ | (self.compression << 32) payload = struct.pack( - " length): + if dest in seen or chunk_len == 0 or dest + chunk_len > length: raise NfcProtocolError( f"AIO IO chunk offset={dest} length={chunk_len} invalid, " - f"request {length}") + f"request {length}" + ) seen.add(dest) ctype = opcode >> 32 if ctype == NFC_COMPRESSION_FASTLZ: @@ -317,25 +300,21 @@ class NfcDisk: chunk = fastlz.decompress(extra, chunk_len) except ValueError as exc: raise NfcProtocolError( - f"FastLZ read fragment failed: {exc}") from exc + f"FastLZ read fragment failed: {exc}" + ) from exc if len(chunk) != chunk_len: raise NfcProtocolError( - f"FastLZ read got {len(chunk)} bytes, " - f"expected {chunk_len}") + f"FastLZ read got {len(chunk)} bytes, expected {chunk_len}" + ) elif ctype == NFC_COMPRESSION_NONE: chunk = _recvn(self._sock, chunk_len) else: - raise NfcProtocolError( - f"unsupported NFC IO compression type {ctype}") - data[dest:dest + chunk_len] = chunk + raise NfcProtocolError(f"unsupported NFC IO compression type {ctype}") + data[dest : dest + chunk_len] = chunk filled += chunk_len return bytes(data) - def write( - self, - start_sector: int, - num_sectors: int, - data: bytes) -> None: + def write(self, start_sector: int, num_sectors: int, data: bytes) -> None: """Write ``num_sectors`` starting at ``start_sector``. Matches ``VixDiskLib_Write``: one ``NFC_AIO_MSG_IO`` ``opId`` @@ -352,19 +331,16 @@ class NfcDisk: raise ValueError("num_sectors must be at least 1") length = num_sectors * self.sector_size if len(data) != length: - raise ValueError( - f"write data is {len(data)} bytes, need {length}") + raise ValueError(f"write data is {len(data)} bytes, need {length}") disk_offset = start_sector * self.sector_size op_id = self._next_op_id() frag_offset = 0 while frag_offset < length: - chunk = data[frag_offset:frag_offset + NFC_AIO_BUFFER_SIZE] + chunk = data[frag_offset : frag_offset + NFC_AIO_BUFFER_SIZE] extra = chunk extra_len = len(chunk) ctype = NFC_COMPRESSION_NONE - if ( - self.compression == NFC_COMPRESSION_FASTLZ - and extra_len >= 16): + if self.compression == NFC_COMPRESSION_FASTLZ and extra_len >= 16: compressed = fastlz.compress(chunk) if compressed and len(compressed) < extra_len: extra = compressed @@ -380,16 +356,18 @@ class NfcDisk: frag_offset, len(chunk), extra_len, - 0) + 0, + ) self._sock.sendall( - _pack_aio_hdr(NFC_AIO_MSG_IO, len(payload), op_id) - + payload + extra) + _pack_aio_hdr(NFC_AIO_MSG_IO, len(payload), op_id) + payload + extra + ) frag_offset += len(chunk) rtype, rop, _body = self._aio_recv_reply() if rtype != NFC_AIO_MSG_IO or rop != op_id: raise NfcProtocolError( f"AIO IO write reply type={rtype} opId={rop}, " - f"expected type={NFC_AIO_MSG_IO} opId={op_id}") + f"expected type={NFC_AIO_MSG_IO} opId={op_id}" + ) def close(self) -> None: """Close the VMDK, the AIO session, and the classic NFC session.""" @@ -397,10 +375,8 @@ class NfcDisk: return self._closed = True try: - self._aio_roundtrip( - NFC_AIO_MSG_CLOSE_FILE, struct.pack(" None: +def _handshake(sock: socket.socket, client_name: str, op_id: str, version: int) -> None: """Run the classic NFC session handshake used by VDDK NBD.""" _send_nfc_msg(sock, NFC_MSG_HANDSHAKE, b"PlainText") _send_nfc_msg(sock, NFC_MSG_SESSION_PARAMS) @@ -427,43 +399,46 @@ def _handshake( if reply_type != NFC_MSG_SESSION_PARAMS_REPLY: raise NfcProtocolError( f"expected session-params reply {NFC_MSG_SESSION_PARAMS_REPLY}, " - f"got {reply_type}") + f"got {reply_type}" + ) _send_nfc_msg(sock, NFC_MSG_VERSION, struct.pack(" None: - disk._aio_roundtrip( - NFC_AIO_MSG_OPEN_SESSION, bytes(16)) - disk._aio_roundtrip( - NFC_AIO_MSG_SET_SOCK_OPTS, bytes(12)) - disk._aio_roundtrip( - NFC_AIO_MSG_SET_RES_POOL, struct.pack(" tuple[int, int]: @@ -472,21 +447,21 @@ def _parse_open_reply(body: bytes) -> tuple[int, int]: handle, file_type, _flags = struct.unpack_from(" NfcDisk: + session: NfcAuthSession, + disk_path: str, + client_name: str = "vddk", + op_id: str = "nbdmode", + version: int = NFC_PROTOCOL_VERSION, + read_only: bool = True, + compression: int = NFC_COMPRESSION_NONE, +) -> NfcDisk: """Open ``disk_path`` over the authenticated authd socket. Matches VDDK ``VixDiskLib_Open`` of a datastore path after the NFC @@ -506,31 +481,31 @@ def open_disk( compression: ``NFC_COMPRESSION_NONE`` or ``NFC_COMPRESSION_FASTLZ``. OPEN_FILE flags are unchanged; compression is per IO message. """ - if compression not in ( - NFC_COMPRESSION_NONE, NFC_COMPRESSION_FASTLZ): + if compression not in (NFC_COMPRESSION_NONE, NFC_COMPRESSION_FASTLZ): raise NotImplementedError( - f"NFC compression type {compression} is not supported") + f"NFC compression type {compression} is not supported" + ) if session.nfc_ssl: - sock = wrap_nfcssl_socket( - session.authd_sock, session.ticket.host) + sock = wrap_nfcssl_socket(session.authd_sock, session.ticket.host) else: sock = takeover_authd_socket(session.authd_sock) try: _handshake(sock, client_name, op_id, version) disk = NfcDisk( - sock, disk_path, handle=0, sector_size=NFC_SECTOR_SIZE, - compression=compression) + sock, + disk_path, + handle=0, + sector_size=NFC_SECTOR_SIZE, + compression=compression, + ) _aio_prepare(disk) path_b = disk_path.encode("utf-8") open_flags = ( - NFC_OPEN_FLAGS_READ_ONLY if read_only - else NFC_OPEN_FLAGS_READ_WRITE) - open_body = struct.pack( - " int: alg = flags & ( VIXDISKLIB_FLAG_OPEN_COMPRESSION_ZLIB | VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ - | VIXDISKLIB_FLAG_OPEN_COMPRESSION_SKIPZ) + | VIXDISKLIB_FLAG_OPEN_COMPRESSION_SKIPZ + ) if alg == 0: return nfc_open.NFC_COMPRESSION_NONE if alg == VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ: return nfc_open.NFC_COMPRESSION_FASTLZ if alg & (alg - 1): raise NotImplementedError( - "Cannot set two or more NBD compression algorithms at the " - "same time") - raise NotImplementedError( - f"NBD compression open flag 0x{alg:x} is not supported") + "Cannot set two or more NBD compression algorithms at the same time" + ) + raise NotImplementedError(f"NBD compression open flag 0x{alg:x} is not supported") -VIX_SUPPORTED_COMPATIBILITY_MODES = [ - "6.0", "6.5", "6.7", "7.0", "8.0"] +VIX_SUPPORTED_COMPATIBILITY_MODES = ["6.0", "6.5", "6.7", "7.0", "8.0"] def get_buffer(size: int): @@ -74,8 +72,7 @@ def get_buffer(size: int): def _parse_vm_moref(vmx_spec: Optional[str]) -> str: if not vmx_spec: - raise ValueError( - "vmx_spec is required (for example 'moref=vm-13098')") + raise ValueError("vmx_spec is required (for example 'moref=vm-13098')") if "=" in vmx_spec: kind, value = vmx_spec.split("=", 1) if kind.lower() != "moref" or not value: @@ -97,21 +94,23 @@ def _select_transport(transport_modes: Optional[str]) -> str: if mode in ("nbdssl", "nbd"): return mode raise NotImplementedError( - f"supported transports are nbdssl and nbd, got {transport_modes!r}") + f"supported transports are nbdssl and nbd, got {transport_modes!r}" + ) class _Connection: """VIM session plus the VM moref needed to issue an NFC ticket at Open.""" def __init__( - self, - si: vim.ServiceInstance, - vm_moref: str, - snapshot_ref: Optional[str], - thumbprint: Optional[str], - allow_untrusted: bool, - read_only: bool, - transport_mode: str) -> None: + self, + si: vim.ServiceInstance, + vm_moref: str, + snapshot_ref: Optional[str], + thumbprint: Optional[str], + allow_untrusted: bool, + read_only: bool, + transport_mode: str, + ) -> None: self.si = si self.vm_moref = vm_moref self.snapshot_ref = snapshot_ref @@ -124,11 +123,7 @@ class _Connection: class _DiskHandle: """Opened NFC disk plus the authd TLS socket it was taken from.""" - def __init__( - self, - disk: nfc_open.NfcDisk, - authd_sock, - transport_mode: str) -> None: + def __init__(self, disk: nfc_open.NfcDisk, authd_sock, transport_mode: str) -> None: self.disk = disk self.authd_sock = authd_sock self.transport_mode = transport_mode @@ -138,9 +133,10 @@ class VixDiskLibHandle: """VDDK-compatible handle backed by pyVmomi and the NFC replacement.""" def __init__( - self, - config_path: Optional[str] = None, - vixdisklib_compatibility_version: Optional[str] = None) -> None: + self, + config_path: Optional[str] = None, + vixdisklib_compatibility_version: Optional[str] = None, + ) -> None: """Accept the VDDK wrapper constructor; no native library is loaded. Args: @@ -164,18 +160,20 @@ class VixDiskLibHandle: raise ValueError( "Unsupported vixDiskLib version format '%s'. vixDiskLib " "compatibility mode must be of the form " - "'$major.$minor'" % version) from ex + "'$major.$minor'" % version + ) from ex version_used = version break if not version_used: raise Exception( "Could not initialize vixDiskLib with any of the following " - "versions: %s" % target_versions) + "versions: %s" % target_versions + ) LOG.info( - "Successfully initialized vixDiskLib with target version '%s'", - version_used) + "Successfully initialized vixDiskLib with target version '%s'", version_used + ) @classmethod def get_vix_disklib_name(cls) -> str: @@ -194,17 +192,18 @@ class VixDiskLibHandle: @contextlib.contextmanager def connect( - self, - server_name: str, - thumbprint: Optional[str], - username: str, - password: str, - vmx_spec: Optional[str] = None, - snapshot_ref: Optional[str] = None, - read_only: bool = True, - transport_modes: Optional[str] = None, - port: int = 443, - allow_untrusted: bool = False) -> Iterator[_Connection]: + self, + server_name: str, + thumbprint: Optional[str], + username: str, + password: str, + vmx_spec: Optional[str] = None, + snapshot_ref: Optional[str] = None, + read_only: bool = True, + transport_modes: Optional[str] = None, + port: int = 443, + allow_untrusted: bool = False, + ) -> Iterator[_Connection]: """Login to vCenter/ESXi. Matches ``VixDiskLib_ConnectEx``. The NFC ticket and authd handshake are deferred to ``open``, as in @@ -235,11 +234,17 @@ class VixDiskLibHandle: password, port=port, thumbprint=thumbprint, - allow_untrusted=allow_untrusted or not thumbprint) + allow_untrusted=allow_untrusted or not thumbprint, + ) conn = _Connection( - si, vm_moref, snapshot_ref, thumbprint, - allow_untrusted or not thumbprint, read_only, - transport_mode) + si, + vm_moref, + snapshot_ref, + thumbprint, + allow_untrusted or not thumbprint, + read_only, + transport_mode, + ) try: yield conn finally: @@ -247,10 +252,11 @@ class VixDiskLibHandle: @contextlib.contextmanager def open( - self, - conn: _Connection, - disk_path: str, - flags: int = VIXDISKLIB_FLAG_OPEN_READ_ONLY) -> Iterator[_DiskHandle]: + self, + conn: _Connection, + disk_path: str, + flags: int = VIXDISKLIB_FLAG_OPEN_READ_ONLY, + ) -> Iterator[_DiskHandle]: """Open ``disk_path`` over NFC. Matches ``VixDiskLib_Open``. Read-only opens request ``NfcGetVmFiles`` (VM only). The VMDK @@ -271,22 +277,21 @@ class VixDiskLibHandle: compression = _nfc_compression(flags) read_only = bool(flags & VIXDISKLIB_FLAG_OPEN_READ_ONLY) if not read_only and conn.read_only: - raise NotImplementedError( - "ConnectEx was read-only; cannot open for write") + raise NotImplementedError("ConnectEx was read-only; cannot open for write") vm = vim.VirtualMachine(conn.vm_moref, conn.si._stub) nfc_ssl = conn.transport_mode == "nbdssl" ticket = nfc_auth.get_nfc_ticket( - conn.si, vm, read_only=read_only, - disk_path=None if read_only else disk_path) + conn.si, vm, read_only=read_only, disk_path=None if read_only else disk_path + ) authd_sock = nfc_auth.connect_authd( - ticket, allow_untrusted=conn.allow_untrusted, nfc_ssl=nfc_ssl) - session = nfc_auth.NfcAuthSession( - conn.si, ticket, authd_sock, nfc_ssl=nfc_ssl) + ticket, allow_untrusted=conn.allow_untrusted, nfc_ssl=nfc_ssl + ) + session = nfc_auth.NfcAuthSession(conn.si, ticket, authd_sock, nfc_ssl=nfc_ssl) try: disk = nfc_open.open_disk( - session, disk_path, read_only=read_only, - compression=compression) + session, disk_path, read_only=read_only, compression=compression + ) except Exception: authd_sock.close() raise @@ -297,11 +302,12 @@ class VixDiskLibHandle: self.close(handle) def read( - self, - disk_handle: _DiskHandle, - start_sector: int, - num_sectors: int, - buf: Union[ctypes.Array, bytearray, memoryview]) -> None: + self, + disk_handle: _DiskHandle, + start_sector: int, + num_sectors: int, + buf: Union[ctypes.Array, bytearray, memoryview], + ) -> None: """Read ``num_sectors`` from ``start_sector`` into ``buf``. Args: @@ -313,18 +319,18 @@ class VixDiskLibHandle: data = disk_handle.disk.read(start_sector, num_sectors) if isinstance(buf, (bytearray, memoryview)): if len(buf) < len(data): - raise Exception( - f"read buffer is {len(buf)} bytes, need {len(data)}") - buf[:len(data)] = data + raise Exception(f"read buffer is {len(buf)} bytes, need {len(data)}") + buf[: len(data)] = data return ctypes.memmove(buf, data, len(data)) def write( - self, - disk_handle: _DiskHandle, - start_sector: int, - num_sectors: int, - buf: Union[ctypes.Array, bytes, bytearray, memoryview]) -> None: + self, + disk_handle: _DiskHandle, + start_sector: int, + num_sectors: int, + buf: Union[ctypes.Array, bytes, bytearray, memoryview], + ) -> None: """Write ``num_sectors`` from ``buf`` starting at ``start_sector``. Args: diff --git a/tests/integration/base.py b/tests/integration/base.py index 133ca5c..f2648ce 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -20,8 +20,7 @@ from pyVmomi import vim from openvixdisklib import nfc_auth from openvixdisklib.nfc_auth import NfcAuthSession -_REPO_ROOT = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..")) +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) _CONFIG_PATH = os.path.join(_REPO_ROOT, ".test_config.yaml") _CONFIG_KEYS = ( "host", @@ -61,9 +60,8 @@ class LabEnv: disk_path: str def authenticate( - self, - read_only: bool = True, - nfc_ssl: bool = True) -> NfcAuthSession: + self, read_only: bool = True, nfc_ssl: bool = True + ) -> NfcAuthSession: """Login to the lab vCenter and complete NFC authd for the temp VM.""" return nfc_auth.authenticate( host=self.host, @@ -74,10 +72,12 @@ class LabEnv: allow_untrusted=self.allow_untrusted, disk_path=None if read_only else self.disk_path, read_only=read_only, - nfc_ssl=nfc_ssl) + nfc_ssl=nfc_ssl, + ) def vixdisklib_connect_kwargs( - self, extra: Optional[dict[str, Any]] = None) -> dict[str, Any]: + self, extra: Optional[dict[str, Any]] = None + ) -> dict[str, Any]: """Return common ``VixDiskLib_ConnectEx`` arguments for the temp VM.""" kwargs: dict[str, Any] = { "server_name": self.host, @@ -107,7 +107,8 @@ def ensure_vddk_library_path() -> None: parts = [p for p in current.split(":") if p] if VDDK_DIR not in parts: os.environ["LD_LIBRARY_PATH"] = ( - VDDK_DIR if not current else f"{VDDK_DIR}:{current}") + VDDK_DIR if not current else f"{VDDK_DIR}:{current}" + ) def require_vddk() -> None: @@ -123,13 +124,13 @@ def _load_test_config() -> dict[str, Any]: if not os.path.isfile(_CONFIG_PATH): pytest.skip( "integration tests need .test_config.yaml in the repo " - "root; see README.md for a sample") + "root; see README.md for a sample" + ) with open(_CONFIG_PATH, encoding="utf-8") as config_file: data = yaml.safe_load(config_file) or {} missing = [key for key in _CONFIG_KEYS if key not in data] if missing: - raise RuntimeError( - f"{_CONFIG_PATH} is missing keys: {', '.join(missing)}") + raise RuntimeError(f"{_CONFIG_PATH} is missing keys: {', '.join(missing)}") return { "host": str(data["host"]), "port": int(data["port"]), @@ -142,28 +143,28 @@ def _load_test_config() -> dict[str, Any]: def _connect_vim( - host: str, - username: str, - password: str, - port: int, - thumbprint: str, - allow_untrusted: bool) -> vim.ServiceInstance: + host: str, + username: str, + password: str, + port: int, + thumbprint: str, + allow_untrusted: bool, +) -> vim.ServiceInstance: return nfc_auth.connect_vim( host, username, password, port=port, thumbprint=thumbprint, - allow_untrusted=allow_untrusted) + allow_untrusted=allow_untrusted, + ) def _wait_for_task(task: vim.Task) -> Any: deadline = time.monotonic() + _TASK_TIMEOUT_S - while task.info.state in ( - vim.TaskInfo.State.running, vim.TaskInfo.State.queued): + while task.info.state in (vim.TaskInfo.State.running, vim.TaskInfo.State.queued): if time.monotonic() > deadline: - raise TimeoutError( - f"timed out waiting for vSphere task {task}") + raise TimeoutError(f"timed out waiting for vSphere task {task}") time.sleep(_TASK_POLL_S) if task.info.state != vim.TaskInfo.State.success: raise RuntimeError(f"vSphere task failed: {task.info.error}") @@ -171,26 +172,28 @@ def _wait_for_task(task: vim.Task) -> Any: def _find_datacenter( - content: vim.ServiceInstanceContent, - datacenter_name: str) -> vim.Datacenter: + content: vim.ServiceInstanceContent, datacenter_name: str +) -> vim.Datacenter: matches = [ - entity for entity in content.rootFolder.childEntity - if isinstance(entity, vim.Datacenter) - and entity.name == datacenter_name] + entity + for entity in content.rootFolder.childEntity + if isinstance(entity, vim.Datacenter) and entity.name == datacenter_name + ] if not matches: raise RuntimeError(f"datacenter {datacenter_name!r} not found") return matches[0] -def _find_datastore( - datacenter: vim.Datacenter, datastore_name: str) -> vim.Datastore: +def _find_datastore(datacenter: vim.Datacenter, datastore_name: str) -> vim.Datastore: matches = [ - datastore for datastore in datacenter.datastore - if datastore.name == datastore_name] + datastore + for datastore in datacenter.datastore + if datastore.name == datastore_name + ] if not matches: raise RuntimeError( - f"datastore {datastore_name!r} not found in " - f"datacenter {datacenter.name!r}") + f"datastore {datastore_name!r} not found in datacenter {datacenter.name!r}" + ) return matches[0] @@ -200,17 +203,14 @@ def _vm_config_spec(vm_name: str, datastore_name: str) -> vim.vm.ConfigSpec: config.guestId = "otherGuest64" config.memoryMB = 128 config.numCPUs = 1 - config.files = vim.vm.FileInfo( - vmPathName=f"[{datastore_name}]") + config.files = vim.vm.FileInfo(vmPathName=f"[{datastore_name}]") controller = vim.vm.device.ParaVirtualSCSIController() controller.key = 1000 controller.busNumber = 0 - controller.sharedBus = ( - vim.vm.device.VirtualSCSIController.Sharing.noSharing) + controller.sharedBus = vim.vm.device.VirtualSCSIController.Sharing.noSharing controller_spec = vim.vm.device.VirtualDeviceSpec() - controller_spec.operation = ( - vim.vm.device.VirtualDeviceSpec.Operation.add) + controller_spec.operation = vim.vm.device.VirtualDeviceSpec.Operation.add controller_spec.device = controller backing = vim.vm.device.VirtualDisk.FlatVer2BackingInfo() @@ -225,8 +225,7 @@ def _vm_config_spec(vm_name: str, datastore_name: str) -> vim.vm.ConfigSpec: disk.backing = backing disk_spec = vim.vm.device.VirtualDeviceSpec() disk_spec.operation = vim.vm.device.VirtualDeviceSpec.Operation.add - disk_spec.fileOperation = ( - vim.vm.device.VirtualDeviceSpec.FileOperation.create) + disk_spec.fileOperation = vim.vm.device.VirtualDeviceSpec.FileOperation.create disk_spec.device = disk config.deviceChange = [controller_spec, disk_spec] @@ -238,8 +237,13 @@ def create_lab_vm() -> LabEnv: cfg = _load_test_config() thumbprint = nfc_auth.get_ssl_cert_thumbprint(cfg["host"], cfg["port"]) si = _connect_vim( - cfg["host"], cfg["username"], cfg["password"], cfg["port"], - thumbprint, cfg["allow_untrusted"]) + cfg["host"], + cfg["username"], + cfg["password"], + cfg["port"], + thumbprint, + cfg["allow_untrusted"], + ) vm = None try: content = si.RetrieveContent() @@ -247,22 +251,23 @@ def create_lab_vm() -> LabEnv: datastore = _find_datastore(datacenter, cfg["datastore"]) if not datastore.host: raise RuntimeError( - f"datastore {cfg['datastore']!r} is not mounted on any host") + f"datastore {cfg['datastore']!r} is not mounted on any host" + ) host = datastore.host[0].key pool = host.parent.resourcePool vm_name = _LAB_VM_PREFIX + uuid.uuid4().hex[:12] vm = _wait_for_task( datacenter.vmFolder.CreateVM_Task( - config=_vm_config_spec(vm_name, datastore.name), - pool=pool, - host=host)) + config=_vm_config_spec(vm_name, datastore.name), pool=pool, host=host + ) + ) disks = [ device.backing.fileName for device in vm.config.hardware.device - if isinstance(device, vim.vm.device.VirtualDisk)] + if isinstance(device, vim.vm.device.VirtualDisk) + ] if not disks: - raise RuntimeError( - f"temporary VM {vm_name!r} has no virtual disks") + raise RuntimeError(f"temporary VM {vm_name!r} has no virtual disks") return LabEnv( host=cfg["host"], port=cfg["port"], @@ -274,7 +279,8 @@ def create_lab_vm() -> LabEnv: thumbprint=thumbprint, vm_moref=vm._moId, vmx_spec=f"moref={vm._moId}", - disk_path=disks[0]) + disk_path=disks[0], + ) except Exception: if vm is not None: try: @@ -289,8 +295,13 @@ def create_lab_vm() -> LabEnv: def destroy_lab_vm(lab: LabEnv) -> None: """Power off and delete the temporary lab VM if it still exists.""" si = _connect_vim( - lab.host, lab.username, lab.password, lab.port, - lab.thumbprint, lab.allow_untrusted) + lab.host, + lab.username, + lab.password, + lab.port, + lab.thumbprint, + lab.allow_untrusted, + ) try: vm = vim.VirtualMachine(lab.vm_moref, si._stub) try: diff --git a/tests/integration/test_crosscheck.py b/tests/integration/test_crosscheck.py index 4a48b57..b8edb81 100644 --- a/tests/integration/test_crosscheck.py +++ b/tests/integration/test_crosscheck.py @@ -9,8 +9,7 @@ import pytest from openvixdisklib import openvixdisklib as open_vix from tests.integration import vixdisklib -from tests.integration.base import ( - LabEnv, SECTOR_AT_1GB, SECTOR_SIZE, pattern_bytes) +from tests.integration.base import SECTOR_AT_1GB, SECTOR_SIZE, LabEnv, pattern_bytes def _connect_extra(lab: LabEnv, module: Any) -> Optional[dict[str, Any]]: @@ -21,14 +20,12 @@ def _connect_extra(lab: LabEnv, module: Any) -> Optional[dict[str, Any]]: def _write_sectors( - lab: LabEnv, - module: Any, - payloads: dict[int, bytes], - flags: int = 0) -> None: + lab: LabEnv, module: Any, payloads: dict[int, bytes], flags: int = 0 +) -> None: """Write one sector at each index using a vixdisklib-compatible module.""" handle = module.VixDiskLibHandle( - vixdisklib_compatibility_version="8.0", - config_path=None) + vixdisklib_compatibility_version="8.0", config_path=None + ) buf = module.get_buffer(SECTOR_SIZE) kwargs = lab.vixdisklib_connect_kwargs(_connect_extra(lab, module)) with handle.connect(**kwargs) as conn: @@ -39,14 +36,12 @@ def _write_sectors( def _read_sectors( - lab: LabEnv, - module: Any, - sectors: tuple[int, ...], - flags: int = 0) -> dict[int, bytes]: + lab: LabEnv, module: Any, sectors: tuple[int, ...], flags: int = 0 +) -> dict[int, bytes]: """Read one sector at each index using a vixdisklib-compatible module.""" handle = module.VixDiskLibHandle( - vixdisklib_compatibility_version="8.0", - config_path=None) + vixdisklib_compatibility_version="8.0", config_path=None + ) buf = module.get_buffer(SECTOR_SIZE) result: dict[int, bytes] = {} kwargs = lab.vixdisklib_connect_kwargs(_connect_extra(lab, module)) @@ -60,26 +55,26 @@ def _read_sectors( def _assert_both_read( - lab: LabEnv, - sectors: tuple[int, ...], - expected: dict[int, bytes], - flags: int = 0) -> None: + lab: LabEnv, sectors: tuple[int, ...], expected: dict[int, bytes], flags: int = 0 +) -> None: vddk_data = _read_sectors(lab, vixdisklib, sectors, flags=flags) replacement = _read_sectors(lab, open_vix, sectors, flags=flags) for start in sectors: - assert vddk_data[start] == expected[start], ( - f"VDDK mismatch at sector {start}") + assert vddk_data[start] == expected[start], f"VDDK mismatch at sector {start}" assert replacement[start] == expected[start], ( - f"openvixdisklib mismatch at sector {start}") + f"openvixdisklib mismatch at sector {start}" + ) class TestCrosscheck: @pytest.mark.parametrize( "open_flags", [0, vixdisklib.VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ], - ids=["plain", "fastlz"]) + ids=["plain", "fastlz"], + ) def test_openvixdisklib_matches_vddk_sectors( - self, lab: LabEnv, vddk: None, open_flags: int) -> None: + self, lab: LabEnv, vddk: None, open_flags: int + ) -> None: """Writes from either library must be visible to both readers.""" sectors = (0, 1, SECTOR_AT_1GB) vddk_payloads = { diff --git a/tests/integration/test_nfc_open.py b/tests/integration/test_nfc_open.py index abb3db3..80bc4d1 100644 --- a/tests/integration/test_nfc_open.py +++ b/tests/integration/test_nfc_open.py @@ -6,24 +6,25 @@ import pytest from openvixdisklib import nfc_open -from tests.integration.base import LabEnv, SECTOR_SIZE, pattern_bytes +from tests.integration.base import SECTOR_SIZE, LabEnv, pattern_bytes class TestNfcOpen: - @pytest.mark.parametrize( - "nfc_ssl", [True, False], ids=["nbdssl", "nbd"]) + @pytest.mark.parametrize("nfc_ssl", [True, False], ids=["nbdssl", "nbd"]) @pytest.mark.parametrize( "compression", [nfc_open.NFC_COMPRESSION_NONE, nfc_open.NFC_COMPRESSION_FASTLZ], - ids=["plain", "fastlz"]) + ids=["plain", "fastlz"], + ) def test_open_disk_and_read_first_sector( - self, lab: LabEnv, nfc_ssl: bool, compression: int) -> None: + self, lab: LabEnv, nfc_ssl: bool, compression: int + ) -> None: """Open the temp VMDK, write sector 0, and read it back.""" expected = pattern_bytes(SECTOR_SIZE, b"NFC-OPEN-S0") with lab.authenticate(read_only=False, nfc_ssl=nfc_ssl) as session: with nfc_open.open_disk( - session, lab.disk_path, read_only=False, - compression=compression) as disk: + session, lab.disk_path, read_only=False, compression=compression + ) as disk: assert disk.path == lab.disk_path assert disk.handle > 0 assert disk.sector_size == SECTOR_SIZE diff --git a/tests/integration/test_nfc_read_write.py b/tests/integration/test_nfc_read_write.py index 1d796e6..a28c0ca 100644 --- a/tests/integration/test_nfc_read_write.py +++ b/tests/integration/test_nfc_read_write.py @@ -8,7 +8,7 @@ import os import pytest from openvixdisklib import nfc_open -from tests.integration.base import LabEnv, SECTOR_SIZE, pattern_bytes +from tests.integration.base import SECTOR_SIZE, LabEnv, pattern_bytes _32MIB = 32 * 1024 * 1024 @@ -17,9 +17,9 @@ class TestNfcReadWrite: @pytest.mark.parametrize( "compression", [nfc_open.NFC_COMPRESSION_NONE, nfc_open.NFC_COMPRESSION_FASTLZ], - ids=["plain", "fastlz"]) - def test_sector_writes_and_reads( - self, lab: LabEnv, compression: int) -> None: + ids=["plain", "fastlz"], + ) + def test_sector_writes_and_reads(self, lab: LabEnv, compression: int) -> None: """Write known patterns and read them back at several ranges.""" ranges = [ (0, 1), @@ -32,8 +32,8 @@ class TestNfcReadWrite: ] with lab.authenticate(read_only=False) as session: with nfc_open.open_disk( - session, lab.disk_path, read_only=False, - compression=compression) as disk: + session, lab.disk_path, read_only=False, compression=compression + ) as disk: for start, n_sectors in ranges: length = n_sectors * SECTOR_SIZE seed = f"NFC-R{start}:{n_sectors}-".encode() @@ -53,29 +53,29 @@ class TestNfcReadWrite: assert disk.read(1, 1) == two_to_write[SECTOR_SIZE:] big_seed = b"NFC-129-SECTOR-WRITE" - big_to_write = pattern_bytes( - 129 * SECTOR_SIZE, big_seed) + big_to_write = pattern_bytes(129 * SECTOR_SIZE, big_seed) disk.write(0, 129, big_to_write) big_got = disk.read(0, 129) assert big_got is not big_to_write assert big_got == big_to_write assert ( - big_got[SECTOR_SIZE:2 * SECTOR_SIZE] - == big_to_write[SECTOR_SIZE:2 * SECTOR_SIZE]) + big_got[SECTOR_SIZE : 2 * SECTOR_SIZE] + == big_to_write[SECTOR_SIZE : 2 * SECTOR_SIZE] + ) @pytest.mark.parametrize( "compression", [nfc_open.NFC_COMPRESSION_NONE, nfc_open.NFC_COMPRESSION_FASTLZ], - ids=["plain", "fastlz"]) - def test_write_and_read_32mb( - self, lab: LabEnv, compression: int) -> None: + ids=["plain", "fastlz"], + ) + def test_write_and_read_32mb(self, lab: LabEnv, compression: int) -> None: """Write 32 MiB (512 AIO chunks) and read it back in one request.""" n_sectors = _32MIB // SECTOR_SIZE to_write = os.urandom(_32MIB) with lab.authenticate(read_only=False) as session: with nfc_open.open_disk( - session, lab.disk_path, read_only=False, - compression=compression) as disk: + session, lab.disk_path, read_only=False, compression=compression + ) as disk: disk.write(0, n_sectors, to_write) got = disk.read(0, n_sectors) assert got is not to_write diff --git a/tests/integration/test_openvixdisklib.py b/tests/integration/test_openvixdisklib.py index 3eb57ed..a186460 100644 --- a/tests/integration/test_openvixdisklib.py +++ b/tests/integration/test_openvixdisklib.py @@ -3,18 +3,24 @@ """Exercise the VDDK-compatible openvixdisklib handle against the lab.""" +import pytest from pyVim.connect import Disconnect from pyVmomi import vim -import pytest from openvixdisklib import openvixdisklib as vixdisklib from tests.integration.base import ( - LabEnv, SECTOR_AT_1GB, SECTOR_SIZE, _connect_vim, _wait_for_task, - pattern_bytes) + SECTOR_AT_1GB, + SECTOR_SIZE, + LabEnv, + _connect_vim, + _wait_for_task, + pattern_bytes, +) def _virtual_disk_backing( - vm: vim.VirtualMachine) -> vim.vm.device.VirtualDevice.BackingInfo: + vm: vim.VirtualMachine, +) -> vim.vm.device.VirtualDevice.BackingInfo: """Return the lab VM's first virtual disk backing.""" for device in vm.config.hardware.device: if isinstance(device, vim.vm.device.VirtualDisk): @@ -27,19 +33,23 @@ class TestOpenvixdisklib: @pytest.mark.parametrize( "open_flags", [0, vixdisklib.VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ], - ids=["plain", "fastlz"]) + ids=["plain", "fastlz"], + ) def test_write_and_read_sector_zero_and_one_gib( - self, lab: LabEnv, transport_mode: str, open_flags: int) -> None: + self, lab: LabEnv, transport_mode: str, open_flags: int + ) -> None: """Write then read sector 0 and the sector at a 1 GiB offset.""" handle = vixdisklib.VixDiskLibHandle( - vixdisklib_compatibility_version="8.0", - config_path=None) + vixdisklib_compatibility_version="8.0", config_path=None + ) write_buf = vixdisklib.get_buffer(SECTOR_SIZE) read_buf = vixdisklib.get_buffer(SECTOR_SIZE) - connect_kwargs = lab.vixdisklib_connect_kwargs({ - "allow_untrusted": lab.allow_untrusted, - "transport_modes": transport_mode, - }) + connect_kwargs = lab.vixdisklib_connect_kwargs( + { + "allow_untrusted": lab.allow_untrusted, + "transport_modes": transport_mode, + } + ) patterns = { 0: pattern_bytes(SECTOR_SIZE, b"OVDL-S0"), SECTOR_AT_1GB: pattern_bytes(SECTOR_SIZE, b"OVDL-1GB"), @@ -63,19 +73,22 @@ class TestOpenvixdisklib: ``device.backing.fileName``; VDDK still opens it with a VM-only ticket and NFC ``OPEN_FILE``. """ - handle = vixdisklib.VixDiskLibHandle( - vixdisklib_compatibility_version="8.0") + handle = vixdisklib.VixDiskLibHandle(vixdisklib_compatibility_version="8.0") expected = pattern_bytes(SECTOR_SIZE, b"OVDL-RO") write_buf = vixdisklib.get_buffer(SECTOR_SIZE) read_buf = vixdisklib.get_buffer(SECTOR_SIZE) write_buf[:SECTOR_SIZE] = expected - write_kwargs = lab.vixdisklib_connect_kwargs({ - "allow_untrusted": lab.allow_untrusted, - }) - read_kwargs = lab.vixdisklib_connect_kwargs({ - "allow_untrusted": lab.allow_untrusted, - "read_only": True, - }) + write_kwargs = lab.vixdisklib_connect_kwargs( + { + "allow_untrusted": lab.allow_untrusted, + } + ) + read_kwargs = lab.vixdisklib_connect_kwargs( + { + "allow_untrusted": lab.allow_untrusted, + "read_only": True, + } + ) read_flags = vixdisklib.VIXDISKLIB_FLAG_OPEN_READ_ONLY def read_sector(path: str) -> bytes: @@ -92,12 +105,16 @@ class TestOpenvixdisklib: assert read_sector(lab.disk_path) == expected si = _connect_vim( - lab.host, lab.username, lab.password, lab.port, - lab.thumbprint, lab.allow_untrusted) + lab.host, + lab.username, + lab.password, + lab.port, + lab.thumbprint, + lab.allow_untrusted, + ) try: vm = vim.VirtualMachine(lab.vm_moref, si._stub) - _wait_for_task( - vm.CreateSnapshot_Task("ovdl-readonly", "", False, False)) + _wait_for_task(vm.CreateSnapshot_Task("ovdl-readonly", "", False, False)) backing = _virtual_disk_backing(vm) parent = getattr(backing, "parent", None) assert parent is not None diff --git a/tests/integration/test_vddk.py b/tests/integration/test_vddk.py index 2f27016..1f102e7 100644 --- a/tests/integration/test_vddk.py +++ b/tests/integration/test_vddk.py @@ -4,16 +4,15 @@ """Exercise native VDDK via tests.integration.vixdisklib against the lab.""" from tests.integration import vixdisklib -from tests.integration.base import LabEnv, SECTOR_SIZE, pattern_bytes +from tests.integration.base import SECTOR_SIZE, LabEnv, pattern_bytes class TestVddk: - def test_write_and_read_first_sector( - self, lab: LabEnv, vddk: None) -> None: + def test_write_and_read_first_sector(self, lab: LabEnv, vddk: None) -> None: """Open the temp VMDK with VDDK, write sector 0, and read it back.""" handle = vixdisklib.VixDiskLibHandle( - vixdisklib_compatibility_version="8.0", - config_path=None) + vixdisklib_compatibility_version="8.0", config_path=None + ) write_buf = vixdisklib.get_buffer(SECTOR_SIZE) read_buf = vixdisklib.get_buffer(SECTOR_SIZE) expected = pattern_bytes(SECTOR_SIZE, b"VDDK-S0") diff --git a/tests/integration/vix_disklib_errors.py b/tests/integration/vix_disklib_errors.py index 2f09e61..d9669fc 100644 --- a/tests/integration/vix_disklib_errors.py +++ b/tests/integration/vix_disklib_errors.py @@ -534,5 +534,5 @@ VIX_ERROR_CODE_MAP = { VIX_E_NET_HTTP_TOO_MANY_REDIRECTS: "Reached the maximum number of redirects.", VIX_E_NET_HTTP_TRANSFER: "Failure sending/receiving network data.", VIX_E_NET_HTTP_SSL_SECURITY: "An SSL error occurred.", - VIX_E_NET_HTTP_GENERIC: "A generic HTTP error occurred." + VIX_E_NET_HTTP_GENERIC: "A generic HTTP error occurred.", } diff --git a/tests/integration/vixdisklib.py b/tests/integration/vixdisklib.py index 4ae71a9..00de693 100755 --- a/tests/integration/vixdisklib.py +++ b/tests/integration/vixdisklib.py @@ -34,8 +34,7 @@ VIXDISKLIB_FLAG_OPEN_COMPRESSION_ZLIB = 16 VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ = 32 VIXDISKLIB_FLAG_OPEN_COMPRESSION_SKIPZ = 64 -VIX_SUPPORTED_COMPATIBILITY_MODES = [ - "6.0", "6.5", "6.7", "7.0", "8.0"] +VIX_SUPPORTED_COMPATIBILITY_MODES = ["6.0", "6.5", "6.7", "7.0", "8.0"] class VixDiskLibUidPasswdCreds(ctypes.Structure): @@ -84,12 +83,10 @@ def get_buffer(size): class VixDiskLibHandle(object): - """ Class which acts as a proxy for vixDiskLib-related operations: - """ - def __init__( - self, config_path=None, vixdisklib_compatibility_version=None): - self._vix_disklib = ctypes.cdll.LoadLibrary( - self.get_vix_disklib_name()) + """Class which acts as a proxy for vixDiskLib-related operations:""" + + def __init__(self, config_path=None, vixdisklib_compatibility_version=None): + self._vix_disklib = ctypes.cdll.LoadLibrary(self.get_vix_disklib_name()) self._setup_vix_disklib() if config_path: @@ -113,86 +110,119 @@ class VixDiskLibHandle(object): raise ValueError( "Unsupported vixDiskLib version format '%s'. vixDiskLib " "compatibility mode must be of the form " - "'$major.$minor'" % version) from ex + "'$major.$minor'" % version + ) from ex try: - self._check_err(self._vix_disklib.VixDiskLib_InitEx( - major_ver, minor_ver, None, None, None, None, config_path)) + self._check_err( + self._vix_disklib.VixDiskLib_InitEx( + major_ver, minor_ver, None, None, None, None, config_path + ) + ) version_used = version break except Exception: LOG.debug( "Failed to initialize vixDiskLib using compatibility " "version '%s'. Trying next version. Error trace: %s", - version, traceback.format_exc()) + version, + traceback.format_exc(), + ) if not version_used: raise Exception( "Could not initialize vixDiskLib with any of the following " - "versions: %s" % target_versions) + "versions: %s" % target_versions + ) LOG.info( - "Successfully initialized vixDiskLib with target version '%s'", - version_used) + "Successfully initialized vixDiskLib with target version '%s'", version_used + ) @classmethod def get_vix_disklib_name(cls): vixDiskLibName = None - if os.name == 'nt': - vixDiskLibName = 'vixDiskLib.dll' + if os.name == "nt": + vixDiskLibName = "vixDiskLib.dll" else: - vixDiskLibName = 'libvixDiskLib.so' + vixDiskLibName = "libvixDiskLib.so" return vixDiskLibName def _setup_vix_disklib(self): self._vix_disklib.VixDiskLib_InitEx.argtypes = [ - ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p, ctypes.c_void_p, - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p] + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p, + ] self._vix_disklib.VixDiskLib_InitEx.restype = ctypes.c_uint64 self._vix_disklib.VixDiskLib_GetErrorText.argtypes = [ - ctypes.c_uint64, ctypes.c_char_p] + ctypes.c_uint64, + ctypes.c_char_p, + ] self._vix_disklib.VixDiskLib_GetErrorText.restype = ctypes.c_void_p - self._vix_disklib.VixDiskLib_FreeErrorText.arg_types = [ - ctypes.c_char_p] + self._vix_disklib.VixDiskLib_FreeErrorText.arg_types = [ctypes.c_char_p] self._vix_disklib.VixDiskLib_FreeErrorText.restype = None self._vix_disklib.VixDiskLib_ListTransportModes.argtypes = [] - self._vix_disklib.VixDiskLib_ListTransportModes.restype = ( - ctypes.c_char_p) + self._vix_disklib.VixDiskLib_ListTransportModes.restype = ctypes.c_char_p - self._vix_disklib.VixDiskLib_GetTransportMode.argtypes = [ - ctypes.c_void_p] - self._vix_disklib.VixDiskLib_GetTransportMode.restype = ( - ctypes.c_char_p) + self._vix_disklib.VixDiskLib_GetTransportMode.argtypes = [ctypes.c_void_p] + self._vix_disklib.VixDiskLib_GetTransportMode.restype = ctypes.c_char_p self._vix_disklib.VixDiskLib_ConnectEx.argtypes = [ - ctypes.POINTER(VixDiskLibConnectParams), ctypes.c_char, - ctypes.c_char_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_void_p)] + ctypes.POINTER(VixDiskLibConnectParams), + ctypes.c_char, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.POINTER(ctypes.c_void_p), + ] self._vix_disklib.VixDiskLib_ConnectEx.restype = ctypes.c_uint64 self._vix_disklib.VixDiskLib_Open.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_uint32, - ctypes.POINTER(ctypes.c_void_p)] + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_uint32, + ctypes.POINTER(ctypes.c_void_p), + ] self._vix_disklib.VixDiskLib_Open.restype = ctypes.c_uint64 self._vix_disklib.VixDiskLib_Read.argtypes = [ - ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint64, ctypes.c_char_p] + ctypes.c_void_p, + ctypes.c_uint64, + ctypes.c_uint64, + ctypes.c_char_p, + ] self._vix_disklib.VixDiskLib_Read.restype = ctypes.c_uint64 self._vix_disklib.VixDiskLib_Write.argtypes = [ - ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint64, ctypes.c_char_p] + ctypes.c_void_p, + ctypes.c_uint64, + ctypes.c_uint64, + ctypes.c_char_p, + ] self._vix_disklib.VixDiskLib_Write.restype = ctypes.c_uint64 self._vix_disklib.VixDiskLib_GetMetadataKeys.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_uint64, - ctypes.POINTER(ctypes.c_uint64)] + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_uint64, + ctypes.POINTER(ctypes.c_uint64), + ] self._vix_disklib.VixDiskLib_GetMetadataKeys.restype = ctypes.c_uint64 self._vix_disklib.VixDiskLib_ReadMetadata.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint64, - ctypes.POINTER(ctypes.c_uint64)] + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_uint64, + ctypes.POINTER(ctypes.c_uint64), + ] self._vix_disklib.VixDiskLib_ReadMetadata.restype = ctypes.c_uint64 self._vix_disklib.VixDiskLib_Close.argtypes = [ctypes.c_void_p] @@ -207,10 +237,10 @@ class VixDiskLibHandle(object): def _check_err(self, err, allowed_values=[vix_disklib_errors.VIX_OK]): if err not in allowed_values: err_msg = self._vix_disklib.VixDiskLib_GetErrorText(err, None) - err_msg_copy = str(ctypes.cast( - err_msg, ctypes.c_char_p).value.decode()) + err_msg_copy = str(ctypes.cast(err_msg, ctypes.c_char_p).value.decode()) self._vix_disklib.VixDiskLib_FreeErrorText( - ctypes.cast(err_msg, ctypes.c_char_p)) + ctypes.cast(err_msg, ctypes.c_char_p) + ) msg = None if err == vix_disklib_errors.VIX_E_OUT_OF_MEMORY: @@ -220,7 +250,8 @@ class VixDiskLibHandle(object): "by vCenter, so enough RAM to run the export is required " "on all hosts. To force the export from the specific host " "the VM is on, create a Coriolis endpoint with the DNS " - "name/IP address of that host.") + "name/IP address of that host." + ) if err == vix_disklib_errors.VIX_E_HOST_NETWORK_CONN_REFUSED: msg = ( "The ESXi host performing the CBT export refused " @@ -231,7 +262,8 @@ class VixDiskLibHandle(object): "facilitate this. Alternatively, try connecting Coriolis " "directly to the specific ESXi host which is running the " "VM(s) to be migrated by creating a Coriolis endpoint " - "using the DNS name/IP address of the host itself.") + "using the DNS name/IP address of the host itself." + ) if err == vix_disklib_errors.VIX_E_CANNOT_CONNECT_TO_HOST: msg = ( @@ -239,7 +271,8 @@ class VixDiskLibHandle(object): "CBT export. If the Coriolis Endpoint connects to a " "vSphere host, please try connecting Coriolis to the ESXi " "host directly. If problem persists, try re-enabling CBT " - "on the VM, or moving it to another ESXi host.") + "on the VM, or moving it to another ESXi host." + ) err_msg = err_msg_copy if msg: @@ -250,7 +283,7 @@ class VixDiskLibHandle(object): def get_transport_modes(self): transport_modes = self._vix_disklib.VixDiskLib_ListTransportModes() - return transport_modes.decode().split(':') + return transport_modes.decode().split(":") def get_transport_mode(self, disk_handle): t_mode = self._vix_disklib.VixDiskLib_GetTransportMode(disk_handle) @@ -258,9 +291,17 @@ class VixDiskLibHandle(object): @contextlib.contextmanager def connect( - self, server_name, thumbprint, username, password, - vmx_spec=None, snapshot_ref=None, read_only=True, - transport_modes=None, port=443): + self, + server_name, + thumbprint, + username, + password, + vmx_spec=None, + snapshot_ref=None, + read_only=True, + transport_modes=None, + port=443, + ): LOG.debug("Connecting VixDiskLib: %s", server_name) connectParams = VixDiskLibConnectParams() @@ -283,9 +324,15 @@ class VixDiskLibHandle(object): snapshot_ref = snapshot_ref.encode() conn = ctypes.c_void_p() - self._check_err(self._vix_disklib.VixDiskLib_ConnectEx( - connectParams, read_only, snapshot_ref, transport_modes, - ctypes.byref(conn))) + self._check_err( + self._vix_disklib.VixDiskLib_ConnectEx( + connectParams, + read_only, + snapshot_ref, + transport_modes, + ctypes.byref(conn), + ) + ) try: yield conn finally: @@ -296,21 +343,30 @@ class VixDiskLibHandle(object): LOG.debug("Openning VixDiskLib disk: %s", disk_path) disk_handle = ctypes.c_void_p() - self._check_err(self._vix_disklib.VixDiskLib_Open( - conn, disk_path.encode(), flags, ctypes.byref(disk_handle))) + self._check_err( + self._vix_disklib.VixDiskLib_Open( + conn, disk_path.encode(), flags, ctypes.byref(disk_handle) + ) + ) try: yield disk_handle finally: self.close(disk_handle) def read(self, disk_handle, start_sector, num_sectors, buf): - self._check_err(self._vix_disklib.VixDiskLib_Read( - disk_handle, start_sector, num_sectors, buf)) + self._check_err( + self._vix_disklib.VixDiskLib_Read( + disk_handle, start_sector, num_sectors, buf + ) + ) def write(self, disk_handle, start_sector, num_sectors, buf): """Write ``num_sectors`` from ``buf`` starting at ``start_sector``.""" - self._check_err(self._vix_disklib.VixDiskLib_Write( - disk_handle, start_sector, num_sectors, buf)) + self._check_err( + self._vix_disklib.VixDiskLib_Write( + disk_handle, start_sector, num_sectors, buf + ) + ) def close(self, disk_handle): LOG.debug("Closing VixDiskLib disk handle: %s", disk_handle) diff --git a/tests/perf/test_compare.py b/tests/perf/test_compare.py index edf187a..1a595f0 100644 --- a/tests/perf/test_compare.py +++ b/tests/perf/test_compare.py @@ -10,7 +10,7 @@ from typing import Any from openvixdisklib import openvixdisklib as open_vix from tests.integration import vixdisklib -from tests.integration.base import LabEnv, SECTOR_SIZE, pattern_bytes +from tests.integration.base import SECTOR_SIZE, LabEnv, pattern_bytes _SIZES = ( ("64KiB", 64 * 1024), @@ -19,8 +19,7 @@ _SIZES = ( ) -def _connect_extra( - lab: LabEnv, module: Any, transport_mode: str) -> dict[str, Any]: +def _connect_extra(lab: LabEnv, module: Any, transport_mode: str) -> dict[str, Any]: """Return extra ``connect`` kwargs needed by ``module``.""" extra: dict[str, Any] = {"transport_modes": transport_mode} if module is open_vix: @@ -29,31 +28,31 @@ def _connect_extra( def _time_write_read( - lab: LabEnv, - module: Any, - payload: bytes, - flags: int = 0, - transport_mode: str = "nbdssl") -> tuple[float, float]: + lab: LabEnv, + module: Any, + payload: bytes, + flags: int = 0, + transport_mode: str = "nbdssl", +) -> tuple[float, float]: """Write ``payload`` at sector 0, read it back, and return durations.""" n_sectors = len(payload) // SECTOR_SIZE handle = module.VixDiskLibHandle( - vixdisklib_compatibility_version="8.0", - config_path=None) + vixdisklib_compatibility_version="8.0", config_path=None + ) write_buf = module.get_buffer(len(payload)) read_buf = module.get_buffer(len(payload)) - write_buf[:len(payload)] = payload - kwargs = lab.vixdisklib_connect_kwargs( - _connect_extra(lab, module, transport_mode)) + write_buf[: len(payload)] = payload + kwargs = lab.vixdisklib_connect_kwargs(_connect_extra(lab, module, transport_mode)) with handle.connect(**kwargs) as conn: with handle.open(conn, lab.disk_path, flags=flags) as disk: started = time.perf_counter() handle.write(disk, 0, n_sectors, write_buf) write_s = time.perf_counter() - started - read_buf[:len(payload)] = b"\xa5" * len(payload) + read_buf[: len(payload)] = b"\xa5" * len(payload) started = time.perf_counter() handle.read(disk, 0, n_sectors, read_buf) read_s = time.perf_counter() - started - assert read_buf.raw[:len(payload)] == payload + assert read_buf.raw[: len(payload)] == payload return write_s, read_s @@ -82,27 +81,42 @@ class TestCompare: for mode_name, flags in open_modes: for name, module in libraries: write_s, read_s = _time_write_read( - lab, module, payload, flags=flags, - transport_mode=transport_mode) - rows.append(( - label, - transport_mode, - mode_name, - name, - write_s, - read_s, - _mib_per_s(nbytes, write_s), - _mib_per_s(nbytes, read_s), - )) + lab, + module, + payload, + flags=flags, + transport_mode=transport_mode, + ) + rows.append( + ( + label, + transport_mode, + mode_name, + name, + write_s, + read_s, + _mib_per_s(nbytes, write_s), + _mib_per_s(nbytes, read_s), + ) + ) print() print( f"{'size':<14} {'transport':<10} {'flags':<8} {'library':<16} " f"{'write_s':>10} {'read_s':>10} " - f"{'write_MiB/s':>12} {'read_MiB/s':>12}") + f"{'write_MiB/s':>12} {'read_MiB/s':>12}" + ) for ( - label, transport_mode, mode_name, name, - write_s, read_s, write_r, read_r) in rows: + label, + transport_mode, + mode_name, + name, + write_s, + read_s, + write_r, + read_r, + ) in rows: print( f"{label:<14} {transport_mode:<10} {mode_name:<8} {name:<16} " f"{write_s:10.3f} {read_s:10.3f} " - f"{write_r:12.1f} {read_r:12.1f}") + f"{write_r:12.1f} {read_r:12.1f}" + )