justin/OpenVixDiskLib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add fastlz compression support

Browse Source
This commit is contained in:
Lucian Petrut
2026-09-07 13:10:10 +00:00
parent b7131ec0af
commit 56a1c36c80
13 changed files with 522 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openvixdisklib/fastlz.py
+296
View File
@@ -0,0 +1,296 @@
# Copyright 2026 Cloudbase Solutions Srl
# All Rights Reserved.
#
# Python port of FastLZ (Byte-aligned LZ77), used by VMware NFC
# VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ.
#
# FastLZ - Copyright (C) 2005-2020 Ariya Hidayat <[email protected]>
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
"""FastLZ compress / decompress used by NFC NBD compression."""
from __future__ import annotations
MAX_COPY = 32
MAX_LEN = 264
MAX_L1_DISTANCE = 8192
MAX_L2_DISTANCE = 8191
MAX_FARDISTANCE = 65535 + MAX_L2_DISTANCE - 1
HASH_LOG = 13
HASH_SIZE = 1 << HASH_LOG
HASH_MASK = HASH_SIZE - 1
def _readu32(buf: bytes | bytearray, offset: int) -> int:
return buf[offset] | (buf[offset + 1] << 8) | (
buf[offset + 2] << 16) | (buf[offset + 3] << 24)
def _hash(value: int) -> int:
return ((value * 2654435769) >> (32 - HASH_LOG)) & HASH_MASK
def _cmp(src: bytes, p: int, q: int, q_end: int) -> int:
start = p
if _readu32(src, p) == _readu32(src, q):
p += 4
q += 4
while q < q_end:
mismatch = src[p] != src[q]
p += 1
q += 1
if mismatch:
break
return p - start
def _extend_match(out: bytearray, ref: int, length: int) -> None:
while length:
take = min(length, len(out) - ref)
out.extend(out[ref:ref + take])
length -= take
def _literals(src: bytes, src_off: int, runs: int, out: bytearray) -> None:
while runs >= MAX_COPY:
out.append(MAX_COPY - 1)
out.extend(src[src_off:src_off + MAX_COPY])
src_off += MAX_COPY
runs -= MAX_COPY
if runs > 0:
out.append(runs - 1)
out.extend(src[src_off:src_off + runs])
def _flz1_match(length: int, distance: int, out: bytearray) -> None:
distance -= 1
if length > MAX_LEN - 2:
while length > MAX_LEN - 2:
out.append((7 << 5) + (distance >> 8))
out.append(MAX_LEN - 2 - 7 - 2)
out.append(distance & 255)
length -= MAX_LEN - 2
if length < 7:
out.append((length << 5) + (distance >> 8))
out.append(distance & 255)
else:
out.append((7 << 5) + (distance >> 8))
out.append(length - 7)
out.append(distance & 255)
def _flz2_match(length: int, distance: int, out: bytearray) -> None:
distance -= 1
if distance < MAX_L2_DISTANCE:
if length < 7:
out.append((length << 5) + (distance >> 8))
out.append(distance & 255)
else:
out.append((7 << 5) + (distance >> 8))
length -= 7
while length >= 255:
out.append(255)
length -= 255
out.append(length)
out.append(distance & 255)
elif length < 7:
distance -= MAX_L2_DISTANCE
out.append((length << 5) + 31)
out.append(255)
out.append(distance >> 8)
out.append(distance & 255)
else:
distance -= MAX_L2_DISTANCE
out.append((7 << 5) + 31)
length -= 7
while length >= 255:
out.append(255)
length -= 255
out.append(length)
out.append(255)
out.append(distance >> 8)
out.append(distance & 255)
def _compress_level(src: bytes, level: int) -> bytes:
length = len(src)
ip = 0
ip_bound = length - 4
ip_limit = length - 12 - 1
out = bytearray()
htab = [0] * HASH_SIZE
max_distance = MAX_L1_DISTANCE if level == 1 else MAX_FARDISTANCE
anchor = 0
ip = 2
while ip < ip_limit:
while True:
seq = _readu32(src, ip) & 0xffffff
h = _hash(seq)
ref = htab[h]
htab[h] = ip
distance = ip - ref
cmp_val = (
_readu32(src, ref) & 0xffffff
if 0 <= ref < ip and distance < max_distance
else 0x1000000)
if ip >= ip_limit:
break
ip += 1
if seq == cmp_val:
break
if ip >= ip_limit:
break
ip -= 1
if level == 2 and distance >= MAX_L2_DISTANCE:
if src[ref + 3] != src[ip + 3] or src[ref + 4] != src[ip + 4]:
ip += 1
continue
if ip > anchor:
_literals(src, anchor, ip - anchor, out)
match_len = _cmp(src, ref + 3, ip + 3, ip_bound)
if level == 1:
_flz1_match(match_len, distance, out)
else:
_flz2_match(match_len, distance, out)
ip += match_len
seq = _readu32(src, ip)
htab[_hash(seq & 0xffffff)] = ip
ip += 1
htab[_hash(seq >> 8)] = ip
ip += 1
anchor = ip
_literals(src, anchor, length - anchor, out)
if level == 2 and out:
out[0] |= 1 << 5
return bytes(out)
def compress(data: bytes) -> bytes:
"""Compress ``data`` with FastLZ (level 1 below 64 KiB, else level 2).
Args:
data: Uncompressed bytes. FastLZ needs at least 16 bytes.
"""
if len(data) < 65536:
return _compress_level(data, 1)
return _compress_level(data, 2)
def decompress(data: bytes, maxout: int) -> bytes:
"""Decompress FastLZ ``data`` into at most ``maxout`` bytes.
Args:
data: Compressed FastLZ buffer.
maxout: Expected uncompressed length (output cap).
"""
if not data:
raise ValueError("FastLZ input is empty")
level = (data[0] >> 5) + 1
if level == 1:
return _decompress_level1(data, maxout)
if level == 2:
return _decompress_level2(data, maxout)
raise ValueError(f"unsupported FastLZ level {level}")
def _decompress_level1(src: bytes, maxout: int) -> bytes:
ip = 0
ip_limit = len(src)
ip_bound = ip_limit - 2
out = bytearray()
ctrl = src[ip] & 31
ip += 1
while True:
if ctrl >= 32:
length = (ctrl >> 5) - 1
ofs = (ctrl & 31) << 8
if length == 6:
if ip > ip_bound:
raise ValueError("truncated FastLZ match")
length += src[ip]
ip += 1
if ip >= ip_limit:
raise ValueError("truncated FastLZ match distance")
ofs += src[ip]
ip += 1
length += 3
ref = len(out) - ofs - 1
if ref < 0 or len(out) + length > maxout:
raise ValueError("FastLZ match out of range")
_extend_match(out, ref, length)
else:
ctrl += 1
if ip + ctrl > ip_limit or len(out) + ctrl > maxout:
raise ValueError("truncated FastLZ literals")
out.extend(src[ip:ip + ctrl])
ip += ctrl
if ip > ip_bound:
break
ctrl = src[ip]
ip += 1
return bytes(out)
def _decompress_level2(src: bytes, maxout: int) -> bytes:
ip = 0
ip_limit = len(src)
ip_bound = ip_limit - 2
out = bytearray()
ctrl = src[ip] & 31
ip += 1
while True:
if ctrl >= 32:
length = (ctrl >> 5) - 1
ofs = (ctrl & 31) << 8
if length == 6:
while True:
if ip > ip_bound:
raise ValueError("truncated FastLZ match")
code = src[ip]
ip += 1
length += code
if code != 255:
break
if ip >= ip_limit:
raise ValueError("truncated FastLZ match distance")
code = src[ip]
ip += 1
ofs += code
length += 3
ref = len(out) - ofs - 1
if code == 255 and ofs == (31 << 8):
if ip >= ip_bound:
raise ValueError("truncated FastLZ far match")
ofs = (src[ip] << 8) + src[ip + 1]
ip += 2
ref = len(out) - ofs - MAX_L2_DISTANCE - 1
if ref < 0 or len(out) + length > maxout:
raise ValueError("FastLZ match out of range")
_extend_match(out, ref, length)
else:
ctrl += 1
if ip + ctrl > ip_limit or len(out) + ctrl > maxout:
raise ValueError("truncated FastLZ literals")
out.extend(src[ip:ip + ctrl])
ip += ctrl
if ip >= ip_limit:
break
ctrl = src[ip]
ip += 1
return bytes(out)
openvixdisklib/nfc_open.py
+64 -10
View File
@@ -26,6 +26,7 @@ import socket
import ssl
import struct
from openvixdisklib import fastlz
from openvixdisklib.nfc_auth import NfcAuthSession, _ssl_client_context
NFC_MSG_SIZE = 264
@@ -69,6 +70,12 @@ NFC_OPEN_FLAGS_READ_WRITE = 0x1A
NFC_AIO_IO_WRITE = 0
NFC_AIO_IO_READ = 1
# High 32 bits of the IO opcode uint64. Captured from VDDK FASTLZ:
# writes that shrink go on the wire as type 2; incompressible writes
# fall back to type 0 with raw extra data.
NFC_COMPRESSION_NONE = 0
NFC_COMPRESSION_FASTLZ = 2
class NfcProtocolError(ConnectionError):
"""Raised when an NFC message is malformed or reports failure."""
@@ -168,7 +175,8 @@ class NfcDisk:
sock: socket.socket,
path: str,
handle: int,
sector_size: int) -> None:
sector_size: int,
compression: int = NFC_COMPRESSION_NONE) -> None:
"""Wrap an AIO session that already has ``path`` open.
Args:
@@ -176,12 +184,15 @@ class NfcDisk:
path: Datastore path that was opened.
handle: Server file handle from OPEN_FILE.
sector_size: Sector size from the OPEN_FILE reply.
compression: NFC IO compression type (``NFC_COMPRESSION_NONE``
or ``NFC_COMPRESSION_FASTLZ``).
"""
self._sock = sock
self._op_id = 0
self.path = path
self.handle = handle
self.sector_size = sector_size
self.compression = compression
self._closed = False
def _next_op_id(self) -> int:
@@ -226,7 +237,8 @@ class NfcDisk:
byte units. If the length exceeds the AIO buffer (64 KiB) the
server replies with several same-``opId`` fragments, which are
placed by the fragment byte offset in the reply (they may arrive
out of order).
out of order). FASTLZ open requests compression in the opcode;
each reply fragment may be compressed (type 2) or raw (type 0).
Args:
start_sector: Sector offset from the start of the disk.
@@ -236,10 +248,11 @@ class NfcDisk:
raise ValueError("num_sectors must be at least 1")
length = num_sectors * self.sector_size
offset = start_sector * self.sector_size
opcode = NFC_AIO_IO_READ | (self.compression << 32)
payload = struct.pack(
"<QQQQIII",
self.handle,
NFC_AIO_IO_READ,
opcode,
offset,
length,
length,
@@ -264,7 +277,10 @@ class NfcDisk:
f"AIO IO reply payload too short: {rsize}")
# Fragments may arrive out of order. Offset 28 is the byte
# offset of this chunk within the request (0, 65536, …),
# not a 0-based index.
# not a 0-based index. Offset 32 is the uncompressed
# fragment length. When the opcode high bits are FastLZ
# (2), extra data is compressed and offset 36 is its size.
opcode = struct.unpack_from("<Q", body, 8)[0]
dest, chunk_len = struct.unpack_from("<II", body, 28)
if (
dest in seen
@@ -274,7 +290,25 @@ class NfcDisk:
f"AIO IO chunk offset={dest} length={chunk_len} invalid, "
f"request {length}")
seen.add(dest)
data[dest:dest + chunk_len] = _recvn(self._sock, chunk_len)
ctype = opcode >> 32
if ctype == NFC_COMPRESSION_FASTLZ:
comp_len = struct.unpack_from("<I", body, 36)[0]
extra = _recvn(self._sock, comp_len)
try:
chunk = fastlz.decompress(extra, chunk_len)
except ValueError as exc:
raise NfcProtocolError(
f"FastLZ read fragment failed: {exc}") from exc
if len(chunk) != chunk_len:
raise NfcProtocolError(
f"FastLZ read got {len(chunk)} bytes, "
f"expected {chunk_len}")
elif ctype == NFC_COMPRESSION_NONE:
chunk = _recvn(self._sock, chunk_len)
else:
raise NfcProtocolError(
f"unsupported NFC IO compression type {ctype}")
data[dest:dest + chunk_len] = chunk
filled += chunk_len
return bytes(data)
@@ -288,6 +322,7 @@ class NfcDisk:
Matches ``VixDiskLib_Write``: one ``NFC_AIO_MSG_IO`` request per
chunk in byte units, with sector bytes sent after the 44-byte
payload. Chunks larger than the AIO buffer (64 KiB) are split.
FASTLZ open compresses each chunk when that shrinks it.
Args:
start_sector: Sector offset from the start of the disk.
@@ -317,16 +352,26 @@ class NfcDisk:
data: bytes) -> None:
length = num_sectors * self.sector_size
offset = start_sector * self.sector_size
extra = data
ctype = NFC_COMPRESSION_NONE
extra_len = length
if self.compression == NFC_COMPRESSION_FASTLZ and length >= 16:
compressed = fastlz.compress(data)
if compressed and len(compressed) < length:
extra = compressed
ctype = NFC_COMPRESSION_FASTLZ
extra_len = len(compressed)
opcode = NFC_AIO_IO_WRITE | (ctype << 32)
payload = struct.pack(
"<QQQQIII",
self.handle,
NFC_AIO_IO_WRITE,
opcode,
offset,
length,
length,
length,
extra_len,
0)
self._aio_roundtrip(NFC_AIO_MSG_IO, payload, extra=data)
self._aio_roundtrip(NFC_AIO_MSG_IO, payload, extra=extra)
def close(self) -> None:
"""Close the VMDK, the AIO session, and the classic NFC session."""
@@ -422,7 +467,8 @@ def open_disk(
client_name: str = "vddk",
op_id: str = "nbdmode",
version: int = NFC_PROTOCOL_VERSION,
read_only: bool = True) -> NfcDisk:
read_only: bool = True,
compression: int = NFC_COMPRESSION_NONE) -> NfcDisk:
"""Open ``disk_path`` over the authenticated authd socket.
Matches VDDK ``VixDiskLib_Open`` of a datastore path after the NFC
@@ -439,7 +485,13 @@ def open_disk(
op_id: NFC operation id; VDDK NBD sends ``nbdmode``.
version: Client NFC protocol version (lab ESXi answered 11).
read_only: When True, open with VDDK's read-only NFC flags.
compression: ``NFC_COMPRESSION_NONE`` or ``NFC_COMPRESSION_FASTLZ``.
OPEN_FILE flags are unchanged; compression is per IO message.
"""
if compression not in (
NFC_COMPRESSION_NONE, NFC_COMPRESSION_FASTLZ):
raise NotImplementedError(
f"NFC compression type {compression} is not supported")
if session.nfc_ssl:
sock = wrap_nfcssl_socket(
session.authd_sock, session.ticket.host)
@@ -447,7 +499,9 @@ def open_disk(
sock = takeover_authd_socket(session.authd_sock)
try:
_handshake(sock, client_name, op_id, version)
disk = NfcDisk(sock, disk_path, handle=0, sector_size=NFC_SECTOR_SIZE)
disk = NfcDisk(
sock, disk_path, handle=0, sector_size=NFC_SECTOR_SIZE,
compression=compression)
_aio_prepare(disk)
path_b = disk_path.encode("utf-8")
open_flags = (
openvixdisklib/openvixdisklib.py
+24 -10
View File
@@ -44,10 +44,24 @@ VIXDISKLIB_FLAG_OPEN_COMPRESSION_ZLIB = 16
VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ = 32
VIXDISKLIB_FLAG_OPEN_COMPRESSION_SKIPZ = 64
_COMPRESSION_FLAGS = (
VIXDISKLIB_FLAG_OPEN_COMPRESSION_ZLIB
| VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ
| VIXDISKLIB_FLAG_OPEN_COMPRESSION_SKIPZ)
def _nfc_compression(flags: int) -> int:
"""Return the NFC IO compression type for VixDiskLib open ``flags``."""
alg = flags & (
VIXDISKLIB_FLAG_OPEN_COMPRESSION_ZLIB
| VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ
| VIXDISKLIB_FLAG_OPEN_COMPRESSION_SKIPZ)
if alg == 0:
return nfc_open.NFC_COMPRESSION_NONE
if alg == VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ:
return nfc_open.NFC_COMPRESSION_FASTLZ
if alg & (alg - 1):
raise NotImplementedError(
"Cannot set two or more NBD compression algorithms at the "
"same time")
raise NotImplementedError(
f"NBD compression open flag 0x{alg:x} is not supported")
VIX_SUPPORTED_COMPATIBILITY_MODES = [
"6.0", "6.5", "6.7", "7.0", "8.0"]
@@ -243,13 +257,12 @@ class VixDiskLibHandle:
conn: Connection from ``connect``.
disk_path: Datastore path of the VMDK.
flags: Open flags. ``VIXDISKLIB_FLAG_OPEN_READ_ONLY`` opens
the disk read-only; omit it for write. Compression flags
are not implemented.
the disk read-only; omit it for write.
``VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ`` compresses
NFC IO. zlib and skipz are not implemented.
"""
LOG.debug("Openning VixDiskLib disk: %s", disk_path)
if flags & _COMPRESSION_FLAGS:
raise NotImplementedError(
"NBD compression open flags are not supported")
compression = _nfc_compression(flags)
read_only = bool(flags & VIXDISKLIB_FLAG_OPEN_READ_ONLY)
if not read_only and conn.read_only:
raise NotImplementedError(
@@ -265,7 +278,8 @@ class VixDiskLibHandle:
conn.si, ticket, authd_sock, nfc_ssl=nfc_ssl)
try:
disk = nfc_open.open_disk(
session, disk_path, read_only=read_only)
session, disk_path, read_only=read_only,
compression=compression)
except Exception:
authd_sock.close()
raise