justin/OpenVixDiskLib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add fastlz compression support

Browse Source
This commit is contained in:
Lucian Petrut
2026-09-07 13:10:10 +00:00
parent b7131ec0af
commit 56a1c36c80
13 changed files with 522 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/test_crosscheck.py
+21 -12
View File
@@ -5,6 +5,8 @@
from typing import Any, Optional
import pytest
from openvixdisklib import openvixdisklib as open_vix
from tests.integration import vixdisklib
from tests.integration.base import (
@@ -21,7 +23,8 @@ def _connect_extra(lab: LabEnv, module: Any) -> Optional[dict[str, Any]]:
def _write_sectors(
lab: LabEnv,
module: Any,
payloads: dict[int, bytes]) -> None:
payloads: dict[int, bytes],
flags: int = 0) -> None:
"""Write one sector at each index using a vixdisklib-compatible module."""
handle = module.VixDiskLibHandle(
vixdisklib_compatibility_version="8.0",
@@ -29,7 +32,7 @@ def _write_sectors(
buf = module.get_buffer(SECTOR_SIZE)
kwargs = lab.vixdisklib_connect_kwargs(_connect_extra(lab, module))
with handle.connect(**kwargs) as conn:
with handle.open(conn, lab.disk_path, flags=0) as disk:
with handle.open(conn, lab.disk_path, flags=flags) as disk:
for start, data in payloads.items():
buf[:SECTOR_SIZE] = data
handle.write(disk, start, 1, buf)
@@ -38,7 +41,8 @@ def _write_sectors(
def _read_sectors(
lab: LabEnv,
module: Any,
sectors: tuple[int, ...]) -> dict[int, bytes]:
sectors: tuple[int, ...],
flags: int = 0) -> dict[int, bytes]:
"""Read one sector at each index using a vixdisklib-compatible module."""
handle = module.VixDiskLibHandle(
vixdisklib_compatibility_version="8.0",
@@ -47,7 +51,7 @@ def _read_sectors(
result: dict[int, bytes] = {}
kwargs = lab.vixdisklib_connect_kwargs(_connect_extra(lab, module))
with handle.connect(**kwargs) as conn:
with handle.open(conn, lab.disk_path, flags=0) as disk:
with handle.open(conn, lab.disk_path, flags=flags) as disk:
for start in sectors:
buf[:SECTOR_SIZE] = b"\xa5" * SECTOR_SIZE
handle.read(disk, start, 1, buf)
@@ -58,9 +62,10 @@ def _read_sectors(
def _assert_both_read(
lab: LabEnv,
sectors: tuple[int, ...],
expected: dict[int, bytes]) -> None:
vddk_data = _read_sectors(lab, vixdisklib, sectors)
replacement = _read_sectors(lab, open_vix, sectors)
expected: dict[int, bytes],
flags: int = 0) -> None:
vddk_data = _read_sectors(lab, vixdisklib, sectors, flags=flags)
replacement = _read_sectors(lab, open_vix, sectors, flags=flags)
for start in sectors:
assert vddk_data[start] == expected[start], (
f"VDDK mismatch at sector {start}")
@@ -69,8 +74,12 @@ def _assert_both_read(
class TestCrosscheck:
@pytest.mark.parametrize(
"open_flags",
[0, vixdisklib.VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ],
ids=["plain", "fastlz"])
def test_openvixdisklib_matches_vddk_sectors(
self, lab: LabEnv, vddk: None) -> None:
self, lab: LabEnv, vddk: None, open_flags: int) -> None:
"""Writes from either library must be visible to both readers."""
sectors = (0, 1, SECTOR_AT_1GB)
vddk_payloads = {
@@ -78,13 +87,13 @@ class TestCrosscheck:
1: pattern_bytes(SECTOR_SIZE, b"XCHK-VDDK-S1"),
SECTOR_AT_1GB: pattern_bytes(SECTOR_SIZE, b"XCHK-VDDK-1G"),
}
_write_sectors(lab, vixdisklib, vddk_payloads)
_assert_both_read(lab, sectors, vddk_payloads)
_write_sectors(lab, vixdisklib, vddk_payloads, flags=open_flags)
_assert_both_read(lab, sectors, vddk_payloads, flags=open_flags)
ovdl_payloads = {
0: pattern_bytes(SECTOR_SIZE, b"XCHK-OVDL-S0"),
1: pattern_bytes(SECTOR_SIZE, b"XCHK-OVDL-S1"),
SECTOR_AT_1GB: pattern_bytes(SECTOR_SIZE, b"XCHK-OVDL-1G"),
}
_write_sectors(lab, open_vix, ovdl_payloads)
_assert_both_read(lab, sectors, ovdl_payloads)
_write_sectors(lab, open_vix, ovdl_payloads, flags=open_flags)
_assert_both_read(lab, sectors, ovdl_payloads, flags=open_flags)
tests/integration/test_nfc_open.py
+7 -2
View File
@@ -12,13 +12,18 @@ from tests.integration.base import LabEnv, SECTOR_SIZE, pattern_bytes
class TestNfcOpen:
@pytest.mark.parametrize(
"nfc_ssl", [True, False], ids=["nbdssl", "nbd"])
@pytest.mark.parametrize(
"compression",
[nfc_open.NFC_COMPRESSION_NONE, nfc_open.NFC_COMPRESSION_FASTLZ],
ids=["plain", "fastlz"])
def test_open_disk_and_read_first_sector(
self, lab: LabEnv, nfc_ssl: bool) -> None:
self, lab: LabEnv, nfc_ssl: bool, compression: int) -> None:
"""Open the temp VMDK, write sector 0, and read it back."""
expected = pattern_bytes(SECTOR_SIZE, b"NFC-OPEN-S0")
with lab.authenticate(read_only=False, nfc_ssl=nfc_ssl) as session:
with nfc_open.open_disk(
session, lab.disk_path, read_only=False) as disk:
session, lab.disk_path, read_only=False,
compression=compression) as disk:
assert disk.path == lab.disk_path
assert disk.handle > 0
assert disk.sector_size == SECTOR_SIZE
tests/integration/test_nfc_read_write.py
+18 -4
View File
@@ -5,6 +5,8 @@
import os
import pytest
from openvixdisklib import nfc_open
from tests.integration.base import LabEnv, SECTOR_SIZE, pattern_bytes
@@ -12,7 +14,12 @@ _32MIB = 32 * 1024 * 1024
class TestNfcReadWrite:
def test_sector_writes_and_reads(self, lab: LabEnv) -> None:
@pytest.mark.parametrize(
"compression",
[nfc_open.NFC_COMPRESSION_NONE, nfc_open.NFC_COMPRESSION_FASTLZ],
ids=["plain", "fastlz"])
def test_sector_writes_and_reads(
self, lab: LabEnv, compression: int) -> None:
"""Write known patterns and read them back at several ranges."""
ranges = [
(0, 1),
@@ -25,7 +32,8 @@ class TestNfcReadWrite:
]
with lab.authenticate(read_only=False) as session:
with nfc_open.open_disk(
session, lab.disk_path, read_only=False) as disk:
session, lab.disk_path, read_only=False,
compression=compression) as disk:
for start, n_sectors in ranges:
length = n_sectors * SECTOR_SIZE
seed = f"NFC-R{start}:{n_sectors}-".encode()
@@ -55,13 +63,19 @@ class TestNfcReadWrite:
big_got[SECTOR_SIZE:2 * SECTOR_SIZE]
== big_to_write[SECTOR_SIZE:2 * SECTOR_SIZE])
def test_write_and_read_32mb(self, lab: LabEnv) -> None:
@pytest.mark.parametrize(
"compression",
[nfc_open.NFC_COMPRESSION_NONE, nfc_open.NFC_COMPRESSION_FASTLZ],
ids=["plain", "fastlz"])
def test_write_and_read_32mb(
self, lab: LabEnv, compression: int) -> None:
"""Write 32 MiB (512 AIO chunks) and read it back in one request."""
n_sectors = _32MIB // SECTOR_SIZE
to_write = os.urandom(_32MIB)
with lab.authenticate(read_only=False) as session:
with nfc_open.open_disk(
session, lab.disk_path, read_only=False) as disk:
session, lab.disk_path, read_only=False,
compression=compression) as disk:
disk.write(0, n_sectors, to_write)
got = disk.read(0, n_sectors)
assert got is not to_write
tests/integration/test_openvixdisklib.py
+6 -2
View File
@@ -12,8 +12,12 @@ from tests.integration.base import (
class TestOpenvixdisklib:
@pytest.mark.parametrize("transport_mode", ["nbdssl", "nbd"])
@pytest.mark.parametrize(
"open_flags",
[0, vixdisklib.VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ],
ids=["plain", "fastlz"])
def test_write_and_read_sector_zero_and_one_gib(
self, lab: LabEnv, transport_mode: str) -> None:
self, lab: LabEnv, transport_mode: str, open_flags: int) -> None:
"""Write then read sector 0 and the sector at a 1 GiB offset."""
handle = vixdisklib.VixDiskLibHandle(
vixdisklib_compatibility_version="8.0",
@@ -30,7 +34,7 @@ class TestOpenvixdisklib:
}
assert handle.get_transport_modes() == ["nbdssl", "nbd"]
with handle.connect(**connect_kwargs) as conn:
with handle.open(conn, lab.disk_path, flags=0) as disk:
with handle.open(conn, lab.disk_path, flags=open_flags) as disk:
assert handle.get_transport_mode(disk) == transport_mode
for start, expected in patterns.items():
write_buf[:SECTOR_SIZE] = expected
tests/perf/test_compare.py
+26 -16
View File
@@ -29,7 +29,8 @@ def _connect_extra(lab: LabEnv, module: Any) -> Optional[dict[str, Any]]:
def _time_write_read(
lab: LabEnv,
module: Any,
payload: bytes) -> tuple[float, float]:
payload: bytes,
flags: int = 0) -> tuple[float, float]:
"""Write ``payload`` at sector 0, read it back, and return durations."""
n_sectors = len(payload) // SECTOR_SIZE
handle = module.VixDiskLibHandle(
@@ -40,7 +41,7 @@ def _time_write_read(
write_buf[:len(payload)] = payload
kwargs = lab.vixdisklib_connect_kwargs(_connect_extra(lab, module))
with handle.connect(**kwargs) as conn:
with handle.open(conn, lab.disk_path, flags=0) as disk:
with handle.open(conn, lab.disk_path, flags=flags) as disk:
started = time.perf_counter()
handle.write(disk, 0, n_sectors, write_buf)
write_s = time.perf_counter() - started
@@ -65,24 +66,33 @@ class TestCompare:
("vddk", vixdisklib),
("openvixdisklib", open_vix),
)
rows: list[tuple[str, str, float, float, float, float]] = []
open_modes = (
("plain", 0),
("fastlz", vixdisklib.VIXDISKLIB_FLAG_OPEN_COMPRESSION_FASTLZ),
)
rows: list[tuple[str, str, str, float, float, float, float]] = []
for label, nbytes in _SIZES:
payload = pattern_bytes(nbytes, f"PERF-{label}-".encode())
for name, module in libraries:
write_s, read_s = _time_write_read(lab, module, payload)
rows.append((
label,
name,
write_s,
read_s,
_mib_per_s(nbytes, write_s),
_mib_per_s(nbytes, read_s),
))
for mode_name, flags in open_modes:
for name, module in libraries:
write_s, read_s = _time_write_read(
lab, module, payload, flags=flags)
rows.append((
label,
mode_name,
name,
write_s,
read_s,
_mib_per_s(nbytes, write_s),
_mib_per_s(nbytes, read_s),
))
print()
print(
f"{'size':<14} {'library':<16} {'write_s':>10} {'read_s':>10} "
f"{'size':<14} {'flags':<8} {'library':<16} "
f"{'write_s':>10} {'read_s':>10} "
f"{'write_MiB/s':>12} {'read_MiB/s':>12}")
for label, name, write_s, read_s, write_r, read_r in rows:
for label, mode_name, name, write_s, read_s, write_r, read_r in rows:
print(
f"{label:<14} {name:<16} {write_s:10.3f} {read_s:10.3f} "
f"{label:<14} {mode_name:<8} {name:<16} "
f"{write_s:10.3f} {read_s:10.3f} "
f"{write_r:12.1f} {read_r:12.1f}")