Add VDO tests converted from vdo-devel perl tests

requirements.txt

@@ -1,4 +1,4 @@
-toml==0.10.2
-pudb
 numpy
+pudb
 pyyaml
+tomli_w

src/dmtest/config.py

@@ -1,4 +1,4 @@
-import toml
+import tomllib
 
 
 # Linux reordered my nvme drives once and I ran tests across
@@ -21,7 +21,7 @@ def validate(cfg):
 
 
 def read_config(path="config.toml"):
-    with open(path, "r") as f:
-        config = toml.load(f)
+    with open(path, "rb") as f:
+        config = tomllib.load(f)
         validate(config)
         return config

src/dmtest/dependency_tracker.py

@@ -1,4 +1,5 @@
-import toml
+import tomllib
+import tomli_w
 from enum import Enum
 from pathlib import Path
 from typing import Union
@@ -61,14 +62,16 @@ def get_all_targets(self):
 
 def read_test_deps(path):
     deps = TestDeps()
-    deps._deps = toml.load(path)
+    with open(path, "rb") as f:
+        deps._deps = tomllib.load(f)
     return deps
 
 
 def write_test_deps(path, deps):
     if deps._updated:
-        with open(path, "w") as f:
-            toml.dump(deps._deps, f)
+        sorted_deps = dict(sorted(deps._deps.items()))
+        with open(path, "wb") as f:
+            tomli_w.dump(sorted_deps, f)
 
 
 global_dep_tracker = None

src/dmtest/device_mapper/interface.py

@@ -68,7 +68,8 @@ def status(name, *args):
 
 
 def table(name):
-    run(f"dmsetup table {name}")
+    (_, stdout, _) = run(f"dmsetup table {name}")
+    return stdout
 
 
 def info(name):
@@ -86,3 +87,7 @@ def parse_event_nr(txt):
 def wait(name, event_nr):
     (_, stdout, _) = run(f"dmsetup wait -v {name} {event_nr}")
     return parse_event_nr(stdout)
+
+
+def rename(old_name, new_name):
+    run(f"dmsetup rename {old_name} {new_name}")

src/dmtest/fs.py

@@ -61,7 +61,8 @@ def check_cmd(self):
 
     def mkfs_cmd(self, opts):
         discard_arg = "discard" if opts.get("discard", True) else "nodiscard"
-        return f"mkfs.ext4 -F -E lazy_itable_init=1,{discard_arg} {self._dev}"
+        lazy_init = 1 if opts.get("lazy_itable_init", True) else 0
+        return f"mkfs.ext4 -F -E lazy_itable_init={lazy_init},{discard_arg} {self._dev}"
 
 
 class Xfs(BaseFS):

src/dmtest/gendatablocks.py

@@ -2,6 +2,7 @@
 import dmtest.process as process
 
 import logging
+import mmap
 import os
 import struct
 
@@ -344,7 +345,7 @@ def trim(self, fsync=False):
         """Trim the block range, if supported."""
         byte_offset = self.block_size * self.offset
         byte_size = self.block_size * self.block_count
-        process.run(f"blkdiscard -o {byte_offset} -l {byte_size} {self.path}")
+        process.run(f"blkdiscard --force -o {byte_offset} -l {byte_size} {self.path}")
         if fsync:
             with open(self.path, "w+") as f:
                 os.fsync(f.fileno())
@@ -458,31 +459,46 @@ def write(self,
         if (compress < 0.0) or (compress > 0.96):
             raise ValueError("the compression fraction " + str(compress)
                              + " is invalid")
-        if direct:
-            # Direct I/O requires special handling to ensure proper
-            # alignment of the in-memory buffer being written to the
-            # destination. We don't do that yet.
-            raise NotImplementedError("direct I/O is not yet supported")
         stream = BlockStream(tag, dedupe, compress)
 
         flags = os.O_WRONLY
+        if direct:
+            flags |= os.O_DIRECT
         if sync:
             flags |= os.O_SYNC
         if self.create:
             flags |= os.O_CREAT
 
         logging.info(f"writing {self.block_count*self.block_size} bytes tagged \"{tag}\""
                      f" to {self.path} at {self.block_size*self.offset} open flags {flags}")
-        with os.fdopen(os.open(self.path, flags), "r+b") as fd:
-            self._seek(fd)
-            for n in range(0, self.block_count):
-                data = stream.generate(n, self.block_size)
-                fd.write(data)
-                stream.counter += 1
-            fd.flush()
-            if fsync:
-                os.fsync(fd)
         self.streams.append(stream)
+        if direct:
+            fd = os.open(self.path, flags)
+            try:
+                os.lseek(fd, self.block_size * self.offset, os.SEEK_SET)
+                buf = mmap.mmap(-1, self.block_size)
+                try:
+                    for n in range(0, self.block_count):
+                        data = stream.generate(n, self.block_size)
+                        buf[:] = data
+                        os.write(fd, buf)
+                        stream.counter += 1
+                finally:
+                    buf.close()
+                if fsync:
+                    os.fsync(fd)
+            finally:
+                os.close(fd)
+        else:
+            with os.fdopen(os.open(self.path, flags), "r+b") as fd:
+                self._seek(fd)
+                for n in range(0, self.block_count):
+                    data = stream.generate(n, self.block_size)
+                    fd.write(data)
+                    stream.counter += 1
+                fd.flush()
+                if fsync:
+                    os.fsync(fd)
 
 def make_block_range(path: str,
                      block_count: int = 1,

src/dmtest/process.py

@@ -33,6 +33,6 @@ def run(command, raise_on_fail=True):
         log.info(f"return code: {proc.returncode}")
     return_code = proc.returncode
     if return_code and raise_on_fail:
-        log.error("process failed unexpectedly, raising exception")
+        log.error(f"process '{command}' failed with exit status {return_code}, raising exception")
         raise subprocess.CalledProcessError(return_code, command)
     return (return_code, stdout.strip(), stderr.strip())

src/dmtest/vdo/basic_01_tests.py

@@ -0,0 +1,60 @@
+"""VDO basic functional test.
+
+Verifies VDO persistence by writing data to a filesystem on VDO, stopping
+the VDO device, restarting it, and verifying the data is still readable.
+"""
+from dmtest.assertions import assert_equal, assert_string_in
+from dmtest.utils import get_dmesg_log
+from dmtest.vdo.utils import standard_vdo, standard_stack, mounted_fs
+import dmtest.process as process
+import time
+
+def t_basic(fix):
+    """Basic VDO functional test: write files, stop/start VDO, verify data persists."""
+    # Create VDO with slab_bits=17 (SLAB_BITS_SMALL)
+    with standard_vdo(fix, slab_bits=17) as vdo:
+        with mounted_fs(vdo.path, format=True) as mount_point:
+            # Create file foo1 with "Hello World"
+            file1 = f"{mount_point}/foo1"
+            process.run(f"bash -c 'echo Hello World > {file1}'")
+
+            # Create subdirectory dir2
+            dir2 = f"{mount_point}/dir2"
+            process.run(f"mkdir {dir2}")
+
+            # Copy foo1 to dir2/foo2
+            file2 = f"{dir2}/foo2"
+            process.run(f"cp {file1} {file2}")
+
+            # Copy foo1 to foo3
+            file3 = f"{mount_point}/foo3"
+            process.run(f"cp {file1} {file3}")
+
+            # Drop caches
+            process.run("echo 1 > /proc/sys/vm/drop_caches")
+
+            # Verify content of foo1 and foo2
+            result1 = process.run(f"cat {file1}")
+            assert_equal(result1[1].strip(), "Hello World")
+
+            result2 = process.run(f"cat {file2}")
+            assert_equal(result2[1].strip(), "Hello World")
+
+    # VDO device is now stopped (exited context manager)
+    # Get kernel log timestamp before restarting
+    start_time = time.time()
+
+    # Restart VDO device without reformatting
+    with standard_vdo(fix, format=False, slab_bits=17) as vdo:
+        with mounted_fs(vdo.path) as mount_point:
+            # Verify content of foo3
+            file3 = f"{mount_point}/foo3"
+            result3 = process.run(f"cat {file3}")
+            assert_equal(result3[1].strip(), "Hello World")
+
+    # Check kernel log for VDO startup message
+    log_message = get_dmesg_log(start_time)
+    assert_string_in(log_message, "VDO commencing normal operation")
+
+def register(tests):
+    tests.register("/vdo/basic/basic01", t_basic)

src/dmtest/vdo/basic_fs_dedupe_tests.py

@@ -0,0 +1,88 @@
+"""
+VDO BasicFSDedupe test - Filesystem-level deduplication verification
+"""
+import logging as log
+import os
+import shutil
+import tempfile
+
+from dmtest.assertions import assert_near
+from dmtest.gendatablocks import make_block_range
+from dmtest.vdo.stats import vdo_stats, make_delta_stats
+from dmtest.vdo.utils import standard_vdo, fsync, mounted_fs
+
+
+def t_basic_fs_dedupe(fix) -> None:
+    """
+    Basic filesystem-level deduplication test that writes a dataset twice
+    and verifies deduplication achieves expected space savings.
+    """
+    MB = 1024 * 1024
+    num_files = 32
+    file_size_mb = 8
+    blocks_per_file = file_size_mb * MB // 4096  # 8MB / 4KB = 2048 blocks
+
+    with standard_vdo(fix) as vdo:
+        with mounted_fs(vdo.path, format=True, lazy_itable_init=False) as mount_point:
+            # Create subdirectories on VDO filesystem
+            original_dir = os.path.join(mount_point, "original")
+            copy1_dir = os.path.join(mount_point, "copy1")
+            os.makedirs(original_dir)
+            os.makedirs(copy1_dir)
+
+            # Record initial stats after filesystem setup
+            fsync(vdo.path)
+            initial_stats = vdo_stats(vdo)
+
+            # Generate dataset in a scratch directory
+            with tempfile.TemporaryDirectory() as scratch_dir:
+                dataset_dir = os.path.join(scratch_dir, "dataset")
+                os.makedirs(dataset_dir)
+
+                log.info(f"Generating dataset: {num_files} files × {file_size_mb}MB each = 256MB total")
+                for i in range(num_files):
+                    file_path = os.path.join(dataset_dir, f"file_{i:08d}")
+                    # Create the file first
+                    with open(file_path, 'w') as f:
+                        pass
+                    # Write data to the file
+                    block_range = make_block_range(file_path, blocks_per_file)
+                    block_range.write(f"BFD{i:04d}", dedupe=0.0, fsync=False)
+
+                # Copy dataset to "original" directory
+                log.info("Copying dataset to 'original' directory")
+                shutil.copytree(dataset_dir, os.path.join(original_dir, "data"))
+
+                # Sync and check stats after first write
+                fsync(vdo.path)
+                stats_after_first = vdo_stats(vdo)
+                delta_first = make_delta_stats(stats_after_first, initial_stats)
+
+                data_blocks = delta_first['dataBlocksUsed']
+                logical_blocks = delta_first['logicalBlocksUsed']
+                ratio_first = data_blocks / logical_blocks if logical_blocks > 0 else 0
+
+                log.info(f"After first write: data={data_blocks}, logical={logical_blocks}, ratio={ratio_first:.3f}")
+                # Verify minimal deduplication on first write (filesystem metadata may cause some variance)
+                assert_near(ratio_first, 1.0, 0.1, "Data-to-logical ratio after first write")
+
+                # Copy the same dataset to "copy1" directory (duplicate copy)
+                log.info("Copying dataset to 'copy1' directory (duplicate)")
+                shutil.copytree(dataset_dir, os.path.join(copy1_dir, "data"))
+
+                # Sync and check stats after second write
+                fsync(vdo.path)
+                stats_after_second = vdo_stats(vdo)
+                delta_second = make_delta_stats(stats_after_second, initial_stats)
+
+                data_blocks_2 = delta_second['dataBlocksUsed']
+                logical_blocks_2 = delta_second['logicalBlocksUsed']
+                ratio_second = data_blocks_2 / logical_blocks_2 if logical_blocks_2 > 0 else 0
+
+                log.info(f"After second write: data={data_blocks_2}, logical={logical_blocks_2}, ratio={ratio_second:.3f}")
+                # Verify significant deduplication on second write (~50% ratio expected)
+                assert_near(ratio_second, 0.5, 0.05, "Data-to-logical ratio after second write (with dedupe)")
+
+
+def register(tests):
+    tests.register("/vdo/basic/fs-dedupe", t_basic_fs_dedupe)