From cbd3f8fa9e5ada40f05c6e4fefb7ae051096e0b9 Mon Sep 17 00:00:00 2001 From: torgeiru Date: Sat, 27 Sep 2025 03:26:49 +0200 Subject: [PATCH 01/10] Added schema and logic for handling several Virtio devices VirtioFS: Adding necessary qemu_args and starting VirtioFSD. Installed pinned VirtioFSD as a dependency. VirtioCON: Redirecting output to a given path for console VirtioPMEM: Using a image path for the persistent memory --- default.nix | 11 ++++++- vmrunner/vm.schema.json | 37 ++++++++++++++++++++++ vmrunner/vmrunner.py | 68 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 114 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index 2e3c73109..58109c87b 100644 --- a/default.nix +++ b/default.nix @@ -1,4 +1,12 @@ -{ pkgs ? import {} }: +{ + pkgs ? import {}, + + virtiofsd_pinned ? import (fetchTarball { + name = "nixpkgs-pinned-for-virtiofsd"; + url = "https://github.com/NixOS/nixpkgs/archive/13e8d35b7d6028b7198f8186bc0347c6abaa2701.tar.gz"; + sha256 = "0nqbvgmm7pbpyd8ihg2bi62pxihj8r673bc9ll4qhi6xwlfqac5q"; + }) {}, + }: pkgs.python3.pkgs.buildPythonPackage rec { pname = "vmrunner"; version = "0.16.0"; @@ -16,6 +24,7 @@ pkgs.python3.pkgs.buildPythonPackage rec { future jsonschema psutil + virtiofsd_pinned.virtiofsd ]; create_bridge = ./vmrunner/bin/create_bridge.sh; diff --git a/vmrunner/vm.schema.json b/vmrunner/vm.schema.json index 8153faaa5..e7631e446 100644 --- a/vmrunner/vm.schema.json +++ b/vmrunner/vm.schema.json @@ -80,6 +80,43 @@ } }, + "virtiocon" : { + "description" : "VirtioCON device. Only used for testing Virtio queues for now", + "type" : "object", + "properties" : { + "path" : { + "description" : "The path to redirect the guest output", + "type" : "string" + } + } + }, + + "virtiofs" : { + "description" : "VirtioFS device", + "type" : "object", + "properties" : { + "socket" : { "type" : "string" }, + "shared" : { + "description" : "Directory to be shared with guest", + "type" : "string" + } + } + }, + + "virtiopmem": { + "description" : "VirtioPMEM device", + "type" : "object", + "properties" : { + "image" : { + "description" : "Path to persistent image file", + "type" : "string" + }, + "size" : { + "description" : "Persistent memory size in megabytes", + "type" : "number" + } + } + }, "mem" : { "description" : "Amount of memory in megabytes", diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 27237cc1c..f4667b4d1 100755 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -20,6 +20,7 @@ import re import traceback import signal +import time from enum import Enum import platform import psutil @@ -186,6 +187,7 @@ def __init__(self, config): self._enable_kvm = False # must be explicitly turned on at boot. self._sudo = False # Set to true if sudo is available self._proc = None # A running subprocess + self._virtiofsd_proc = None # pylint: disable-next=unused-argument def boot_in_hypervisor(self, multiboot=False, debug=False, kernel_args="", image_name="", allow_sudo = False, enable_kvm = False): @@ -516,6 +518,39 @@ def net_arg(self, backend, device, if_name = "net0", mac = None, bridge = None, return ["-device", device, "-netdev", netdev] + def init_virtiocon(self, path): + """ creates a console device and redirects to the path given """ + qemu_args = ["-device", "virtio-serial-pci,disable-legacy=on,id=virtio-serial0"] + qemu_args += ["-device", "virtserialport,chardev=char0"] + qemu_args += ["-chardev", f"file,id=char0,path={path}"] + + return qemu_args + + def init_virtiofs(self, socket, shared, mem): + """ initializes virtiofs by launching virtiofsd and creating a virtiofs device """ + virtiofsd_args = ["virtiofsd", "--log-level", "debug", "--socket", socket, "--shared-dir", shared, "--sandbox", "none"] + self._virtiofsd_proc = subprocess.Popen(virtiofsd_args) # pylint: disable=consider-using-with + # , stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + time.sleep(0.1) + if self._virtiofsd_proc.poll(): + raise Exception("VirtioFSD failed to start") + + info("Successfully started VirtioFSD!") + + qemu_args = ["-machine", "memory-backend=mem0"] + qemu_args += ["-chardev", f"socket,id=virtiofsd0,path={socket}"] + qemu_args += ["-device", "vhost-user-fs-pci,chardev=virtiofsd0,tag=virtiofs0"] + qemu_args += ["-object", f"memory-backend-memfd,id=mem0,size={mem}M,share=on"] + + return qemu_args + + def init_pmem(self, path, size): + """ creates a pmem device with image path as memory mapped backend """ + qemu_args = ["-object", f"memory-backend-file,id=pmemdev0,mem-path={path},size={size}M,share=on"] + qemu_args += ["-device", "virtio-pmem-pci,memdev=pmemdev0"] + + return qemu_args + def kvm_present(self): """ returns true if KVM is present and available """ if not self._enable_kvm: @@ -674,6 +709,36 @@ def boot_in_hypervisor(self, multiboot=True, debug = False, kernel_args = "", im if "vfio" in self._config: pci_arg = ["-device", "vfio-pci,host=" + self._config["vfio"]] + virtiocon_args = [] + if "virtiocon" in self._config: + if "path" not in self._config["virtiocon"]: + raise Exception("Missing redirection path for guest output") + virtiocon_args = self.init_virtiocon(self._config["virtiocon"]["path"]) + + virtiofs_args = [] + if "virtiofs" in self._config: + socket = "/tmp/virtiofsd.sock" + if "socket" in self._config["virtiofs"]: + socket = self._config["virtiofs"]["socket"] + + if "shared" not in self._config["virtiofs"]: + raise Exception("Shared directory not specified for VirtioFS!") + shared = self._config["virtiofs"]["shared"] + + virtiofs_args = self.init_virtiofs(socket, shared, self._config["mem"]) + + virtiopmem_args = [] + if "virtiopmem" in self._config: + if "image" not in self._config["virtiopmem"]: + raise Exception("Missing path to persistent image file") + image = self._config["virtiopmem"]["image"] + + if "size" not in self._config["virtiopmem"]: + raise Exception("Missing persistent memory size") + size = self._config["virtiopmem"]["size"] + + virtiopmem_args = self.init_pmem(image, size) + # custom qemu binary/location qemu_binary = "qemu-system-x86_64" if "qemu" in self._config: @@ -703,7 +768,8 @@ def boot_in_hypervisor(self, multiboot=True, debug = False, kernel_args = "", im command += kernel_args command += disk_args + debug_args + net_args + mem_arg + mod_args - command += vga_arg + trace_arg + pci_arg + command += vga_arg + trace_arg + pci_arg + virtiocon_args + virtiofs_args + command += virtiopmem_args #command_str = " ".join(command) #command_str.encode('ascii','ignore') From 9b45ce05e07e8324ddc8b361f24a76a4c5affa00 Mon Sep 17 00:00:00 2001 From: torgeiru Date: Sat, 27 Sep 2025 14:11:55 +0200 Subject: [PATCH 02/10] Added a maxmem option for mem_arg --- vmrunner/vmrunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index f4667b4d1..d97fc054f 100755 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -695,7 +695,7 @@ def boot_in_hypervisor(self, multiboot=True, debug = False, kernel_args = "", im mem_arg = [] if "mem" in self._config: - mem_arg = ["-m", str(self._config["mem"])] + mem_arg = ["-m", f"size={self._config["mem"]},maxmem={64}G"] vga_arg = ["-nographic" ] if "vga" in self._config: From 39a0ec2ee107368f05382f1519ad378a059a8c29 Mon Sep 17 00:00:00 2001 From: torgeiru Date: Sat, 27 Sep 2025 17:50:53 +0200 Subject: [PATCH 03/10] Finished vmrunner Virtio part hopefully Removed debug stuff --- vmrunner/vmrunner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index d97fc054f..43e83bb05 100755 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -528,9 +528,9 @@ def init_virtiocon(self, path): def init_virtiofs(self, socket, shared, mem): """ initializes virtiofs by launching virtiofsd and creating a virtiofs device """ - virtiofsd_args = ["virtiofsd", "--log-level", "debug", "--socket", socket, "--shared-dir", shared, "--sandbox", "none"] + virtiofsd_args = ["virtiofsd", "--socket", socket, "--shared-dir", shared, "--sandbox", "none"] self._virtiofsd_proc = subprocess.Popen(virtiofsd_args) # pylint: disable=consider-using-with - # , stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + time.sleep(0.1) if self._virtiofsd_proc.poll(): raise Exception("VirtioFSD failed to start") From fab5ab379a1c9e28aae5e025c73e81794e6a9742 Mon Sep 17 00:00:00 2001 From: torgeiru Date: Sun, 28 Sep 2025 10:53:55 +0200 Subject: [PATCH 04/10] Making so that VirtioFSD doesn't say anything Future enhancement is having a schema option for printing VirtioFSD debug messages. --- vmrunner/vmrunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 43e83bb05..81af4c6de 100755 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -529,7 +529,7 @@ def init_virtiocon(self, path): def init_virtiofs(self, socket, shared, mem): """ initializes virtiofs by launching virtiofsd and creating a virtiofs device """ virtiofsd_args = ["virtiofsd", "--socket", socket, "--shared-dir", shared, "--sandbox", "none"] - self._virtiofsd_proc = subprocess.Popen(virtiofsd_args) # pylint: disable=consider-using-with + self._virtiofsd_proc = subprocess.Popen(virtiofsd_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # pylint: disable=consider-using-with time.sleep(0.1) if self._virtiofsd_proc.poll(): From f3d06e1ba5366577f6e1c15608c281541b61d1d9 Mon Sep 17 00:00:00 2001 From: torgeiru Date: Sun, 28 Sep 2025 12:45:05 +0200 Subject: [PATCH 05/10] Passhtrough pinned VirtioFSD --- default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/default.nix b/default.nix index 58109c87b..a4f21eb38 100644 --- a/default.nix +++ b/default.nix @@ -31,6 +31,7 @@ pkgs.python3.pkgs.buildPythonPackage rec { passthru = { inherit create_bridge; + virtiofsd = virtiofsd_pinned.virtiofsd; }; meta = { From acdfb896be8a0a58be96d1f5f038fcb502ad5007 Mon Sep 17 00:00:00 2001 From: torgeiru Date: Sun, 28 Sep 2025 17:05:26 +0200 Subject: [PATCH 06/10] Removed mistake --- default.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/default.nix b/default.nix index a4f21eb38..1adec97e9 100644 --- a/default.nix +++ b/default.nix @@ -24,7 +24,6 @@ pkgs.python3.pkgs.buildPythonPackage rec { future jsonschema psutil - virtiofsd_pinned.virtiofsd ]; create_bridge = ./vmrunner/bin/create_bridge.sh; From 2bc9aea6d68c7ba5a20533271fa872dafcf403ac Mon Sep 17 00:00:00 2001 From: torgeiru Date: Fri, 3 Oct 2025 21:32:33 +0200 Subject: [PATCH 07/10] Removing duplicated pinned nixpkgs --- default.nix | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/default.nix b/default.nix index 1adec97e9..2e3c73109 100644 --- a/default.nix +++ b/default.nix @@ -1,12 +1,4 @@ -{ - pkgs ? import {}, - - virtiofsd_pinned ? import (fetchTarball { - name = "nixpkgs-pinned-for-virtiofsd"; - url = "https://github.com/NixOS/nixpkgs/archive/13e8d35b7d6028b7198f8186bc0347c6abaa2701.tar.gz"; - sha256 = "0nqbvgmm7pbpyd8ihg2bi62pxihj8r673bc9ll4qhi6xwlfqac5q"; - }) {}, - }: +{ pkgs ? import {} }: pkgs.python3.pkgs.buildPythonPackage rec { pname = "vmrunner"; version = "0.16.0"; @@ -30,7 +22,6 @@ pkgs.python3.pkgs.buildPythonPackage rec { passthru = { inherit create_bridge; - virtiofsd = virtiofsd_pinned.virtiofsd; }; meta = { From 79eaede695eeef1555bc2a1f56f6b4b6954819f0 Mon Sep 17 00:00:00 2001 From: torgeiru Date: Sat, 4 Oct 2025 17:07:01 +0200 Subject: [PATCH 08/10] Fixed random tmp file for VirtioFS socket and removed racy sleep. Also, made it possible with multiple pmem devices per VM --- vmrunner/vm.schema.json | 37 ++++++++++++++++++------------ vmrunner/vmrunner.py | 50 ++++++++++++++++++++++------------------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/vmrunner/vm.schema.json b/vmrunner/vm.schema.json index e7631e446..ddee20511 100644 --- a/vmrunner/vm.schema.json +++ b/vmrunner/vm.schema.json @@ -88,34 +88,43 @@ "description" : "The path to redirect the guest output", "type" : "string" } - } + }, + + "required" : ["path"] }, "virtiofs" : { "description" : "VirtioFS device", "type" : "object", "properties" : { - "socket" : { "type" : "string" }, "shared" : { "description" : "Directory to be shared with guest", "type" : "string" } - } + }, + + "required" : ["shared"] }, "virtiopmem": { - "description" : "VirtioPMEM device", - "type" : "object", - "properties" : { - "image" : { - "description" : "Path to persistent image file", - "type" : "string" - }, - "size" : { - "description" : "Persistent memory size in megabytes", - "type" : "number" + "description" : "VirtioPMEM devices", + "type" : "array", + "items" : { + "description" : "Device configuration", + "type" : "object", + "properties" : { + "image" : { + "description" : "Path to persistent image file", + "type" : "string" + }, + "size" : { + "description" : "Persistent memory size in megabytes", + "type" : "number" + } } - } + }, + + "required" : ["image", "size"] }, "mem" : { diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 81af4c6de..c48af7897 100755 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -20,7 +20,7 @@ import re import traceback import signal -import time +import tempfile from enum import Enum import platform import psutil @@ -185,9 +185,9 @@ def __init__(self, config): self._config = config self._allow_sudo = False # must be explicitly turned on at boot. self._enable_kvm = False # must be explicitly turned on at boot. - self._sudo = False # Set to true if sudo is available - self._proc = None # A running subprocess - self._virtiofsd_proc = None + self._sudo = False # Set to true if sudo is available + self._proc = None # A running subprocess + self._tmp_dirs = [] # A list of tmp dirs created using tempfile module. Used for socket creation for automatic cleanup and garbage collection # pylint: disable-next=unused-argument def boot_in_hypervisor(self, multiboot=False, debug=False, kernel_args="", image_name="", allow_sudo = False, enable_kvm = False): @@ -420,6 +420,7 @@ class qemu(hypervisor): def __init__(self, config): super().__init__(config) self._proc = None + self._virtiofsd_proc = None self._stopped = False self._sudo = False self._image_name = self._config if "image" in self._config else self.name() + " vm" @@ -521,8 +522,8 @@ def net_arg(self, backend, device, if_name = "net0", mac = None, bridge = None, def init_virtiocon(self, path): """ creates a console device and redirects to the path given """ qemu_args = ["-device", "virtio-serial-pci,disable-legacy=on,id=virtio-serial0"] - qemu_args += ["-device", "virtserialport,chardev=char0"] - qemu_args += ["-chardev", f"file,id=char0,path={path}"] + qemu_args += ["-device", "virtserialport,chardev=virtiocon0"] + qemu_args += ["-chardev", f"file,id=virtiocon0,path={path}"] return qemu_args @@ -531,23 +532,25 @@ def init_virtiofs(self, socket, shared, mem): virtiofsd_args = ["virtiofsd", "--socket", socket, "--shared-dir", shared, "--sandbox", "none"] self._virtiofsd_proc = subprocess.Popen(virtiofsd_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # pylint: disable=consider-using-with - time.sleep(0.1) if self._virtiofsd_proc.poll(): raise Exception("VirtioFSD failed to start") info("Successfully started VirtioFSD!") + while not os.path.exists(socket): + ... + qemu_args = ["-machine", "memory-backend=mem0"] qemu_args += ["-chardev", f"socket,id=virtiofsd0,path={socket}"] - qemu_args += ["-device", "vhost-user-fs-pci,chardev=virtiofsd0,tag=virtiofs0"] + qemu_args += ["-device", "vhost-user-fs-pci,chardev=virtiofsd0,tag=vfs"] qemu_args += ["-object", f"memory-backend-memfd,id=mem0,size={mem}M,share=on"] return qemu_args - def init_pmem(self, path, size): + def init_pmem(self, path, size, pmem_id): """ creates a pmem device with image path as memory mapped backend """ - qemu_args = ["-object", f"memory-backend-file,id=pmemdev0,mem-path={path},size={size}M,share=on"] - qemu_args += ["-device", "virtio-pmem-pci,memdev=pmemdev0"] + qemu_args = ["-object", f"memory-backend-file,id=pmemdev{pmem_id},mem-path={path},size={size}M,share=on"] + qemu_args += ["-device", f"virtio-pmem-pci,memdev=pmemdev{pmem_id}"] return qemu_args @@ -695,7 +698,7 @@ def boot_in_hypervisor(self, multiboot=True, debug = False, kernel_args = "", im mem_arg = [] if "mem" in self._config: - mem_arg = ["-m", f"size={self._config["mem"]},maxmem={64}G"] + mem_arg = ["-m", f"size={self._config["mem"]},maxmem=1000G"] vga_arg = ["-nographic" ] if "vga" in self._config: @@ -717,27 +720,28 @@ def boot_in_hypervisor(self, multiboot=True, debug = False, kernel_args = "", im virtiofs_args = [] if "virtiofs" in self._config: - socket = "/tmp/virtiofsd.sock" - if "socket" in self._config["virtiofs"]: - socket = self._config["virtiofs"]["socket"] + tmp_virtiofs_dir = tempfile.TemporaryDirectory(prefix="virtiofs-") # pylint: disable=consider-using-with + self._tmp_dirs.append(tmp_virtiofs_dir) + socket_path = os.path.join(tmp_virtiofs_dir.name, "virtiofsd.sock") if "shared" not in self._config["virtiofs"]: raise Exception("Shared directory not specified for VirtioFS!") shared = self._config["virtiofs"]["shared"] - virtiofs_args = self.init_virtiofs(socket, shared, self._config["mem"]) + virtiofs_args = self.init_virtiofs(socket_path, shared, self._config["mem"]) virtiopmem_args = [] if "virtiopmem" in self._config: - if "image" not in self._config["virtiopmem"]: - raise Exception("Missing path to persistent image file") - image = self._config["virtiopmem"]["image"] + for pmem_id, virtiopmem in enumerate(self._config["virtiopmem"]): + if "image" not in virtiopmem: + raise Exception("Missing path to persistent image file") + image = virtiopmem["image"] - if "size" not in self._config["virtiopmem"]: - raise Exception("Missing persistent memory size") - size = self._config["virtiopmem"]["size"] + if "size" not in virtiopmem: + raise Exception("Missing persistent memory size") + size = virtiopmem["size"] - virtiopmem_args = self.init_pmem(image, size) + virtiopmem_args += self.init_pmem(image, size, pmem_id) # custom qemu binary/location qemu_binary = "qemu-system-x86_64" From 5657b7bc94633b5ca4abc0e99cab1e3391950041 Mon Sep 17 00:00:00 2001 From: torgeiru Date: Sat, 4 Oct 2025 23:59:40 +0200 Subject: [PATCH 09/10] Removed redundant ifs. Schema requirements raises missing requirements anyway --- vmrunner/vmrunner.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index c48af7897..04ae1169c 100755 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -724,8 +724,6 @@ def boot_in_hypervisor(self, multiboot=True, debug = False, kernel_args = "", im self._tmp_dirs.append(tmp_virtiofs_dir) socket_path = os.path.join(tmp_virtiofs_dir.name, "virtiofsd.sock") - if "shared" not in self._config["virtiofs"]: - raise Exception("Shared directory not specified for VirtioFS!") shared = self._config["virtiofs"]["shared"] virtiofs_args = self.init_virtiofs(socket_path, shared, self._config["mem"]) @@ -733,14 +731,8 @@ def boot_in_hypervisor(self, multiboot=True, debug = False, kernel_args = "", im virtiopmem_args = [] if "virtiopmem" in self._config: for pmem_id, virtiopmem in enumerate(self._config["virtiopmem"]): - if "image" not in virtiopmem: - raise Exception("Missing path to persistent image file") image = virtiopmem["image"] - - if "size" not in virtiopmem: - raise Exception("Missing persistent memory size") size = virtiopmem["size"] - virtiopmem_args += self.init_pmem(image, size, pmem_id) # custom qemu binary/location From afffb7dcb022c7b393895638c2363783ab7c3bed Mon Sep 17 00:00:00 2001 From: torgeiru Date: Sun, 5 Oct 2025 00:01:23 +0200 Subject: [PATCH 10/10] Removed one more redundant exception raise --- vmrunner/vmrunner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 04ae1169c..5a167a74b 100755 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -714,8 +714,6 @@ def boot_in_hypervisor(self, multiboot=True, debug = False, kernel_args = "", im virtiocon_args = [] if "virtiocon" in self._config: - if "path" not in self._config["virtiocon"]: - raise Exception("Missing redirection path for guest output") virtiocon_args = self.init_virtiocon(self._config["virtiocon"]["path"]) virtiofs_args = []