diff --git a/README.md b/README.md index 7dfb784..becd7b8 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,42 @@ As we already use ginkgo as testing framework - and that seems to scale as well - QEMU (no KVM) - Docker - Virtualbox +- Cloud Hypervisor They share the same common apis, so you can control machine created with the engines in the same way from a testing perspective. Design notes: QEMU has no KVM support to allow running QEMU machines inside docker containers. peg doesn't try to be smart by downloading all the required dependencies, instead it uses the smallest possible user-set from such, and tries to abstract from that to guarantee compatibilities between versions. Software like QEMU, Docker, and Virtualbox needs to be installed in the machine. +Cloud Hypervisor notes: it boots from a disk image via firmware (`--firmware` + +`--disk`), not from CD ISOs — the `iso` field is ignored for this engine, so +provide a bootable disk image via `image`/`drives` and a firmware via the +`firmware` field or `--firmware`. Rootless networking uses `pasta` (install it +on the host/CI) to forward the local SSH port to the guest, mirroring the QEMU +user-networking model. Set `disable_default_networking` to provide your own +`args` networking instead. + +### Installing pasta (Cloud Hypervisor networking) + +The Cloud Hypervisor engine needs the `pasta` binary on the host (or in `$PATH`) +for its default rootless networking. `pasta` ships as part of the `passt` +package — a single tiny binary with no runtime dependencies: + +| Platform | Command | +|----------|---------| +| Debian/Ubuntu | `sudo apt-get install -y passt` | +| Fedora/RHEL | `sudo dnf install -y passt` | +| Arch | `sudo pacman -S passt` | +| Alpine | `sudo apk add passt` | +| openSUSE | `sudo zypper install passt` | +| From source | `git clone https://passt.top/passt && cd passt && make && sudo make install` | + +The `passt` package installs both `passt` and `pasta`. Verify with +`pasta --version`. On GitHub Actions, `ubuntu-24.04` runners have `passt` in +apt; on older runners build it from source (it compiles in seconds) or fetch a +release from . If you set `disable_default_networking`, +`pasta` is not required. + If you are running tests on Github, keep in mind that the Virtualbox engine is specifically tailored for it - you should just be good to go as is with no additional configuration. ## Usage diff --git a/main.go b/main.go index c6091b5..10390d1 100644 --- a/main.go +++ b/main.go @@ -166,6 +166,16 @@ $ peg --iso path_to_iso_file Usage: "forces VBox engine", EnvVar: "PEG_VBOX", }, + cli.BoolFlag{ + Name: "cloud-hypervisor", + Usage: "forces cloud-hypervisor engine", + EnvVar: "PEG_CLOUDHYPERVISOR", + }, + cli.StringFlag{ + Name: "firmware", + Usage: "firmware path for cloud-hypervisor (rust-hypervisor-firmware or CLOUDHV.fd)", + EnvVar: "PEG_FIRMWARE", + }, }, UsageText: ``, Copyright: "Spectro Cloud", @@ -190,6 +200,7 @@ $ peg --iso path_to_iso_file types.WithImage(c.String("image")), types.WithISO(c.String("iso")), types.WithISOChecksum(c.String("iso-checksum")), + types.WithFirmware(c.String("firmware")), } if c.Bool("vbox") { @@ -200,6 +211,10 @@ $ peg --iso path_to_iso_file machineOpts = append(machineOpts, types.QEMUEngine) } + if c.Bool("cloud-hypervisor") { + machineOpts = append(machineOpts, types.CloudHypervisorEngine) + } + pegOpts := []peg.Option{ peg.WithLabelFilter(c.String("label")), peg.WithMachineOptions(machineOpts...), diff --git a/pkg/machine/cloudhypervisor.go b/pkg/machine/cloudhypervisor.go new file mode 100644 index 0000000..288a31c --- /dev/null +++ b/pkg/machine/cloudhypervisor.go @@ -0,0 +1,248 @@ +package machine + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + + process "github.com/mudler/go-processmanager" + "github.com/spectrocloud/peg/internal/utils" + controller "github.com/spectrocloud/peg/pkg/controller" + "github.com/spectrocloud/peg/pkg/machine/types" +) + +type CloudHypervisor struct { + machineConfig types.MachineConfig + process *process.Process +} + +// commonCHBinaryPaths are searched (in order) when no Process override is set. +var commonCHBinaryPaths = []string{ + "/usr/bin/cloud-hypervisor", + "/usr/local/bin/cloud-hypervisor", + "/home/linuxbrew/.linuxbrew/bin/cloud-hypervisor", +} + +// commonPastaPaths are searched (in order) when resolving the pasta binary. +var commonPastaPaths = []string{ + "/usr/bin/pasta", + "/usr/local/bin/pasta", +} + +// commonFirmwarePaths are searched (in order) when no Firmware is configured. +var commonFirmwarePaths = []string{ + "/usr/share/cloud-hypervisor/hypervisor-fw", + "/usr/lib/cloud-hypervisor/hypervisor-fw", + "/usr/share/cloud-hypervisor/CLOUDHV.fd", + "/usr/share/edk2/x64/CLOUDHV.fd", +} + +func firstExisting(paths []string) (string, error) { + for _, p := range paths { + if _, err := os.Stat(p); err == nil { + return p, nil + } + } + return "", fmt.Errorf("none of the candidate paths exist: %v", paths) +} + +// buildCloudHypervisorArgs builds the cloud-hypervisor CLI args (excluding the +// binary name and any pasta wrapper). Pure function for testability. +func buildCloudHypervisorArgs(mc types.MachineConfig, firmware string, drives []string) []string { + args := []string{ + "--api-socket", filepath.Join(mc.StateDir, "ch-api.sock"), + "--cpus", fmt.Sprintf("boot=%s", mc.CPU), + "--memory", fmt.Sprintf("size=%sM", mc.Memory), + "--firmware", firmware, + "--serial", "tty", + "--console", "off", + } + + disks := []string{} + for _, d := range drives { + disks = append(disks, fmt.Sprintf("path=%s", d)) + } + if mc.DataSource != "" { + disks = append(disks, fmt.Sprintf("path=%s,readonly=on", mc.DataSource)) + } + if len(disks) > 0 { + args = append(args, "--disk") + args = append(args, disks...) + } + + if !mc.DisableDefaultNetworking { + args = append(args, "--net", "tap=,mac=") + } + + args = append(args, mc.Args...) + return args +} + +// buildLaunchCommand wraps the cloud-hypervisor invocation with pasta when +// default networking is enabled. Pasta builds a rootless netns + tap and +// forwards host -> guest 22, preserving 127.0.0.1: SSH. +// pastaBinary must be an absolute path (resolved by resolvePasta before calling). +// When DisableDefaultNetworking is true, pastaBinary is ignored. +func buildLaunchCommand(mc types.MachineConfig, pastaBinary string, chBinary string, chArgs []string) (string, []string) { + if mc.DisableDefaultNetworking { + return chBinary, chArgs + } + port := "" + if mc.SSH != nil { + port = mc.SSH.Port + } + pastaArgs := []string{"--config-net", "-t", fmt.Sprintf("%s:22", port), "--", chBinary} + return pastaBinary, append(pastaArgs, chArgs...) +} + +func (q *CloudHypervisor) resolveBinary() (string, error) { + if q.machineConfig.Process != "" { + return q.machineConfig.Process, nil + } + if p, err := firstExisting(commonCHBinaryPaths); err == nil { + return p, nil + } + p, err := exec.LookPath("cloud-hypervisor") + if err != nil { + return "", fmt.Errorf("cloud-hypervisor not found in common paths or PATH: %w", err) + } + return p, nil +} + +func (q *CloudHypervisor) resolveFirmware() (string, error) { + if q.machineConfig.Firmware != "" { + return q.machineConfig.Firmware, nil + } + fw, err := firstExisting(commonFirmwarePaths) + if err != nil { + return "", fmt.Errorf("no firmware configured and none found in common paths: %w", err) + } + return fw, nil +} + +func resolvePasta() (string, error) { + if p, err := firstExisting(commonPastaPaths); err == nil { + return p, nil + } + p, err := exec.LookPath("pasta") + if err != nil { + return "", fmt.Errorf("pasta not found in common paths or PATH: %w", err) + } + return p, nil +} + +func (q *CloudHypervisor) driveSizes() []string { + sizes := []string{} + for _, s := range q.machineConfig.DriveSizes { + sizes = append(sizes, fmt.Sprintf("%sM", s)) + } + if len(sizes) == 0 { + sizes = append(sizes, fmt.Sprintf("%sM", types.DefaultDriveSize)) + } + return sizes +} + +func (q *CloudHypervisor) CreateDisk(diskname, size string) error { + if err := os.MkdirAll(q.machineConfig.StateDir, os.ModePerm); err != nil { + return err + } + out, err := utils.SH(fmt.Sprintf("qemu-img create -f qcow2 %s %s", + filepath.Join(q.machineConfig.StateDir, diskname), size)) + if err != nil { + return fmt.Errorf("%s : %w", out, err) + } + return nil +} + +func (q *CloudHypervisor) Create(ctx context.Context) (context.Context, error) { + log.Info("Create cloud-hypervisor machine") + + if q.machineConfig.ISO != "" { + log.Warn("cloud-hypervisor cannot boot ISO images; 'iso' is ignored. Provide a bootable disk image via 'image'/'drives'.") + } + + userDrives := q.machineConfig.Drives + if q.machineConfig.AutoDriveSetup && len(userDrives) == 0 { + for i, s := range q.driveSizes() { + filename := fmt.Sprintf("%s-%d.img", q.machineConfig.ID, i) + if err := q.CreateDisk(filename, s); err != nil { + return ctx, fmt.Errorf("creating disk with size %s: %w", s, err) + } + userDrives = append(userDrives, filepath.Join(q.machineConfig.StateDir, filename)) + } + } + + binary, err := q.resolveBinary() + if err != nil { + return ctx, fmt.Errorf("failed to find cloud-hypervisor binary: %w", err) + } + firmware, err := q.resolveFirmware() + if err != nil { + return ctx, fmt.Errorf("failed to resolve firmware: %w", err) + } + + chArgs := buildCloudHypervisorArgs(q.machineConfig, firmware, userDrives) + + pastaBinary := "" + if !q.machineConfig.DisableDefaultNetworking { + pastaBinary, err = resolvePasta() + if err != nil { + return ctx, fmt.Errorf("failed to find pasta: %w", err) + } + } + name, args := buildLaunchCommand(q.machineConfig, pastaBinary, binary, chArgs) + + log.Infof("Starting VM with %s [ Memory: %s, CPU: %s ]", binary, q.machineConfig.Memory, q.machineConfig.CPU) + + p := process.New( + process.WithName(name), + process.WithArgs(args...), + process.WithStateDir(q.machineConfig.StateDir), + ) + q.process = p + + newCtx := monitor(ctx, p, q.machineConfig.OnFailure) + return newCtx, p.Run() +} + +func (q *CloudHypervisor) Config() types.MachineConfig { + return q.machineConfig +} + +func (q *CloudHypervisor) Stop() error { + return process.New(process.WithStateDir(q.machineConfig.StateDir)).Stop() +} + +func (q *CloudHypervisor) Clean() error { + if q.machineConfig.StateDir != "" { + return os.RemoveAll(q.machineConfig.StateDir) + } + return nil +} + +func (q *CloudHypervisor) Alive() bool { + return process.New(process.WithStateDir(q.machineConfig.StateDir)).IsAlive() +} + +func (q *CloudHypervisor) Command(cmd string) (string, error) { + return controller.SSHCommand(q, cmd) +} + +func (q *CloudHypervisor) Screenshot() (string, error) { + return "", errors.New("Screenshot is not implemented in cloud-hypervisor machine") +} + +func (q *CloudHypervisor) DetachCD() error { + return nil // Not applicable: cloud-hypervisor has no CD-ROM concept. +} + +func (q *CloudHypervisor) ReceiveFile(src, dst string) error { + return controller.ReceiveFile(q, src, dst) +} + +func (q *CloudHypervisor) SendFile(src, dst, permissions string) error { + return controller.SendFile(q, src, dst, permissions) +} diff --git a/pkg/machine/cloudhypervisor_test.go b/pkg/machine/cloudhypervisor_test.go new file mode 100644 index 0000000..938b20d --- /dev/null +++ b/pkg/machine/cloudhypervisor_test.go @@ -0,0 +1,145 @@ +package machine + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spectrocloud/peg/pkg/machine/types" +) + +func cfg() types.MachineConfig { + return types.MachineConfig{ + StateDir: "/state", + CPU: "2", + Memory: "2048", + SSH: &types.SSH{Port: "2222"}, + } +} + +func joined(args []string) string { return strings.Join(args, " ") } + +func TestBuildArgsFirmwareDrives(t *testing.T) { + args := buildCloudHypervisorArgs(cfg(), "/fw/hypervisor-fw", []string{"/state/d0.img"}) + got := joined(args) + for _, want := range []string{ + "--api-socket /state/ch-api.sock", + "--cpus boot=2", + "--memory size=2048M", + "--firmware /fw/hypervisor-fw", + "--disk path=/state/d0.img", + "--net tap=,mac=", + } { + if !strings.Contains(got, want) { + t.Fatalf("missing %q in: %s", want, got) + } + } +} + +func TestBuildArgsDatasourceReadonly(t *testing.T) { + c := cfg() + c.DataSource = "/state/seed.img" + args := buildCloudHypervisorArgs(c, "/fw", []string{"/state/d0.img"}) + if !strings.Contains(joined(args), "path=/state/seed.img,readonly=on") { + t.Fatalf("datasource not attached readonly: %s", joined(args)) + } +} + +func TestBuildArgsNetworkingDisabled(t *testing.T) { + c := cfg() + c.DisableDefaultNetworking = true + args := buildCloudHypervisorArgs(c, "/fw", []string{"/state/d0.img"}) + if strings.Contains(joined(args), "--net") { + t.Fatalf("--net present despite disabled networking: %s", joined(args)) + } +} + +func TestBuildArgsAppendsUserArgs(t *testing.T) { + c := cfg() + c.Args = []string{"--rng", "src=/dev/urandom"} + args := buildCloudHypervisorArgs(c, "/fw", nil) + if !strings.HasSuffix(joined(args), "--rng src=/dev/urandom") { + t.Fatalf("user args not appended last: %s", joined(args)) + } +} + +func TestBuildLaunchCommandPasta(t *testing.T) { + name, args := buildLaunchCommand(cfg(), "/usr/bin/pasta", "/bin/cloud-hypervisor", []string{"--cpus", "boot=2"}) + if name != "/usr/bin/pasta" { + t.Fatalf("expected /usr/bin/pasta, got %s", name) + } + got := joined(args) + for _, want := range []string{"--config-net", "-t 2222:22", "-- /bin/cloud-hypervisor", "--cpus boot=2"} { + if !strings.Contains(got, want) { + t.Fatalf("missing %q in: %s", want, got) + } + } +} + +func TestBuildLaunchCommandNoNet(t *testing.T) { + c := cfg() + c.DisableDefaultNetworking = true + name, args := buildLaunchCommand(c, "", "/bin/cloud-hypervisor", []string{"--cpus", "boot=2"}) + if name != "/bin/cloud-hypervisor" { + t.Fatalf("expected direct binary, got %s", name) + } + if joined(args) != "--cpus boot=2" { + t.Fatalf("args altered: %s", joined(args)) + } +} + +func TestBuildLaunchCommandPastaUsesResolvedPath(t *testing.T) { + resolvedPasta := "/usr/local/bin/pasta" + name, _ := buildLaunchCommand(cfg(), resolvedPasta, "/bin/cloud-hypervisor", []string{"--cpus", "boot=2"}) + if name != resolvedPasta { + t.Fatalf("expected process name to be resolved absolute path %q, got %q", resolvedPasta, name) + } + if name == "pasta" { + t.Fatal("process name must not be bare 'pasta': os.StartProcess does no PATH lookup") + } +} + +func TestFirstExisting(t *testing.T) { + dir := t.TempDir() + real := filepath.Join(dir, "fw") + if err := os.WriteFile(real, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + got, err := firstExisting([]string{filepath.Join(dir, "missing"), real}) + if err != nil || got != real { + t.Fatalf("got %q err %v", got, err) + } + if _, err := firstExisting([]string{filepath.Join(dir, "none")}); err == nil { + t.Fatal("expected error when no path exists") + } +} + +func TestNewSelectsCloudHypervisor(t *testing.T) { + m, err := New(types.WithStateDir(t.TempDir()), types.CloudHypervisorEngine, + types.WithSSHPort("2222")) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, ok := m.(*CloudHypervisor); !ok { + t.Fatalf("expected *CloudHypervisor, got %T", m) + } +} + +func TestCloudHypervisorImplementsMachine(t *testing.T) { + var _ types.Machine = (*CloudHypervisor)(nil) +} + +func TestScreenshotNotImplemented(t *testing.T) { + c := &CloudHypervisor{machineConfig: cfg()} + if _, err := c.Screenshot(); err == nil { + t.Fatal("expected screenshot error") + } +} + +func TestDetachCDNoop(t *testing.T) { + c := &CloudHypervisor{machineConfig: cfg()} + if err := c.DetachCD(); err != nil { + t.Fatalf("DetachCD should be nil, got %v", err) + } +} diff --git a/pkg/machine/machine.go b/pkg/machine/machine.go index 0a11747..800cbc1 100644 --- a/pkg/machine/machine.go +++ b/pkg/machine/machine.go @@ -165,6 +165,8 @@ func New(opts ...types.MachineOption) (types.Machine, error) { return &Docker{machineConfig: *mc}, nil case types.VBox: return &VBox{machineConfig: *mc}, nil + case types.CloudHypervisor: + return &CloudHypervisor{machineConfig: *mc}, nil } return nil, fmt.Errorf("invalid engine: %s, obj: %+v", mc.Engine, mc) diff --git a/pkg/machine/types/config.go b/pkg/machine/types/config.go index 9616e0f..9b30549 100644 --- a/pkg/machine/types/config.go +++ b/pkg/machine/types/config.go @@ -36,7 +36,10 @@ type MachineConfig struct { // only for qemu Display string `yaml:"display,omitempty"` - CPUType string `yaml:"cpu,omitempty"` + CPUType string `yaml:"cpuType,omitempty"` + + // Firmware path for engines that boot via firmware (cloud-hypervisor). + Firmware string `yaml:"firmware,omitempty"` // Network configuration DisableDefaultNetworking bool `yaml:"disable_default_networking,omitempty"` @@ -50,9 +53,10 @@ type MachineConfig struct { type Engine string const ( - VBox Engine = "vbox" - QEMU Engine = "qemu" - Docker Engine = "docker" + VBox Engine = "vbox" + QEMU Engine = "qemu" + Docker Engine = "docker" + CloudHypervisor Engine = "cloud-hypervisor" ) type MachineOption func(*MachineConfig) error @@ -109,6 +113,15 @@ func WithDisplay(display string) MachineOption { } } +func WithFirmware(fw string) MachineOption { + return func(mc *MachineConfig) error { + if fw != "" { + mc.Firmware = fw + } + return nil + } +} + func WithDataSource(ds string) MachineOption { return func(mc *MachineConfig) error { if ds != "" { @@ -256,6 +269,12 @@ var QEMUEngine MachineOption = func(mc *MachineConfig) error { return nil } +// CloudHypervisorEngine sets the machine engine to cloud-hypervisor. +var CloudHypervisorEngine MachineOption = func(mc *MachineConfig) error { + mc.Engine = CloudHypervisor + return nil +} + // EnableAutoDriveSetup automatically setup a VM disk if nothing is specified. var EnableAutoDriveSetup MachineOption = func(mc *MachineConfig) error { mc.AutoDriveSetup = true diff --git a/pkg/machine/types/config_test.go b/pkg/machine/types/config_test.go new file mode 100644 index 0000000..0ccde1d --- /dev/null +++ b/pkg/machine/types/config_test.go @@ -0,0 +1,48 @@ +package types + +import ( + "testing" + + "gopkg.in/yaml.v3" +) + +func TestWithFirmware(t *testing.T) { + mc := DefaultMachineConfig() + if err := mc.Apply(WithFirmware("/fw/hypervisor-fw")); err != nil { + t.Fatalf("apply: %v", err) + } + if mc.Firmware != "/fw/hypervisor-fw" { + t.Fatalf("got %q", mc.Firmware) + } + // empty value must not overwrite + if err := mc.Apply(WithFirmware("")); err != nil { + t.Fatalf("apply empty: %v", err) + } + if mc.Firmware != "/fw/hypervisor-fw" { + t.Fatalf("empty overwrote firmware: %q", mc.Firmware) + } +} + +func TestCloudHypervisorEngineOption(t *testing.T) { + mc := DefaultMachineConfig() + if err := mc.Apply(CloudHypervisorEngine); err != nil { + t.Fatalf("apply: %v", err) + } + if mc.Engine != CloudHypervisor { + t.Fatalf("got engine %q", mc.Engine) + } +} + +func TestEngineYAMLParse(t *testing.T) { + mc := DefaultMachineConfig() + in := []byte("engine: cloud-hypervisor\nfirmware: /fw/CLOUDHV.fd\n") + if err := yaml.Unmarshal(in, mc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if mc.Engine != CloudHypervisor { + t.Fatalf("engine %q", mc.Engine) + } + if mc.Firmware != "/fw/CLOUDHV.fd" { + t.Fatalf("firmware %q", mc.Firmware) + } +}