Give Firecracker forks their own mem-file instead of deferring the copy

lib/forkvm/README.md

@@ -59,14 +59,22 @@ instead of reusing the source identity.
 - Network override fields are supplied at snapshot load to bind the fork to its
   own TAP device.
 - Vsock CID remains stable for snapshot-based flows.
+- Standby forks hardlink the source's snapshot mem-file instead of copying it:
+  fanout costs no memory I/O and every fork of a snapshot faults against one
+  inode, so the kernel page cache (and the UFFD pager cache, keyed by the
+  inherited snapshot cache key) is shared across siblings. Deleting the source
+  is still safe immediately — unlink only drops a name; forks keep the inode
+  alive via its link count.
+- Sharing an inode is safe because Firecracker mmaps the mem-file MAP_PRIVATE
+  (guest writes never reach the file) and the only file writer — the in-place
+  diff-snapshot merge on standby — first replaces any mem-file with `nlink > 1`
+  with a private copy (reflink-cloned where the filesystem supports FICLONE,
+  sparse-copied otherwise). A fork therefore becomes fully independent at its
+  first standby, and a source that standbys while forks still share its base
+  unshares the same way instead of mutating memory a fork reads.
 - When the Firecracker snapshot memory backend is configured as UFFD, UFFD is
   used as a one-shot acceleration for the first restore of a newly forked
-  standby snapshot. The fork initially reuses the source snapshot memory as the
-  pager backing file instead of cloning the large memory file during fanout.
-- That deferred memory clone is paid when the fork later enters standby. Before
-  Firecracker writes the fork's diff snapshot, Hypeman materializes the fork's
-  own `snapshot-latest/memory` file from the original backing memory. After that
-  point the fork has a normal on-disk snapshot base, independent from the source.
+  standby snapshot.
 - Subsequent direct restores of that same fork use Firecracker's normal
   file-backed memory backend. If that standby fork is itself forked again, the
   new child gets its own one-shot UFFD restore.

lib/forkvm/copy.go

@@ -176,7 +176,9 @@ func shouldSkipDirectory(relPath string) bool {
 }
 
 func shouldSkipRegularFile(relPath string) bool {
-	return strings.HasSuffix(relPath, ".lz4.tmp") || strings.HasSuffix(relPath, ".zst.tmp")
+	return strings.HasSuffix(relPath, ".lz4.tmp") ||
+		strings.HasSuffix(relPath, ".zst.tmp") ||
+		strings.HasSuffix(relPath, ".unshare.tmp")
 }
 
 func shouldSkipRuntimeSocket(relPath string) bool {

lib/forkvm/copy_test.go

@@ -22,6 +22,7 @@ func TestCopyGuestDirectory(t *testing.T) {
 	require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "config.json"), []byte(`{}`), 0644))
 	require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp"), []byte("partial"), 0644))
 	require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.zst.tmp"), []byte("partial"), 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory.unshare.tmp"), []byte("partial"), 0644))
 	require.NoError(t, os.Symlink("metadata.json", filepath.Join(src, "meta-link")))
 
 	require.NoError(t, CopyGuestDirectory(src, dst))
@@ -32,6 +33,7 @@ func TestCopyGuestDirectory(t *testing.T) {
 	assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "config.json"))
 	assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp"))
 	assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.zst.tmp"))
+	assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory.unshare.tmp"))
 	assert.NoFileExists(t, filepath.Join(dst, "logs", "app.log"))
 	assert.FileExists(t, filepath.Join(dst, "meta-link"))
 

lib/guestmemory/controller_test.go

@@ -47,11 +47,9 @@ func (s *stubHypervisor) Shutdown(ctx context.Context) error { return nil }
 func (s *stubHypervisor) GetVMInfo(ctx context.Context) (*hypervisor.VMInfo, error) {
 	return &hypervisor.VMInfo{State: hypervisor.StateRunning}, nil
 }
-func (s *stubHypervisor) Pause(ctx context.Context) error  { return nil }
-func (s *stubHypervisor) Resume(ctx context.Context) error { return nil }
-func (s *stubHypervisor) Snapshot(ctx context.Context, destPath string, _ hypervisor.SnapshotOptions) error {
-	return nil
-}
+func (s *stubHypervisor) Pause(ctx context.Context) error                     { return nil }
+func (s *stubHypervisor) Resume(ctx context.Context) error                    { return nil }
+func (s *stubHypervisor) Snapshot(ctx context.Context, destPath string) error { return nil }
 func (s *stubHypervisor) ResizeMemory(ctx context.Context, bytes int64) error { return nil }
 func (s *stubHypervisor) ResizeMemoryAndWait(ctx context.Context, bytes int64, timeout time.Duration) error {
 	return nil

lib/hypervisor/cloudhypervisor/cloudhypervisor.go

@@ -160,7 +160,7 @@ func (c *CloudHypervisor) Resume(ctx context.Context) error {
 }
 
 // Snapshot creates a VM snapshot.
-func (c *CloudHypervisor) Snapshot(ctx context.Context, destPath string, _ hypervisor.SnapshotOptions) error {
+func (c *CloudHypervisor) Snapshot(ctx context.Context, destPath string) error {
 	snapshotURL := "file://" + destPath
 	snapshotConfig := vmm.VmSnapshotConfig{DestinationUrl: &snapshotURL}
 	resp, err := c.client.PutVmSnapshotWithResponse(ctx, snapshotConfig)

lib/hypervisor/firecracker/config_test.go

@@ -101,37 +101,6 @@ func TestSnapshotLoadParamsSupportsUFFDBackend(t *testing.T) {
 	assert.Equal(t, "/tmp/pager.sock", load.MemBackend.BackendPath)
 }
 
-func TestMaterializeDeferredSnapshotMemory(t *testing.T) {
-	t.Parallel()
-
-	sourcePath := filepath.Join(t.TempDir(), "source-memory")
-	snapshotDir := filepath.Join(t.TempDir(), "snapshot-latest")
-	require.NoError(t, os.WriteFile(sourcePath, []byte("memory"), 0644))
-
-	require.NoError(t, materializeDeferredSnapshotMemory(snapshotDir, sourcePath))
-
-	got, err := os.ReadFile(filepath.Join(snapshotDir, "memory"))
-	require.NoError(t, err)
-	assert.Equal(t, []byte("memory"), got)
-}
-
-func TestMaterializeDeferredSnapshotMemoryUsesRetainedSnapshotAlternate(t *testing.T) {
-	t.Parallel()
-
-	root := t.TempDir()
-	sourcePath := filepath.Join(root, "snapshots", "snapshot-base", "memory")
-	alternatePath := filepath.Join(root, "snapshots", "snapshot-latest", "memory")
-	destPath := filepath.Join(t.TempDir(), "snapshot-latest")
-	require.NoError(t, os.MkdirAll(filepath.Dir(alternatePath), 0755))
-	require.NoError(t, os.WriteFile(alternatePath, []byte("memory"), 0644))
-
-	require.NoError(t, materializeDeferredSnapshotMemory(destPath, sourcePath))
-
-	got, err := os.ReadFile(filepath.Join(destPath, "memory"))
-	require.NoError(t, err)
-	assert.Equal(t, []byte("memory"), got)
-}
-
 func TestToBalloonConfig(t *testing.T) {
 	cfg := hypervisor.VMConfig{
 		GuestMemory: hypervisor.GuestMemoryConfig{

lib/hypervisor/firecracker/firecracker.go

@@ -14,7 +14,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kernel/hypeman/lib/forkvm"
 	"github.com/kernel/hypeman/lib/hypervisor"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
@@ -114,76 +113,17 @@ func (f *Firecracker) Resume(ctx context.Context) error {
 	return nil
 }
 
-func (f *Firecracker) Snapshot(ctx context.Context, destPath string, opts hypervisor.SnapshotOptions) error {
+func (f *Firecracker) Snapshot(ctx context.Context, destPath string) error {
 	if err := os.MkdirAll(destPath, 0755); err != nil {
 		return fmt.Errorf("create snapshot directory: %w", err)
 	}
-	if err := materializeDeferredSnapshotMemory(destPath, opts.DeferredMemoryBackingPath); err != nil {
-		return err
-	}
 	params := toSnapshotCreateParams(destPath)
 	if _, err := f.do(ctx, http.MethodPut, "/snapshot/create", params, http.StatusNoContent); err != nil {
 		return fmt.Errorf("create snapshot: %w", err)
 	}
 	return nil
 }
 
-func materializeDeferredSnapshotMemory(destPath, sourcePath string) error {
-	sourcePath = strings.TrimSpace(sourcePath)
-	if sourcePath == "" {
-		return nil
-	}
-	targetPath := filepath.Join(destPath, "memory")
-	if _, err := os.Stat(targetPath); err == nil {
-		return nil
-	} else if !os.IsNotExist(err) {
-		return fmt.Errorf("stat deferred snapshot memory target: %w", err)
-	}
-	resolvedSourcePath, err := resolveDeferredSnapshotMemorySourcePath(sourcePath)
-	if err != nil {
-		return err
-	}
-	if err := forkvm.CopyRegularFile(resolvedSourcePath, targetPath); err != nil {
-		return fmt.Errorf("materialize deferred snapshot memory: %w", err)
-	}
-	return nil
-}
-
-func resolveDeferredSnapshotMemorySourcePath(sourcePath string) (string, error) {
-	if _, err := os.Stat(sourcePath); err == nil {
-		return sourcePath, nil
-	} else if !os.IsNotExist(err) {
-		return "", fmt.Errorf("stat deferred snapshot memory source: %w", err)
-	}
-
-	alternatePath := alternateRetainedSnapshotMemoryPath(sourcePath)
-	if alternatePath == "" {
-		return sourcePath, nil
-	}
-	if _, err := os.Stat(alternatePath); err == nil {
-		return alternatePath, nil
-	} else if !os.IsNotExist(err) {
-		return "", fmt.Errorf("stat alternate deferred snapshot memory source: %w", err)
-	}
-	return sourcePath, nil
-}
-
-func alternateRetainedSnapshotMemoryPath(sourcePath string) string {
-	if filepath.Base(sourcePath) != "memory" {
-		return ""
-	}
-	snapshotDir := filepath.Dir(sourcePath)
-	snapshotsDir := filepath.Dir(snapshotDir)
-	switch filepath.Base(snapshotDir) {
-	case "snapshot-base":
-		return filepath.Join(snapshotsDir, "snapshot-latest", "memory")
-	case "snapshot-latest":
-		return filepath.Join(snapshotsDir, "snapshot-base", "memory")
-	default:
-		return ""
-	}
-}
-
 func (f *Firecracker) ResizeMemory(ctx context.Context, bytes int64) error {
 	return hypervisor.ErrNotSupported
 }

lib/hypervisor/hypervisor.go

@@ -138,10 +138,6 @@ type RestoreOptions struct {
 	SnapshotMemorySessionID   string
 }
 
-type SnapshotOptions struct {
-	DeferredMemoryBackingPath string
-}
-
 // ForkNetworkConfig contains network identity fields for fork preparation.
 type ForkNetworkConfig struct {
 	TAPDevice string
@@ -199,7 +195,7 @@ type Hypervisor interface {
 
 	// Snapshot creates a VM snapshot at the given path.
 	// Check Capabilities().SupportsSnapshot before calling.
-	Snapshot(ctx context.Context, destPath string, opts SnapshotOptions) error
+	Snapshot(ctx context.Context, destPath string) error
 
 	// ResizeMemory changes the VM's memory allocation.
 	// Check Capabilities().SupportsHotplugMemory before calling.

lib/hypervisor/qemu/qemu.go

@@ -140,7 +140,7 @@ func (q *QEMU) Resume(ctx context.Context) error {
 // Snapshot creates a VM snapshot using QEMU's migrate-to-file mechanism.
 // The VM state is saved to destPath/memory file.
 // The VM config is copied to destPath for restore (QEMU requires exact arg match).
-func (q *QEMU) Snapshot(ctx context.Context, destPath string, _ hypervisor.SnapshotOptions) error {
+func (q *QEMU) Snapshot(ctx context.Context, destPath string) error {
 	// QEMU uses migrate to file for snapshots
 	// The "file:" protocol is deprecated in QEMU 7.2+, use "exec:cat > path" instead
 	memoryFile := destPath + "/memory"

lib/hypervisor/tracing.go

@@ -217,14 +217,14 @@ func (h *tracingHypervisor) Resume(ctx context.Context) (err error) {
 	return h.next.Resume(ctx)
 }
 
-func (h *tracingHypervisor) Snapshot(ctx context.Context, destPath string, opts SnapshotOptions) (err error) {
+func (h *tracingHypervisor) Snapshot(ctx context.Context, destPath string) (err error) {
 	ctx, span := startTraceSpan(ctx, h.tracer, "hypervisor.snapshot",
 		h.spanAttrs(
 			attribute.String("operation", "snapshot"),
 		)...,
 	)
 	defer func() { finishTraceSpan(span, err) }()
-	return h.next.Snapshot(ctx, destPath, opts)
+	return h.next.Snapshot(ctx, destPath)
 }
 
 func (h *tracingHypervisor) ResizeMemory(ctx context.Context, bytes int64) (err error) {

lib/hypervisor/tracing_test.go

@@ -24,11 +24,9 @@ func (fakeHypervisor) Shutdown(context.Context) error { return nil }
 func (fakeHypervisor) GetVMInfo(context.Context) (*VMInfo, error) {
 	return &VMInfo{State: StateRunning}, nil
 }
-func (fakeHypervisor) Pause(context.Context) error  { return nil }
-func (fakeHypervisor) Resume(context.Context) error { return nil }
-func (fakeHypervisor) Snapshot(context.Context, string, SnapshotOptions) error {
-	return nil
-}
+func (fakeHypervisor) Pause(context.Context) error               { return nil }
+func (fakeHypervisor) Resume(context.Context) error              { return nil }
+func (fakeHypervisor) Snapshot(context.Context, string) error    { return nil }
 func (fakeHypervisor) ResizeMemory(context.Context, int64) error { return nil }
 func (fakeHypervisor) ResizeMemoryAndWait(context.Context, int64, time.Duration) error {
 	return nil
@@ -43,11 +41,9 @@ func (fakeHypervisorGetVMInfoError) Shutdown(context.Context) error { return nil
 func (fakeHypervisorGetVMInfoError) GetVMInfo(context.Context) (*VMInfo, error) {
 	return nil, errors.New("vm info failed")
 }
-func (fakeHypervisorGetVMInfoError) Pause(context.Context) error  { return nil }
-func (fakeHypervisorGetVMInfoError) Resume(context.Context) error { return nil }
-func (fakeHypervisorGetVMInfoError) Snapshot(context.Context, string, SnapshotOptions) error {
-	return nil
-}
+func (fakeHypervisorGetVMInfoError) Pause(context.Context) error            { return nil }
+func (fakeHypervisorGetVMInfoError) Resume(context.Context) error           { return nil }
+func (fakeHypervisorGetVMInfoError) Snapshot(context.Context, string) error { return nil }
 func (fakeHypervisorGetVMInfoError) ResizeMemory(context.Context, int64) error {
 	return nil
 }

lib/hypervisor/vz/client.go

@@ -301,7 +301,7 @@ func (c *Client) rawVMState(ctx context.Context) (string, error) {
 	return info.State, nil
 }
 
-func (c *Client) Snapshot(ctx context.Context, destPath string, _ hypervisor.SnapshotOptions) error {
+func (c *Client) Snapshot(ctx context.Context, destPath string) error {
 	req := snapshotRequest{DestinationPath: destPath}
 	body, err := json.Marshal(req)
 	if err != nil {

lib/instances/firecracker_memfile.go

@@ -0,0 +1,92 @@
+package instances
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/kernel/hypeman/lib/forkvm"
+	"github.com/kernel/hypeman/lib/logger"
+)
+
+// linkForkFirecrackerMemFile hardlinks the source's snapshot mem-file into the
+// fork's guest dir so all forks of a snapshot share one inode: fanout costs no
+// copy I/O, and the pager's backing reads hit the kernel page cache warmed by
+// sibling forks. Falls back to a reflink/sparse copy when linking fails.
+// Sharing an inode is safe because Firecracker mmaps the mem-file MAP_PRIVATE
+// and the only file writer, the standby diff snapshot, unshares first via
+// ensureExclusiveSnapshotMemoryOwnership.
+func (m *manager) linkForkFirecrackerMemFile(ctx context.Context, srcGuestDir, dstGuestDir string) error {
+	srcMem := firecrackerSnapshotMemoryPathInGuestDir(srcGuestDir)
+	dstMem := firecrackerSnapshotMemoryPathInGuestDir(dstGuestDir)
+	if err := os.MkdirAll(filepath.Dir(dstMem), 0755); err != nil {
+		return fmt.Errorf("create fork snapshot dir: %w", err)
+	}
+	err := os.Link(srcMem, dstMem)
+	if err == nil {
+		return nil
+	}
+	// A fallback copy loses the zero-copy fanout and the shared page cache
+	// across sibling forks; it should never happen on a correctly provisioned
+	// host, so make it visible.
+	logger.FromContext(ctx).WarnContext(ctx, "hardlink of fork snapshot memory failed; falling back to copy",
+		"source", srcMem, "target", dstMem, "error", err)
+	m.recordForkMemFileShareFallback(ctx, linkFallbackReason(err))
+	if err := forkvm.CopyRegularFile(srcMem, dstMem); err != nil {
+		return fmt.Errorf("copy fork snapshot memory: %w", err)
+	}
+	return nil
+}
+
+func linkFallbackReason(err error) string {
+	var errno syscall.Errno
+	if errors.As(err, &errno) {
+		return errno.Error()
+	}
+	return "unknown"
+}
+
+// ensureExclusiveSnapshotMemoryOwnership replaces the snapshot mem-file with a
+// private copy when other hardlinks to its inode exist (fanout forks).
+// Firecracker merges diff snapshots by writing dirty pages into this file in
+// place, which must never mutate memory another instance still reads.
+//
+// The stat -> copy -> rename sequence is not internally synchronized; callers
+// must hold the instance's write lock (the standby path does) so no fork can
+// take a new hardlink between the link-count check and the replacement.
+func ensureExclusiveSnapshotMemoryOwnership(ctx context.Context, snapshotDir string) error {
+	memPath := filepath.Join(snapshotDir, "memory")
+	tmpPath := memPath + ".unshare.tmp"
+	// Sweep any stale tmp from a crash between copy and rename; it is
+	// mem-file-sized and must not linger.
+	_ = os.Remove(tmpPath)
+
+	info, err := os.Stat(memPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("stat snapshot memory: %w", err)
+	}
+	stat, ok := info.Sys().(*syscall.Stat_t)
+	if !ok || stat.Nlink <= 1 {
+		return nil
+	}
+
+	start := time.Now()
+	if err := forkvm.CopyRegularFile(memPath, tmpPath); err != nil {
+		_ = os.Remove(tmpPath)
+		return fmt.Errorf("copy shared snapshot memory: %w", err)
+	}
+	if err := os.Rename(tmpPath, memPath); err != nil {
+		_ = os.Remove(tmpPath)
+		return fmt.Errorf("replace shared snapshot memory: %w", err)
+	}
+	logger.FromContext(ctx).InfoContext(ctx, "unshared snapshot memory before diff snapshot",
+		"path", memPath, "links", stat.Nlink, "duration", time.Since(start).String())
+	return nil
+}