diff --git a/agent/bootstrap_test.go b/agent/bootstrap_test.go index e075fd788..6faf92b5d 100644 --- a/agent/bootstrap_test.go +++ b/agent/bootstrap_test.go @@ -10,6 +10,7 @@ import ( "time" boshlogstarprovider "github.com/cloudfoundry/bosh-agent/v2/agent/logstarprovider" + "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -1132,6 +1133,7 @@ var _ = Describe("bootstrap", func() { monitRetryStrategy := boshretry.NewAttemptRetryStrategy(10, 1*time.Second, monitRetryable, logger) devicePathResolver := fakedevicepathresolver.NewFakeDevicePathResolver() + symlinkDeviceResolver := devicepathresolver.NewSymlinkDeviceResolver(fs, udev, logger) fakeUUIDGenerator := boshuuid.NewGenerator() routesSearcher := boshnet.NewRoutesSearcher(logger, runner, nil) @@ -1153,6 +1155,7 @@ var _ = Describe("bootstrap", func() { ubuntuCertManager, monitRetryStrategy, devicePathResolver, + symlinkDeviceResolver, state, linuxOptions, logger, diff --git a/infrastructure/devicepathresolver/fakes/fake_device_path_resolver.go b/infrastructure/devicepathresolver/fakes/fake_device_path_resolver.go index 5b0426e69..c5bad212e 100644 --- a/infrastructure/devicepathresolver/fakes/fake_device_path_resolver.go +++ b/infrastructure/devicepathresolver/fakes/fake_device_path_resolver.go @@ -5,7 +5,7 @@ import ( ) type FakeDevicePathResolver struct { - GetRealDevicePathDiskSettings boshsettings.DiskSettings + GetRealDevicePathDiskSettings []boshsettings.DiskSettings RealDevicePath string GetRealDevicePathStub func(boshsettings.DiskSettings) (string, bool, error) GetRealDevicePathTimedOut bool @@ -17,15 +17,15 @@ func NewFakeDevicePathResolver() *FakeDevicePathResolver { } func (r *FakeDevicePathResolver) GetRealDevicePath(diskSettings boshsettings.DiskSettings) (string, bool, error) { - r.GetRealDevicePathDiskSettings = diskSettings - - if r.GetRealDevicePathErr != nil { - return "", r.GetRealDevicePathTimedOut, r.GetRealDevicePathErr - } + r.GetRealDevicePathDiskSettings = append(r.GetRealDevicePathDiskSettings, diskSettings) if r.GetRealDevicePathStub != nil { return r.GetRealDevicePathStub(diskSettings) } + if r.GetRealDevicePathErr != nil { + return "", r.GetRealDevicePathTimedOut, r.GetRealDevicePathErr + } + return r.RealDevicePath, false, nil } diff --git a/infrastructure/devicepathresolver/fallback_device_path_resolver_test.go b/infrastructure/devicepathresolver/fallback_device_path_resolver_test.go index f26db30b8..1a588da6a 100644 --- a/infrastructure/devicepathresolver/fallback_device_path_resolver_test.go +++ b/infrastructure/devicepathresolver/fallback_device_path_resolver_test.go @@ -51,7 +51,7 @@ var _ = Describe("FallbackDevicePathResolver", func() { It("does not call the secondary resolver", func() { _, _, err := pathResolver.GetRealDevicePath(diskSettings) Expect(err).ToNot(HaveOccurred()) - Expect(secondaryResolver.GetRealDevicePathDiskSettings).To(Equal(boshsettings.DiskSettings{})) + Expect(secondaryResolver.GetRealDevicePathDiskSettings).To(BeEmpty()) }) }) @@ -68,7 +68,7 @@ var _ = Describe("FallbackDevicePathResolver", func() { Expect(timeout).To(BeFalse()) Expect(realPath).To(Equal("/dev/sdc")) - Expect(secondaryResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings)) + Expect(secondaryResolver.GetRealDevicePathDiskSettings).To(ContainElement(diskSettings)) }) Context("when secondary resolver also errors", func() { diff --git a/infrastructure/devicepathresolver/scsi_device_path_resolver_test.go b/infrastructure/devicepathresolver/scsi_device_path_resolver_test.go index b23a89610..3746e3788 100644 --- a/infrastructure/devicepathresolver/scsi_device_path_resolver_test.go +++ b/infrastructure/devicepathresolver/scsi_device_path_resolver_test.go @@ -42,7 +42,7 @@ var _ = Describe("scsiDevicePathResolver", func() { Expect(timeout).To(BeFalse()) Expect(realPath).To(Equal("fake-id-resolved-device-path")) - Expect(scsiIDDevicePathResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings)) + Expect(scsiIDDevicePathResolver.GetRealDevicePathDiskSettings).To(ContainElement(diskSettings)) }) }) @@ -60,7 +60,7 @@ var _ = Describe("scsiDevicePathResolver", func() { Expect(timeout).To(BeFalse()) Expect(realPath).To(Equal("fake-volume-id-resolved-device-path")) - Expect(scsiVolumeIDDevicePathResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings)) + Expect(scsiVolumeIDDevicePathResolver.GetRealDevicePathDiskSettings).To(ContainElement(diskSettings)) }) }) @@ -80,7 +80,7 @@ var _ = Describe("scsiDevicePathResolver", func() { Expect(timeout).To(BeFalse()) Expect(realPath).To(Equal("fake-lun-resolved-device-path")) - Expect(scsiLunDevicePathResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings)) + Expect(scsiLunDevicePathResolver.GetRealDevicePathDiskSettings).To(ContainElement(diskSettings)) }) }) diff --git a/infrastructure/devicepathresolver/symlink_device_resolver.go b/infrastructure/devicepathresolver/symlink_device_resolver.go new file mode 100644 index 000000000..e47f7a104 --- /dev/null +++ b/infrastructure/devicepathresolver/symlink_device_resolver.go @@ -0,0 +1,101 @@ +package devicepathresolver + +import ( + bosherr "github.com/cloudfoundry/bosh-utils/errors" + boshlog "github.com/cloudfoundry/bosh-utils/logger" + boshsys "github.com/cloudfoundry/bosh-utils/system" + + boshudev "github.com/cloudfoundry/bosh-agent/v2/platform/udevdevice" +) + +const ( + // NVMeDevicePattern is a glob pattern matching NVMe namespace devices. + NVMeDevicePattern = "/dev/nvme*n1" + + // NVMeDevicePathPrefix is the common prefix for NVMe device paths. + // Used to detect if a device path is an NVMe device. + NVMeDevicePathPrefix = "/dev/nvme" +) + +type SymlinkDeviceResolver struct { + fs boshsys.FileSystem + udev boshudev.UdevDevice + logger boshlog.Logger + logTag string +} + +// NewSymlinkDeviceResolver creates a new symlink device resolver. +func NewSymlinkDeviceResolver( + fs boshsys.FileSystem, + udev boshudev.UdevDevice, + logger boshlog.Logger, +) *SymlinkDeviceResolver { + return &SymlinkDeviceResolver{ + fs: fs, + udev: udev, + logger: logger, + logTag: "SymlinkDeviceResolver", + } +} + +// ResolveSymlinksToDevices resolves all symlinks matching the given pattern +// and returns a map of resolved device paths -> symlink paths. +// +// udevadm trigger and settle are called before globbing to avoid a race condition: +// NVMe block devices (/dev/nvme*) appear synchronously at boot, but the +// /dev/disk/by-id/ symlinks are created asynchronously by udev. Without waiting, +// globbing may return no symlinks, causing all NVMe devices to be misidentified +// as instance storage (instead of EBS/managed volumes). +func (r *SymlinkDeviceResolver) ResolveSymlinksToDevices(symlinkPattern string) (map[string]string, error) { + if err := r.udev.Trigger(); err != nil { + return nil, bosherr.WrapError(err, "Running udevadm trigger") + } + if err := r.udev.Settle(); err != nil { + return nil, bosherr.WrapError(err, "Running udevadm settle") + } + + symlinks, err := r.fs.Glob(symlinkPattern) + if err != nil { + return nil, bosherr.WrapErrorf(err, "Globbing symlinks with pattern '%s'", symlinkPattern) + } + + result := make(map[string]string) + for _, symlink := range symlinks { + absPath, err := r.fs.ReadAndFollowLink(symlink) + if err != nil { + return nil, bosherr.WrapErrorf(err, "Resolving managed volume symlink '%s'", symlink) + } + + r.logger.Debug(r.logTag, "Resolved symlink: %s -> %s", symlink, absPath) + result[absPath] = symlink + } + + return result, nil +} + +// GetDevicesByPattern returns all devices matching the given pattern. +func (r *SymlinkDeviceResolver) GetDevicesByPattern(devicePattern string) ([]string, error) { + devices, err := r.fs.Glob(devicePattern) + if err != nil { + return nil, bosherr.WrapErrorf(err, "Globbing devices with pattern '%s'", devicePattern) + } + + r.logger.Debug(r.logTag, "Found devices matching '%s': %v", devicePattern, devices) + return devices, nil +} + +// FilterDevices returns devices that are NOT in the exclusion map. +// This is used to filter out IaaS-managed volumes (EBS, Azure Managed Disks, etc.) +// from the list of all NVMe devices, leaving only instance/ephemeral storage. +func (r *SymlinkDeviceResolver) FilterDevices(allDevices []string, excludeDevices map[string]string) []string { + var filtered []string + for _, device := range allDevices { + if _, excluded := excludeDevices[device]; !excluded { + filtered = append(filtered, device) + r.logger.Debug(r.logTag, "Including device: %s", device) + } else { + r.logger.Debug(r.logTag, "Excluding device: %s (symlink: %s)", device, excludeDevices[device]) + } + } + return filtered +} diff --git a/infrastructure/devicepathresolver/symlink_device_resolver_test.go b/infrastructure/devicepathresolver/symlink_device_resolver_test.go new file mode 100644 index 000000000..b64e80c64 --- /dev/null +++ b/infrastructure/devicepathresolver/symlink_device_resolver_test.go @@ -0,0 +1,157 @@ +package devicepathresolver_test + +import ( + "errors" + "os" + "runtime" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + fakesys "github.com/cloudfoundry/bosh-utils/system/fakes" + + . "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver" + fakeudev "github.com/cloudfoundry/bosh-agent/v2/platform/udevdevice/fakes" +) + +var _ = Describe("SymlinkDeviceResolver", func() { + var ( + fs *fakesys.FakeFileSystem + udev *fakeudev.FakeUdevDevice + logger boshlog.Logger + resolver *SymlinkDeviceResolver + ) + + BeforeEach(func() { + if runtime.GOOS == "windows" { + Skip("Not applicable on Windows") + } + + fs = fakesys.NewFakeFileSystem() + udev = fakeudev.NewFakeUdevDevice() + logger = boshlog.NewLogger(boshlog.LevelNone) + resolver = NewSymlinkDeviceResolver(fs, udev, logger) + }) + + Describe("ResolveSymlinksToDevices", func() { + It("returns empty map when no symlinks match the pattern", func() { + fs.SetGlob("/dev/disk/by-id/nvme-*", []string{}) + + result, err := resolver.ResolveSymlinksToDevices("/dev/disk/by-id/nvme-*") + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeEmpty()) + }) + + It("resolves symlinks to their target device paths", func() { + err := fs.MkdirAll("/dev/disk/by-id", os.FileMode(0750)) + Expect(err).ToNot(HaveOccurred()) + + // Create target device files + err = fs.WriteFileString("/dev/nvme1n1", "") + Expect(err).ToNot(HaveOccurred()) + err = fs.WriteFileString("/dev/nvme2n1", "") + Expect(err).ToNot(HaveOccurred()) + + fs.SetGlob("/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*", []string{ + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123", + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol456", + }) + err = fs.Symlink("/dev/nvme1n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123") + Expect(err).ToNot(HaveOccurred()) + err = fs.Symlink("/dev/nvme2n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol456") + Expect(err).ToNot(HaveOccurred()) + + result, err := resolver.ResolveSymlinksToDevices("/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*") + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(HaveLen(2)) + Expect(result["/dev/nvme1n1"]).To(Equal("/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123")) + Expect(result["/dev/nvme2n1"]).To(Equal("/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol456")) + }) + + It("returns error when a managed volume symlink cannot be resolved", func() { + err := fs.MkdirAll("/dev/disk/by-id", os.FileMode(0750)) + Expect(err).ToNot(HaveOccurred()) + + // Create target device file for valid symlink + err = fs.WriteFileString("/dev/nvme1n1", "") + Expect(err).ToNot(HaveOccurred()) + + fs.SetGlob("/dev/disk/by-id/nvme-*", []string{ + "/dev/disk/by-id/nvme-valid", + "/dev/disk/by-id/nvme-invalid", + }) + err = fs.Symlink("/dev/nvme1n1", "/dev/disk/by-id/nvme-valid") + Expect(err).ToNot(HaveOccurred()) + // nvme-invalid has no symlink target + + _, err = resolver.ResolveSymlinksToDevices("/dev/disk/by-id/nvme-*") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("nvme-invalid")) + }) + + It("returns error when glob fails", func() { + fs.GlobErr = errors.New("glob error") + + _, err := resolver.ResolveSymlinksToDevices("/dev/disk/by-id/nvme-*") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("glob error")) + }) + }) + + Describe("GetDevicesByPattern", func() { + It("returns devices matching the pattern", func() { + fs.SetGlob("/dev/nvme*n1", []string{"/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1"}) + + devices, err := resolver.GetDevicesByPattern("/dev/nvme*n1") + Expect(err).ToNot(HaveOccurred()) + Expect(devices).To(ConsistOf("/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1")) + }) + + It("returns empty slice when no devices match", func() { + fs.SetGlob("/dev/nvme*n1", []string{}) + + devices, err := resolver.GetDevicesByPattern("/dev/nvme*n1") + Expect(err).ToNot(HaveOccurred()) + Expect(devices).To(BeEmpty()) + }) + + It("returns error when glob fails", func() { + fs.GlobErr = errors.New("glob error") + + _, err := resolver.GetDevicesByPattern("/dev/nvme*n1") + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("FilterDevices", func() { + It("returns devices not in the exclusion map", func() { + allDevices := []string{"/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1", "/dev/nvme3n1"} + excludeDevices := map[string]string{ + "/dev/nvme1n1": "/dev/disk/by-id/ebs-vol1", + "/dev/nvme2n1": "/dev/disk/by-id/ebs-vol2", + } + + filtered := resolver.FilterDevices(allDevices, excludeDevices) + Expect(filtered).To(ConsistOf("/dev/nvme0n1", "/dev/nvme3n1")) + }) + + It("returns all devices when exclusion map is empty", func() { + allDevices := []string{"/dev/nvme0n1", "/dev/nvme1n1"} + excludeDevices := map[string]string{} + + filtered := resolver.FilterDevices(allDevices, excludeDevices) + Expect(filtered).To(ConsistOf("/dev/nvme0n1", "/dev/nvme1n1")) + }) + + It("returns empty slice when all devices are excluded", func() { + allDevices := []string{"/dev/nvme0n1"} + excludeDevices := map[string]string{ + "/dev/nvme0n1": "/dev/disk/by-id/ebs-vol1", + } + + filtered := resolver.FilterDevices(allDevices, excludeDevices) + Expect(filtered).To(BeEmpty()) + }) + }) +}) diff --git a/infrastructure/devicepathresolver/virtio_device_path_resolver_test.go b/infrastructure/devicepathresolver/virtio_device_path_resolver_test.go index b7873c916..142a695a8 100644 --- a/infrastructure/devicepathresolver/virtio_device_path_resolver_test.go +++ b/infrastructure/devicepathresolver/virtio_device_path_resolver_test.go @@ -63,7 +63,7 @@ var _ = Describe("VirtioDevicePathResolver", func() { Expect(timeout).To(BeFalse()) Expect(realPath).To(Equal("fake-mapped-resolved-device-path")) - Expect(mappedDevicePathResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings)) + Expect(mappedDevicePathResolver.GetRealDevicePathDiskSettings).To(ContainElement(diskSettings)) }) Context("when mappedDevicePathResolver times out", func() { diff --git a/platform/linux_platform.go b/platform/linux_platform.go index 17e5bfbbf..e19364496 100644 --- a/platform/linux_platform.go +++ b/platform/linux_platform.go @@ -7,6 +7,7 @@ import ( "path" "path/filepath" "regexp" + "sort" "strconv" "strings" "text/template" @@ -78,6 +79,14 @@ type LinuxOptions struct { // possible values: virtio, scsi, iscsi, "" DevicePathResolutionType string + // Pattern for identifying IaaS-managed volumes (e.g., EBS on AWS) + // Used to filter out non-instance storage on NVMe systems + InstanceStorageManagedVolumePattern string + + // Pattern for discovering all potential instance storage devices + // Used for device enumeration on NVMe systems + InstanceStorageDevicePattern string + // Strategy for resolving ephemeral & persistent disk partitioners; // possible values: parted, "" (default is sfdisk if disk < 2TB, parted otherwise) PartitionerType string @@ -114,6 +123,7 @@ type linux struct { certManager boshcert.Manager monitRetryStrategy boshretry.RetryStrategy devicePathResolver boshdpresolv.DevicePathResolver + symlinkDeviceResolver *boshdpresolv.SymlinkDeviceResolver options LinuxOptions state *BootstrapState logger boshlog.Logger @@ -139,6 +149,7 @@ func NewLinuxPlatform( certManager boshcert.Manager, monitRetryStrategy boshretry.RetryStrategy, devicePathResolver boshdpresolv.DevicePathResolver, + symlinkDeviceResolver *boshdpresolv.SymlinkDeviceResolver, state *BootstrapState, options LinuxOptions, logger boshlog.Logger, @@ -162,6 +173,7 @@ func NewLinuxPlatform( certManager: certManager, monitRetryStrategy: monitRetryStrategy, devicePathResolver: devicePathResolver, + symlinkDeviceResolver: symlinkDeviceResolver, state: state, options: options, logger: logger, @@ -735,19 +747,24 @@ func (p linux) SetupRawEphemeralDisks(devices []boshsettings.DiskSettings) (err return nil } - p.logger.Info(logTag, "Setting up raw ephemeral disks") + if len(devices) == 0 { + return nil + } - for i, device := range devices { - realPath, _, err := p.devicePathResolver.GetRealDevicePath(device) - if err != nil { - return bosherr.WrapError(err, "Getting real device path") - } + p.logger.Info(logTag, "Setting up %d raw ephemeral disk(s)", len(devices)) + + instanceStorageDevices, err := p.discoverInstanceStorageDevices(devices) + if err != nil { + return bosherr.WrapError(err, "Discovering instance storage devices") + } + // Partition each discovered device + for i, devicePath := range instanceStorageDevices { // check if device is already partitioned correctly stdout, stderr, _, err := p.cmdRunner.RunCommand( "parted", "-s", - realPath, + devicePath, "p", ) @@ -755,21 +772,22 @@ func (p linux) SetupRawEphemeralDisks(devices []boshsettings.DiskSettings) (err // "unrecognised disk label" is acceptable, since the disk may not have been partitioned if !strings.Contains(stdout, "unrecognised disk label") && !strings.Contains(stderr, "unrecognised disk label") { - return bosherr.WrapError(err, "Setting up raw ephemeral disks") + return bosherr.WrapErrorf(err, "Checking partition on %s", devicePath) } } if strings.Contains(stdout, "Partition Table: gpt") && strings.Contains(stdout, "raw-ephemeral-") { + p.logger.Info(logTag, "Device %s already partitioned, skipping", devicePath) continue } // change to gpt partition type, change units to percentage, make partition with name and span from 0-100% - p.logger.Info(logTag, "Creating partition on `%s'", realPath) + p.logger.Info(logTag, "Creating partition on %s as raw-ephemeral-%d", devicePath, i) _, _, _, err = p.cmdRunner.RunCommand( "parted", "-s", - realPath, + devicePath, "mklabel", "gpt", "unit", @@ -781,13 +799,97 @@ func (p linux) SetupRawEphemeralDisks(devices []boshsettings.DiskSettings) (err ) if err != nil { - return bosherr.WrapError(err, "Setting up raw ephemeral disks") + return bosherr.WrapErrorf(err, "Creating partition on %s", devicePath) } } return nil } +// discoverInstanceStorageDevices finds the actual device paths for instance storage. +// For NVMe devices with a configured managed volume pattern, it uses symlink-based +// filtering to exclude IaaS-managed volumes (e.g., EBS on AWS). +// Otherwise, it uses the DevicePathResolver to resolve each device directly. +func (p linux) discoverInstanceStorageDevices(devices []boshsettings.DiskSettings) ([]string, error) { + if len(devices) == 0 { + return []string{}, nil + } + + // Use NVMe symlink filtering if: + // 1. A managed volume pattern is configured (tells us what to exclude) + // 2. The CPI reports NVMe device paths + // 3. The symlink resolver is available + hasPattern := p.options.InstanceStorageManagedVolumePattern != "" + hasResolver := p.symlinkDeviceResolver != nil + hasNVMe := p.hasNVMeDevices(devices) + p.logger.Debug(logTag, "Instance storage resolution: hasPattern=%v, hasResolver=%v, hasNVMe=%v", hasPattern, hasResolver, hasNVMe) + + if hasPattern && hasResolver && hasNVMe { + return p.discoverNVMeInstanceStorage(devices) + } + + return p.discoverIdentityInstanceStorage(devices) +} + +// hasNVMeDevices checks if any device path from the CPI is an NVMe device. +func (p linux) hasNVMeDevices(devices []boshsettings.DiskSettings) bool { + for _, device := range devices { + if strings.HasPrefix(device.Path, boshdpresolv.NVMeDevicePathPrefix) { + return true + } + } + return false +} + +// discoverNVMeInstanceStorage discovers NVMe instance storage by filtering out +// IaaS-managed volumes (e.g., EBS on AWS, Managed Disks on Azure) using symlinks. +func (p linux) discoverNVMeInstanceStorage(devices []boshsettings.DiskSettings) ([]string, error) { + nvmePattern := p.options.InstanceStorageDevicePattern + if nvmePattern == "" { + nvmePattern = boshdpresolv.NVMeDevicePattern + } + p.logger.Debug(logTag, "Discovering NVMe instance storage: device pattern=%s, managed volume pattern=%s", nvmePattern, p.options.InstanceStorageManagedVolumePattern) + + allNvmeDevices, err := p.symlinkDeviceResolver.GetDevicesByPattern(nvmePattern) + if err != nil { + return nil, bosherr.WrapError(err, "Globbing NVMe devices") + } + p.logger.Debug(logTag, "Found NVMe devices: %v", allNvmeDevices) + + managedDevices, err := p.symlinkDeviceResolver.ResolveSymlinksToDevices(p.options.InstanceStorageManagedVolumePattern) + if err != nil { + return nil, bosherr.WrapError(err, "Resolving managed disk symlinks") + } + p.logger.Debug(logTag, "Found %d managed (IaaS) devices to exclude: %v", len(managedDevices), managedDevices) + + instanceStorage := p.symlinkDeviceResolver.FilterDevices(allNvmeDevices, managedDevices) + sort.Strings(instanceStorage) + + for _, devicePath := range instanceStorage { + p.logger.Info(logTag, "Discovered instance storage: %s", devicePath) + } + + if len(instanceStorage) != len(devices) { + return nil, bosherr.Errorf("Expected %d instance storage devices but discovered %d: %v", + len(devices), len(instanceStorage), instanceStorage) + } + + return instanceStorage, nil +} + +// discoverIdentityInstanceStorage uses the DevicePathResolver for each device. +func (p linux) discoverIdentityInstanceStorage(devices []boshsettings.DiskSettings) ([]string, error) { + paths := make([]string, len(devices)) + for i, device := range devices { + realPath, _, err := p.devicePathResolver.GetRealDevicePath(device) + if err != nil { + return nil, bosherr.WrapErrorf(err, "Getting device %s path", device) + } + paths[i] = realPath + } + return paths, nil +} + func (p linux) SetupDynamicDisk(diskSetting boshsettings.DiskSettings) error { devicePath, timedOut, err := p.devicePathResolver.GetRealDevicePath(diskSetting) if err != nil { diff --git a/platform/linux_platform_test.go b/platform/linux_platform_test.go index 55ac7e471..c062193ae 100644 --- a/platform/linux_platform_test.go +++ b/platform/linux_platform_test.go @@ -22,6 +22,7 @@ import ( fakeuuidgen "github.com/cloudfoundry/bosh-utils/uuid/fakes" fakelogstarprovider "github.com/cloudfoundry/bosh-agent/v2/agent/logstarprovider/logstarproviderfakes" + boshdpresolv "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver" fakedpresolv "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver/fakes" . "github.com/cloudfoundry/bosh-agent/v2/platform" fakecdrom "github.com/cloudfoundry/bosh-agent/v2/platform/cdrom/fakes" @@ -33,6 +34,7 @@ import ( fakenet "github.com/cloudfoundry/bosh-agent/v2/platform/net/fakes" boship "github.com/cloudfoundry/bosh-agent/v2/platform/net/ip" fakestats "github.com/cloudfoundry/bosh-agent/v2/platform/stats/fakes" + fakeudev "github.com/cloudfoundry/bosh-agent/v2/platform/udevdevice/fakes" boshvitals "github.com/cloudfoundry/bosh-agent/v2/platform/vitals" "github.com/cloudfoundry/bosh-agent/v2/servicemanager/servicemanagerfakes" boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings" @@ -47,6 +49,7 @@ var _ = Describe("LinuxPlatform", func() { diskManager *diskfakes.FakeManager dirProvider boshdirs.Provider devicePathResolver *fakedpresolv.FakeDevicePathResolver + symlinkDeviceResolver *boshdpresolv.SymlinkDeviceResolver platform Platform cdutil *fakecdrom.FakeCDUtil compressor boshcmd.Compressor @@ -89,6 +92,8 @@ var _ = Describe("LinuxPlatform", func() { certManager = new(certfakes.FakeManager) monitRetryStrategy = fakeretry.NewFakeRetryStrategy() devicePathResolver = fakedpresolv.NewFakeDevicePathResolver() + symlinkDeviceResolver = boshdpresolv.NewSymlinkDeviceResolver(fs, fakeudev.NewFakeUdevDevice(), logger) + fakeDefaultNetworkResolver = &fakenet.FakeDefaultNetworkResolver{} serviceManager = &servicemanagerfakes.FakeServiceManager{} @@ -148,6 +153,7 @@ var _ = Describe("LinuxPlatform", func() { certManager, monitRetryStrategy, devicePathResolver, + symlinkDeviceResolver, state, options, logger, @@ -466,6 +472,7 @@ bosh_foobar:...` certManager, monitRetryStrategy, devicePathResolver, + symlinkDeviceResolver, state, options, logger, @@ -703,6 +710,7 @@ bosh_foobar:...` certManager, monitRetryStrategy, devicePathResolver, + symlinkDeviceResolver, state, options, logger, @@ -1906,6 +1914,148 @@ Number Start End Size File system Name Flags Expect(len(cmdRunner.RunCommands)).To(Equal(0)) }) }) + + Context("NVMe instance storage discovery", func() { + BeforeEach(func() { + // Enable NVMe instance storage discovery by setting the managed volume pattern + options.InstanceStorageManagedVolumePattern = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*" + + devicePathResolver.GetRealDevicePathStub = func(diskSettings boshsettings.DiskSettings) (string, bool, error) { + return diskSettings.Path, false, nil + } + }) + + It("discovers instance storage by excluding EBS volumes via symlinks", func() { + // Setup: 3 NVMe devices, 2 are EBS (nvme0n1, nvme1n1), 1 is instance storage (nvme2n1) + + // Create the NVMe device files + err := fs.WriteFileString("/dev/nvme0n1", "") + Expect(err).ToNot(HaveOccurred()) + err = fs.WriteFileString("/dev/nvme1n1", "") + Expect(err).ToNot(HaveOccurred()) + err = fs.WriteFileString("/dev/nvme2n1", "") + Expect(err).ToNot(HaveOccurred()) + + // Create symlink directory + err = fs.MkdirAll("/dev/disk/by-id", os.FileMode(0750)) + Expect(err).ToNot(HaveOccurred()) + + // Create symlinks for EBS volumes + err = fs.Symlink("/dev/nvme0n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123") + Expect(err).ToNot(HaveOccurred()) + err = fs.Symlink("/dev/nvme1n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol456") + Expect(err).ToNot(HaveOccurred()) + + // Set up glob patterns for the symlinkDeviceResolver + fs.SetGlob("/dev/nvme*n1", []string{"/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1"}) + fs.SetGlob("/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*", []string{ + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123", + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol456", + }) + + // Mock parted output for nvme2n1 (instance storage - needs partitioning) + cmdRunner.AddCmdResult("parted -s /dev/nvme2n1 p", fakesys.FakeCmdResult{ + Error: errors.New("unrecognised disk label"), + Stdout: "Error: /dev/nvme2n1: unrecognised disk label", + }) + + err = platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/nvme0n1"}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(len(cmdRunner.RunCommands)).To(Equal(2)) + Expect(cmdRunner.RunCommands[0]).To(Equal([]string{"parted", "-s", "/dev/nvme2n1", "p"})) + Expect(cmdRunner.RunCommands[1]).To(Equal([]string{"parted", "-s", "/dev/nvme2n1", "mklabel", "gpt", "unit", "%", "mkpart", "raw-ephemeral-0", "0", "100"})) + }) + + It("returns error when no instance storage devices found but CPI expects some", func() { + // Create the NVMe device file + err := fs.WriteFileString("/dev/nvme0n1", "") + Expect(err).ToNot(HaveOccurred()) + + // Create symlink directory + err = fs.MkdirAll("/dev/disk/by-id", os.FileMode(0750)) + Expect(err).ToNot(HaveOccurred()) + + // All NVMe devices are EBS - no instance storage available + fs.SetGlob("/dev/nvme*n1", []string{"/dev/nvme0n1"}) + fs.SetGlob("/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*", []string{ + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123", + }) + err = fs.Symlink("/dev/nvme0n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123") + Expect(err).ToNot(HaveOccurred()) + + err = platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/nvme2n1"}}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Expected 1 instance storage devices but discovered 0")) + }) + + It("returns error when globbing NVMe devices fails", func() { + fs.GlobErr = errors.New("permission denied reading /dev") + + err := platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/nvme1n1"}}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Globbing NVMe devices")) + }) + + It("skips symlinks that fail to resolve and continues", func() { + // Create the NVMe device files + err := fs.WriteFileString("/dev/nvme0n1", "") + Expect(err).ToNot(HaveOccurred()) + err = fs.WriteFileString("/dev/nvme1n1", "") + Expect(err).ToNot(HaveOccurred()) + err = fs.WriteFileString("/dev/nvme2n1", "") + Expect(err).ToNot(HaveOccurred()) + + // Create symlink directory + err = fs.MkdirAll("/dev/disk/by-id", os.FileMode(0750)) + Expect(err).ToNot(HaveOccurred()) + + // Set up NVMe devices: nvme0n1 (EBS), nvme1n1 and nvme2n1 (instance storage) + fs.SetGlob("/dev/nvme*n1", []string{"/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1"}) + fs.SetGlob("/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*", []string{ + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123", + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_broken", // broken symlink + }) + err = fs.Symlink("/dev/nvme0n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123") + Expect(err).ToNot(HaveOccurred()) + // Note: nvme-Amazon_Elastic_Block_Store_broken has no symlink target - it will be skipped + + // Mock parted for nvme1n1 and nvme2n1 (instance storage) + cmdRunner.AddCmdResult("parted -s /dev/nvme1n1 p", fakesys.FakeCmdResult{ + Error: errors.New("unrecognised disk label"), + Stdout: "Error: /dev/nvme1n1: unrecognised disk label", + }) + cmdRunner.AddCmdResult("parted -s /dev/nvme2n1 p", fakesys.FakeCmdResult{ + Error: errors.New("unrecognised disk label"), + Stdout: "Error: /dev/nvme2n1: unrecognised disk label", + }) + + err = platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/nvme0n1"}, {Path: "/dev/nvme1n1"}}) + + Expect(err).ToNot(HaveOccurred()) + // Should partition nvme1n1 and nvme2n1 (only nvme0n1 was identified as EBS) + Expect(len(cmdRunner.RunCommands)).To(Equal(4)) + }) + + It("uses CPI paths directly for non-NVMe devices", func() { + // For non-NVMe devices, the device path resolver is used directly + devicePathResolver.RealDevicePath = "/dev/xvdb" + + cmdRunner.AddCmdResult("parted -s /dev/xvdb p", fakesys.FakeCmdResult{ + Error: errors.New("unrecognised disk label"), + Stdout: "Error: /dev/xvdb: unrecognised disk label", + }) + + err := platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/xvdb"}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(len(cmdRunner.RunCommands)).To(Equal(2)) + Expect(cmdRunner.RunCommands[0]).To(Equal([]string{"parted", "-s", "/dev/xvdb", "p"})) + Expect(cmdRunner.RunCommands[1]).To(Equal([]string{"parted", "-s", "/dev/xvdb", "mklabel", "gpt", "unit", "%", "mkpart", "raw-ephemeral-0", "0", "100"})) + }) + }) }) Describe("SetupDataDir", func() { @@ -3799,6 +3949,7 @@ from-device-path dm-0 NETAPP ,LUN C-Mode certManager, monitRetryStrategy, devicePathResolver, + symlinkDeviceResolver, state, options, logger, diff --git a/platform/provider.go b/platform/provider.go index 7349c0110..a4101c87d 100644 --- a/platform/provider.go +++ b/platform/provider.go @@ -158,6 +158,9 @@ func NewProvider(logger boshlog.Logger, dirProvider boshdirs.Provider, statsColl devicePathResolver = devicepathresolver.NewFallbackDevicePathResolver(symlinkLunResolver, devicePathResolver, logger) } + // Symlink device resolver for NVMe instance storage discovery (filtering out EBS/managed disks) + symlinkDeviceResolver := devicepathresolver.NewSymlinkDeviceResolver(fs, boshudev.NewConcreteUdevDevice(runner, logger), logger) + uuidGenerator := boshuuid.NewGenerator() logsTarProvider := boshlogstarprovider.NewLogsTarProvider(compressor, copier, dirProvider) @@ -176,6 +179,7 @@ func NewProvider(logger boshlog.Logger, dirProvider boshdirs.Provider, statsColl centosCertManager, monitRetryStrategy, devicePathResolver, + symlinkDeviceResolver, bootstrapState, options.Linux, logger, @@ -202,6 +206,7 @@ func NewProvider(logger boshlog.Logger, dirProvider boshdirs.Provider, statsColl ubuntuCertManager, monitRetryStrategy, devicePathResolver, + symlinkDeviceResolver, bootstrapState, options.Linux, logger,