Merge #156821

156821: roachprod/roachtest: uniform storage capabilities r=DarrylWong a=golgeek

Until now, each cloud provider implementation had its own capabilities with regards to storage options. GCE was the only first class citizen with the most available options exposed in roachtest.

This patch attempts to bridge the feature parity gap between the cloud providers (up to what's exposed by each providers), bringing support for the following options in roachprod and roachtest:
- GCE:
  - local SSD
  - network disk size
  - network disk type (pd-standard, pd-ssd)
  - network disk count
  - RAID0 or multiple stores
- AWS:
  - local SSD
  - network disk size
  - network disk throughput
  - network disk IOPS
  - NEW: network disk type (gp2, gp3, io1, io2, st1, sc1, standard)
  - NEW: network disk count
  - NEW: RAID0 or multiple stores
- Azure:
  - local SSD
  - network disk size
  - NEW: network disk IOPS (ultra-disk only)
  - NEW: network disk type (standard-ssd, premium-ssd, premium-ssd-v2,
      ultra-disk)
  - NEW: network disk count
  - NEW: RAID0 or multiple stores
- IBM:
  - network disk size
  - network disk IOPS
  - network disk type (general-purpose, 5iops-tier, 10iops-tier, custom)
  - network disk count
  - RAID0 or multiple stores

This patch also splits the disk setup startup script snippets, with:
- a provider-specific way of detecting the attached disks
- a common logic to mount, format and aggregate the disks

This allows to offer the following filesystems across the board in roachprod (and roachtest):
- Ext4
- ZFS
- XFS
- F2FS (available for AWS, Azure and GCP, pending newer kernel for IBM)
- Btrfs

Epic: none
Closes: #123775
Informs: #146661, #113869
Release note: None

Co-authored-by: Ludovic Leroux <ludo.leroux@cockroachlabs.com>

Author: craig[bot]

pkg/cmd/roachprod/cli/flags.go

@@ -137,8 +137,8 @@ func initCreateCmdFlags(createCmd *cobra.Command) {
 		"lifetime", "l", 12*time.Hour, "Lifetime of the cluster")
 	createCmd.Flags().BoolVar(&createVMOpts.SSDOpts.UseLocalSSD,
 		"local-ssd", true, "Use local SSD")
-	createCmd.Flags().StringVar(&createVMOpts.SSDOpts.FileSystem,
-		"filesystem", vm.Ext4, "The underlying file system(ext4/zfs). ext4 is used by default.")
+	createCmd.Flags().StringVar((*string)(&createVMOpts.SSDOpts.FileSystem),
+		"filesystem", string(vm.Ext4), "The underlying file system(ext4/zfs/xfs/f2fs/btrfs). ext4 is used by default.")
 	createCmd.Flags().BoolVar(&createVMOpts.SSDOpts.NoExt4Barrier,
 		"local-ssd-no-ext4-barrier", true,
 		`Mount the local SSD with the "-o nobarrier" flag. Ignored if --local-ssd=false is specified.`)

pkg/cmd/roachtest/cluster.go

@@ -1909,7 +1909,7 @@ func (c *clusterImpl) CreateSnapshot(
 func (c *clusterImpl) ApplySnapshots(ctx context.Context, snapshots []vm.VolumeSnapshot) error {
 	opts := vm.VolumeCreateOpts{
 		Size: c.spec.VolumeSize,
-		Type: c.spec.GCE.VolumeType, // TODO(irfansharif): This is only applicable to GCE. Change that.
+		Type: c.spec.VolumeType,
 		Labels: map[string]string{
 			vm.TagUsage: "roachtest",
 		},

pkg/cmd/roachtest/roachtestflags/flags.go

@@ -523,10 +523,10 @@ var (
 		Usage: `Override use of local SSD`,
 	})
 
-	OverrideFilesystem string
+	OverrideFilesystem vm.Filesystem
 	_                  = registerRunFlag(&OverrideFilesystem, FlagInfo{
 		Name:  "filesystem",
-		Usage: `Override the underlying file system(ext4/zfs)`,
+		Usage: `Override the underlying file system(ext4/zfs/xfs/f2fs/btrfs)`,
 	})
 
 	OverrideNoExt4Barrier bool

pkg/cmd/roachtest/roachtestflags/manager.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/vm"
 	"github.com/cockroachdb/errors"
 	"github.com/spf13/pflag"
 )
@@ -74,6 +75,8 @@ func (m *manager) AddFlagsToCommand(cmd cmdID, cmdFlags *pflag.FlagSet) {
 			cmdFlags.StringToStringVarP(p, f.Name, f.Shorthand, *p, usage)
 		case *spec.Cloud:
 			cmdFlags.VarP(&cloudValue{val: p}, f.Name, f.Shorthand, usage)
+		case *vm.Filesystem:
+			cmdFlags.VarP(&filesystemValue{val: p}, f.Name, f.Shorthand, usage)
 		default:
 			panic(fmt.Sprintf("unsupported pointer type %T", p))
 		}
@@ -146,3 +149,26 @@ func (cv *cloudValue) Set(str string) error {
 	*cv.val = val
 	return nil
 }
+
+type filesystemValue struct {
+	val *vm.Filesystem
+}
+
+var _ pflag.Value = (*filesystemValue)(nil)
+
+func (fv *filesystemValue) String() string {
+	return string(*fv.val)
+}
+
+func (fv *filesystemValue) Type() string {
+	return "string"
+}
+
+func (fv *filesystemValue) Set(str string) error {
+	val, err := vm.ParseFilesystemString(str)
+	if err != nil {
+		return err
+	}
+	*fv.val = val
+	return nil
+}

pkg/cmd/roachtest/spec/cluster_spec.go

@@ -25,13 +25,11 @@ import (
 type fileSystemType int
 
 const (
-	// Since ext4 is the default of 0, it isn't being
-	// used anywhere in the code. Therefore, it isn't
-	// added as a const here since it is unused, and
-	// leads to a lint error.
-
-	// Zfs file system.
-	Zfs fileSystemType = 1
+	Ext4 fileSystemType = iota
+	Zfs
+	Xfs
+	F2fs
+	Btrfs
 
 	// Extra labels added by roachtest
 	RoachtestBranch = "roachtest-branch"
@@ -205,6 +203,8 @@ type ClusterSpec struct {
 	SSDs                 int
 	RAID0                bool
 	VolumeSize           int
+	VolumeType           string
+	VolumeCount          int
 	LocalSSD             LocalSSDSetting
 	Geo                  bool
 	Lifetime             time.Duration
@@ -224,8 +224,6 @@ type ClusterSpec struct {
 	GCE struct {
 		MachineType    string
 		MinCPUPlatform string
-		VolumeType     string
-		VolumeCount    int // volume count is only supported for GCE. This can be moved up if we start supporting other clouds
 		Zones          string
 	} `cloud:"gce"`
 
@@ -242,13 +240,13 @@ type ClusterSpec struct {
 	// Azure-specific arguments. These values apply only on clusters instantiated on Azure.
 	Azure struct {
 		Zones string
+		// VolumeIOPS is the provisioned IOPS for ultra-disks.
+		VolumeIOPS int
 	} `cloud:"azure"`
 	// IBM-specific arguments. These values apply only on clusters instantiated on IBM.
 	IBM struct {
 		MachineType string
-		VolumeType  string
 		VolumeIOPS  int
-		VolumeCount int
 		Zones       string
 	} `cloud:"ibm"`
 }
@@ -319,16 +317,24 @@ func awsMachineSupportsSSD(machineType string) bool {
 
 func getAWSOpts(
 	machineType string,
-	volumeSize, ebsThroughput int,
+	volumeSize, volumeCount, ebsThroughput int,
+	volumeType string,
 	ebsIOPS int,
 	localSSD bool,
+	RAID0 bool,
 	useSpotVMs bool,
 	bootDiskOnly bool,
 ) vm.ProviderOpts {
 	opts := aws.DefaultProviderOpts()
 	if volumeSize != 0 {
 		opts.DefaultEBSVolume.Disk.VolumeSize = volumeSize
 	}
+	if volumeType != "" {
+		opts.DefaultEBSVolume.Disk.VolumeType = volumeType
+	}
+	if volumeCount != 0 {
+		opts.EBSVolumeCount = volumeCount
+	}
 	if ebsIOPS != 0 {
 		opts.DefaultEBSVolume.Disk.IOPs = ebsIOPS
 	}
@@ -340,6 +346,7 @@ func getAWSOpts(
 	} else {
 		opts.MachineType = machineType
 	}
+	opts.UseMultipleDisks = !RAID0
 	opts.UseSpot = useSpotVMs
 	opts.BootDiskOnly = bootDiskOnly
 	return opts
@@ -389,13 +396,31 @@ func getGCEOpts(
 	return opts
 }
 
-func getAzureOpts(machineType string, volumeSize int, bootDiskOnly bool) vm.ProviderOpts {
+func getAzureOpts(
+	machineType string,
+	volumeSize int,
+	volumeType string,
+	volumeCount int,
+	volumeIOPS int,
+	RAID0 bool,
+	bootDiskOnly bool,
+) vm.ProviderOpts {
 	opts := azure.DefaultProviderOpts()
 	opts.MachineType = machineType
 	if volumeSize != 0 {
 		opts.NetworkDiskSize = int32(volumeSize)
 	}
 	opts.BootDiskOnly = bootDiskOnly
+	if volumeType != "" {
+		opts.NetworkDiskType = volumeType
+	}
+	if volumeCount != 0 {
+		opts.NetworkDiskCount = volumeCount
+	}
+	if volumeIOPS != 0 {
+		opts.UltraDiskIOPS = int64(volumeIOPS)
+	}
+	opts.UseMultipleDisks = !RAID0
 	return opts
 }
 
@@ -405,7 +430,7 @@ func getIBMOpts(
 	volumeSize int,
 	volumeType string,
 	volumeIOPS int,
-	extraVolumeCount int,
+	volumeCount int,
 	RAID0 bool,
 	bootDiskOnly bool,
 ) vm.ProviderOpts {
@@ -424,17 +449,10 @@ func getIBMOpts(
 	}
 
 	// We reuse the parameters of the default data volume for extra volumes.
-	opts.AttachedVolumes = make(ibm.IbmVolumeList, 0)
-	if extraVolumeCount > 0 {
-		for i := 0; i < extraVolumeCount; i++ {
-			opts.AttachedVolumes = append(opts.AttachedVolumes, &ibm.IbmVolume{
-				VolumeType: opts.DefaultVolume.VolumeType,
-				VolumeSize: opts.DefaultVolume.VolumeSize,
-				IOPS:       opts.DefaultVolume.IOPS,
-			})
-		}
-		opts.UseMultipleDisks = !RAID0
+	if volumeCount != 0 {
+		opts.AttachedVolumesCount = volumeCount
 	}
+	opts.UseMultipleDisks = !RAID0
 	opts.BootDiskOnly = bootDiskOnly
 
 	return opts
@@ -594,18 +612,25 @@ func (s *ClusterSpec) RoachprodOpts(
 		}
 	}
 
-	if s.FileSystem == Zfs {
-		if cloud != GCE && cloud != IBM {
-			return vm.CreateOpts{}, nil, nil, "", errors.Errorf(
-				"node creation with zfs file system not yet supported on %s", cloud,
-			)
+	switch s.FileSystem {
+	case Ext4:
+		// ext4 is the default, do nothing unless we randomly want to use zfs
+		if s.RandomlyUseZfs {
+			rng, _ := randutil.NewPseudoRand()
+			if rng.Float64() <= 0.2 {
+				createVMOpts.SSDOpts.FileSystem = vm.Zfs
+			}
 		}
+	case Zfs:
 		createVMOpts.SSDOpts.FileSystem = vm.Zfs
-	} else if s.RandomlyUseZfs && (cloud == GCE || cloud == IBM) {
-		rng, _ := randutil.NewPseudoRand()
-		if rng.Float64() <= 0.2 {
-			createVMOpts.SSDOpts.FileSystem = vm.Zfs
-		}
+	case Xfs:
+		createVMOpts.SSDOpts.FileSystem = vm.Xfs
+	case F2fs:
+		createVMOpts.SSDOpts.FileSystem = vm.F2fs
+	case Btrfs:
+		createVMOpts.SSDOpts.FileSystem = vm.Btrfs
+	default:
+		return vm.CreateOpts{}, nil, nil, "", errors.Errorf("unknown file system type: %v", s.FileSystem)
 	}
 
 	var workloadMachineType string
@@ -633,30 +658,34 @@ func (s *ClusterSpec) RoachprodOpts(
 	var workloadProviderOpts vm.ProviderOpts
 	switch cloud {
 	case AWS:
-		providerOpts = getAWSOpts(machineType, s.VolumeSize, s.AWS.VolumeThroughput, s.AWS.VolumeIOPS,
-			createVMOpts.SSDOpts.UseLocalSSD, s.UseSpotVMs, false)
-		workloadProviderOpts = getAWSOpts(workloadMachineType, s.VolumeSize, s.AWS.VolumeThroughput, s.AWS.VolumeIOPS,
-			createVMOpts.SSDOpts.UseLocalSSD, s.UseSpotVMs, !s.WorkloadRequiresDisk)
+		providerOpts = getAWSOpts(machineType, s.VolumeSize, s.VolumeCount, s.AWS.VolumeThroughput, s.VolumeType, s.AWS.VolumeIOPS,
+			createVMOpts.SSDOpts.UseLocalSSD, s.RAID0, s.UseSpotVMs, false)
+		workloadProviderOpts = getAWSOpts(workloadMachineType, s.VolumeSize, s.VolumeCount, s.AWS.VolumeThroughput, s.VolumeType, s.AWS.VolumeIOPS,
+			createVMOpts.SSDOpts.UseLocalSSD, s.RAID0, s.UseSpotVMs, !s.WorkloadRequiresDisk)
 	case GCE:
 		providerOpts = getGCEOpts(machineType, s.VolumeSize, ssdCount,
 			createVMOpts.SSDOpts.UseLocalSSD, s.RAID0, s.TerminateOnMigration,
-			s.GCE.MinCPUPlatform, vm.ParseArch(createVMOpts.Arch), s.GCE.VolumeType,
-			s.GCE.VolumeCount, s.UseSpotVMs, false,
+			s.GCE.MinCPUPlatform, vm.ParseArch(createVMOpts.Arch), s.VolumeType,
+			s.VolumeCount, s.UseSpotVMs, false,
 		)
 		workloadProviderOpts = getGCEOpts(workloadMachineType, s.VolumeSize, ssdCount,
 			createVMOpts.SSDOpts.UseLocalSSD, s.RAID0, s.TerminateOnMigration,
-			s.GCE.MinCPUPlatform, vm.ParseArch(createVMOpts.Arch), s.GCE.VolumeType,
-			s.GCE.VolumeCount, s.UseSpotVMs, !s.WorkloadRequiresDisk,
+			s.GCE.MinCPUPlatform, vm.ParseArch(createVMOpts.Arch), s.VolumeType,
+			s.VolumeCount, s.UseSpotVMs, !s.WorkloadRequiresDisk,
 		)
 	case Azure:
-		providerOpts = getAzureOpts(machineType, s.VolumeSize, false)
-		workloadProviderOpts = getAzureOpts(workloadMachineType, s.VolumeSize, true)
+		providerOpts = getAzureOpts(machineType,
+			s.VolumeSize, s.VolumeType, s.VolumeCount, s.Azure.VolumeIOPS, s.RAID0, false,
+		)
+		workloadProviderOpts = getAzureOpts(workloadMachineType,
+			s.VolumeSize, s.VolumeType, s.VolumeCount, s.Azure.VolumeIOPS, s.RAID0, true,
+		)
 	case IBM:
-		providerOpts = getIBMOpts(machineType, s.TerminateOnMigration, s.VolumeSize,
-			s.IBM.VolumeType, s.IBM.VolumeIOPS, s.IBM.VolumeCount, s.RAID0, false,
+		providerOpts = getIBMOpts(machineType, s.TerminateOnMigration,
+			s.VolumeSize, s.VolumeType, s.IBM.VolumeIOPS, s.VolumeCount, s.RAID0, false,
 		)
-		workloadProviderOpts = getIBMOpts(workloadMachineType, s.TerminateOnMigration, s.VolumeSize,
-			s.IBM.VolumeType, s.IBM.VolumeIOPS, s.IBM.VolumeCount, s.RAID0, true,
+		workloadProviderOpts = getIBMOpts(workloadMachineType, s.TerminateOnMigration,
+			s.VolumeSize, s.VolumeType, s.IBM.VolumeIOPS, s.VolumeCount, s.RAID0, true,
 		)
 	}
 

pkg/cmd/roachtest/spec/cluster_spec_test.go

@@ -25,15 +25,22 @@ func TestClustersCompatible(t *testing.T) {
 	t.Run("spec has different GCE spec with cloud as GCE", func(t *testing.T) {
 		s1 := ClusterSpec{NodeCount: 5}
 		s2 := ClusterSpec{NodeCount: 5}
-		s1.GCE.VolumeType = "mock_volume1"
-		s2.GCE.VolumeType = "mock_volume2"
+		s1.VolumeType = "mock_volume1"
+		s2.VolumeType = "mock_volume2"
 		require.False(t, ClustersCompatible(s1, s2, GCE))
 	})
 	t.Run("spec has different GCE spec with cloud as AWS", func(t *testing.T) {
 		s1 := ClusterSpec{NodeCount: 5}
 		s2 := ClusterSpec{NodeCount: 5}
-		s1.GCE.VolumeType = "mock_volume1"
-		s2.GCE.VolumeType = "mock_volume2"
+		s1.VolumeType = "mock_volume1"
+		s2.VolumeType = "mock_volume2"
+		require.False(t, ClustersCompatible(s1, s2, AWS))
+	})
+	t.Run("spec has different spec with cloud as AWS", func(t *testing.T) {
+		s1 := ClusterSpec{NodeCount: 5}
+		s2 := ClusterSpec{NodeCount: 5}
+		s1.GCE.MinCPUPlatform = "mock_platform1"
+		s2.GCE.MinCPUPlatform = "mock_platform2"
 		require.True(t, ClustersCompatible(s1, s2, AWS))
 	})
 }