STOR-2893: add storage BYOK feature tests

pkg/clioptions/clusterdiscovery/cluster.go

@@ -348,6 +348,7 @@ func LoadConfig(state *ClusterState) (*ClusterConfiguration, error) {
 		}
 		config.ConfigFile = tmpFile.Name()
 	case state.PlatformStatus.IBMCloud != nil:
+		config.ProviderName = "ibmcloud"
 		// Determine if Managed IBM Cloud cluster (ROKS)
 		if *state.ControlPlaneTopology == configv1.ExternalTopologyMode {
 			config.IsIBMROKS = true

pkg/clioptions/clusterdiscovery/provider.go

@@ -29,7 +29,10 @@ import (
 	// Initialize nutanix as a provider
 	_ "github.com/openshift/origin/test/extended/util/nutanix"
 
-	// Initialize nutanix as a provider
+	// Initialize ibmcloud as a provider
+	_ "github.com/openshift/origin/test/extended/util/ibmcloud"
+
+	// Initialize external as a provider
 	_ "github.com/openshift/origin/test/extended/util/external"
 
 	// these are loading important global flags that we need to get and set
@@ -144,7 +147,7 @@ func DecodeProvider(providerTypeOrJSON string, dryRun, discover bool, clusterSta
 		}
 		fallthrough
 
-	case "azure", "aws", "baremetal", "gce", "vsphere", "alibabacloud", "external":
+	case "azure", "aws", "baremetal", "gce", "vsphere", "alibabacloud", "ibmcloud", "external":
 		if clusterState == nil {
 			clientConfig, err := e2e.LoadConfig(true)
 			if err != nil {

test/extended/storage/const.go

@@ -0,0 +1,187 @@
+package storage
+
+// ProvisionerType represents the type of CSI provisioner
+type ProvisionerType string
+
+const (
+	// ProvisionerTypeBlock represents block storage provisioners (AWS EBS,Azure Disk, etc.)
+	ProvisionerTypeBlock ProvisionerType = "block"
+	// ProvisionerTypeFile represents file storage provisioners (AWS EFS, Azure File, etc.)
+	ProvisionerTypeFile ProvisionerType = "file"
+)
+
+// FeatureSupport defines which features a provisioner supports
+type FeatureSupport struct {
+	SupportsBYOK bool
+}
+
+// ProvisionerInfo contains information about a CSI provisioner
+type ProvisionerInfo struct {
+	Name                     string
+	Type                     ProvisionerType
+	Features                 FeatureSupport
+	EncryptionKeyName        string   // The parameter name for encryption key in StorageClass
+	ManagedStorageClassNames []string // List of managed/preset storage classes name
+}
+
+// PlatformConfig contains configuration for a cloud platform
+type PlatformConfig struct {
+	Provisioners        []ProvisionerInfo
+	DefaultStorageClass string
+}
+
+// Platforms maps cloud platforms to their configuration
+var Platforms = map[string]PlatformConfig{
+	"aws": {
+		Provisioners: []ProvisionerInfo{
+			{
+				Name: "ebs.csi.aws.com",
+				Type: ProvisionerTypeBlock,
+				Features: FeatureSupport{
+					SupportsBYOK: true,
+				},
+				EncryptionKeyName:        "kmsKeyId",
+				ManagedStorageClassNames: []string{"gp2-csi", "gp3-csi"},
+			},
+			{
+				Name:                     "efs.csi.aws.com",
+				Type:                     ProvisionerTypeFile,
+				Features:                 FeatureSupport{},
+				EncryptionKeyName:        "",
+				ManagedStorageClassNames: []string{"efs-sc"},
+			},
+		},
+		DefaultStorageClass: "gp3-csi",
+	},
+	"gce": {
+		Provisioners: []ProvisionerInfo{
+			{
+				Name: "pd.csi.storage.gke.io",
+				Type: ProvisionerTypeBlock,
+				Features: FeatureSupport{
+					SupportsBYOK: true,
+				},
+				EncryptionKeyName:        "disk-encryption-kms-key",
+				ManagedStorageClassNames: []string{"standard-csi", "ssd-csi"},
+			},
+			{
+				Name:                     "filestore.csi.storage.gke.io",
+				Type:                     ProvisionerTypeFile,
+				Features:                 FeatureSupport{},
+				EncryptionKeyName:        "",
+				ManagedStorageClassNames: []string{"filestore-csi"},
+			},
+		},
+		DefaultStorageClass: "standard-csi",
+	},
+	"azure": {
+		Provisioners: []ProvisionerInfo{
+			{
+				Name: "disk.csi.azure.com",
+				Type: ProvisionerTypeBlock,
+				Features: FeatureSupport{
+					SupportsBYOK: true,
+				},
+				EncryptionKeyName:        "diskEncryptionSetID",
+				ManagedStorageClassNames: []string{"managed-csi"},
+			},
+			{
+				Name:                     "file.csi.azure.com",
+				Type:                     ProvisionerTypeFile,
+				Features:                 FeatureSupport{},
+				EncryptionKeyName:        "",
+				ManagedStorageClassNames: []string{"azurefile-csi"},
+			},
+		},
+		DefaultStorageClass: "managed-csi",
+	},
+	"ibmcloud": {
+		Provisioners: []ProvisionerInfo{
+			{
+				Name: "vpc.block.csi.ibm.io",
+				Type: ProvisionerTypeBlock,
+				Features: FeatureSupport{
+					SupportsBYOK: true,
+				},
+				EncryptionKeyName: "encryptionKey",
+				ManagedStorageClassNames: []string{
+					"ibmc-vpc-block-10iops-tier",
+					"ibmc-vpc-block-5iops-tier",
+					"ibmc-vpc-block-custom"},
+			},
+		},
+		DefaultStorageClass: "ibmc-vpc-block-10iops-tier",
+	},
+}
+
+// GetProvisionersByPlatform returns all provisioners for a given platform
+func GetProvisionersByPlatform(platform string) []ProvisionerInfo {
+	if config, ok := Platforms[platform]; ok {
+		return config.Provisioners
+	}
+	return nil
+}
+
+// GetProvisionerByName finds a provisioner by its name across all platforms
+func GetProvisionerByName(provisioner string) *ProvisionerInfo {
+	for _, config := range Platforms {
+		for _, p := range config.Provisioners {
+			if p.Name == provisioner {
+				return &p
+			}
+		}
+	}
+	return nil
+}
+
+// GetBYOKProvisioners returns provisioners that support BYOK for a given platform
+func GetBYOKProvisioners(platform string) []ProvisionerInfo {
+	var result []ProvisionerInfo
+	provisioners := GetProvisionersByPlatform(platform)
+	for _, prov := range provisioners {
+		if prov.Features.SupportsBYOK {
+			result = append(result, prov)
+		}
+	}
+	return result
+}
+
+// GetProvisionersByType returns provisioners of a specific type for a given platform
+func GetProvisionersByType(platform string, provType ProvisionerType) []ProvisionerInfo {
+	var result []ProvisionerInfo
+	provisioners := GetProvisionersByPlatform(platform)
+	for _, prov := range provisioners {
+		if prov.Type == provType {
+			result = append(result, prov)
+		}
+	}
+	return result
+}
+
+// GetProvisionerNames returns just the names of provisioners for a given platform
+func GetProvisionerNames(platform string) []string {
+	provisioners := GetProvisionersByPlatform(platform)
+	names := make([]string, len(provisioners))
+	for i, p := range provisioners {
+		names[i] = p.Name
+	}
+	return names
+}
+
+// GetBYOKProvisionerNames returns names of provisioners that support BYOK for a given platform
+func GetBYOKProvisionerNames(platform string) []string {
+	byokProvisioners := GetBYOKProvisioners(platform)
+	names := make([]string, len(byokProvisioners))
+	for i, p := range byokProvisioners {
+		names[i] = p.Name
+	}
+	return names
+}
+
+// GetDefaultStorageClass returns the default storage class for a given platform
+func GetDefaultStorageClass(platform string) string {
+	if config, ok := Platforms[platform]; ok {
+		return config.DefaultStorageClass
+	}
+	return ""
+}

test/extended/storage/csi_byok.go

@@ -0,0 +1,139 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+
+	g "github.com/onsi/ginkgo/v2"
+	o "github.com/onsi/gomega"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+	admissionapi "k8s.io/pod-security-admission/api"
+
+	exutil "github.com/openshift/origin/test/extended/util"
+)
+
+var _ = g.Describe(`[sig-storage][Feature:BYOK][Jira:"Storage"]`, func() {
+	defer g.GinkgoRecover()
+
+	var (
+		oc                                 = exutil.NewCLIWithPodSecurityLevel("csi-byok", admissionapi.LevelPrivileged)
+		cloudProvider                      = ""
+		featureProviderSupportProvisioners = []string{}
+	)
+
+	g.BeforeEach(func() {
+		// Detect cloud provider and set supported provisioners
+		cloudProvider = e2e.TestContext.Provider
+		featureProviderSupportProvisioners = GetBYOKProvisionerNames(cloudProvider)
+
+		if len(featureProviderSupportProvisioners) == 0 {
+			g.Skip("Skip for scenario non-supported provisioner!!!")
+		}
+
+	})
+
+	g.It("managed storage classes should be set with the specified encryption key", func() {
+
+		for _, provisioner := range featureProviderSupportProvisioners {
+
+			// Get provisioner info from const
+			provisionerInfo := GetProvisionerByName(provisioner)
+			if provisionerInfo == nil {
+				g.Skip("Provisioner not found in configuration: " + provisioner)
+			}
+
+			// Skipped for test clusters not installed with the BYOK
+			byokKeyID := getByokKeyIDFromClusterCSIDriver(oc, provisioner)
+			if len(byokKeyID) == 0 {
+				g.Skip("Skipped: the cluster is not byok cluster, no key settings in clustercsidriver/" + provisioner)
+			}
+
+			g.By("Get preset storageClass names from configuration")
+			presetStorageClassNames := getPresetStorageClassNamesByProvisioner(provisioner)
+			if len(presetStorageClassNames) == 0 {
+				g.Skip(fmt.Sprintf("No preset storage classes configured for provisioner %s. Expected one of: %v",
+					provisioner, provisionerInfo.ManagedStorageClassNames))
+			}
+
+			g.By("Verify all preset storage classes have encryption key configured")
+			for _, scName := range presetStorageClassNames {
+				g.By(fmt.Sprintf("Verifying storage class: %s", scName))
+
+				sc, err := oc.AdminKubeClient().StorageV1().StorageClasses().Get(context.Background(), scName, metav1.GetOptions{})
+				if err != nil {
+					g.Skip(fmt.Sprintf("Storage class %s not found in cluster: %v", scName, err))
+				}
+
+				// Double check the storage class is properly configured
+				o.Expect(sc.Provisioner).Should(o.Equal(provisioner),
+					fmt.Sprintf("Storage class %s provisioner mismatch", sc.Name))
+
+				// Verify BYOK key parameter exists
+				storedKeyID, exists := sc.Parameters[provisionerInfo.EncryptionKeyName]
+				if !exists {
+					g.Fail(fmt.Sprintf("Storage class %s does not have BYOK key parameter %s configured",
+						sc.Name, provisionerInfo.EncryptionKeyName))
+				}
+				o.Expect(storedKeyID).Should(o.Equal(byokKeyID),
+					fmt.Sprintf("Storage class %s has different BYOK key than ClusterCSIDriver", sc.Name))
+			}
+		}
+	})
+})
+
+func getByokKeyIDFromClusterCSIDriver(oc *exutil.CLI, provisioner string) string {
+	clusterCSIDriver, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Get(context.Background(), provisioner, metav1.GetOptions{})
+	if err != nil {
+		e2e.Logf("Failed to get ClusterCSIDriver %s: %v", provisioner, err)
+		return ""
+	}
+
+	driverConfig := clusterCSIDriver.Spec.DriverConfig
+	if driverConfig.DriverType == "" {
+		return ""
+	}
+
+	// Extract key ID based on driver type using struct fields
+	switch driverConfig.DriverType {
+	case "AWS":
+		if driverConfig.AWS != nil {
+			return driverConfig.AWS.KMSKeyARN
+		}
+	case "Azure":
+		if driverConfig.Azure != nil && driverConfig.Azure.DiskEncryptionSet != nil {
+			// Build the full disk encryption set ID
+			des := driverConfig.Azure.DiskEncryptionSet
+			return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/diskEncryptionSets/%s",
+				des.SubscriptionID, des.ResourceGroup, des.Name)
+		}
+	case "GCP":
+		if driverConfig.GCP != nil && driverConfig.GCP.KMSKey != nil {
+			// For GCP, return the full KMS key reference or just the key ring based on what's needed
+			kmsKey := driverConfig.GCP.KMSKey
+			location := kmsKey.Location
+			if location == "" {
+				location = "global"
+			}
+			return fmt.Sprintf("projects/%s/locations/%s/keyRings/%s/cryptoKeys/%s",
+				kmsKey.ProjectID, location, kmsKey.KeyRing, kmsKey.Name)
+		}
+	case "IBMCloud":
+		if driverConfig.IBMCloud != nil {
+			return driverConfig.IBMCloud.EncryptionKeyCRN
+		}
+	}
+
+	return ""
+}
+
+func getPresetStorageClassNamesByProvisioner(provisioner string) []string {
+	// Get provisioner info to get managed storage class names from static config
+	provisionerInfo := GetProvisionerByName(provisioner)
+	if provisionerInfo == nil {
+		e2e.Logf("Provisioner not found in configuration: %s", provisioner)
+		return []string{}
+	}
+	return provisionerInfo.ManagedStorageClassNames
+}