diff --git a/internal/controller/concurrency_subnet_patch_test.go b/internal/controller/concurrency_subnet_patch_test.go index e69e24c..d6f6a95 100644 --- a/internal/controller/concurrency_subnet_patch_test.go +++ b/internal/controller/concurrency_subnet_patch_test.go @@ -28,7 +28,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -44,8 +43,8 @@ func TestSubnetCreateDeleteConcurrency(t *testing.T) { t.Skip("Skipping in short mode") } - // Setup test environment - testEnv := setupEnvTest(t) + // Setup test environment with private scheme + testEnv, testScheme := setupEnvTest(t) defer stopEnvTest(t, testEnv) cfg, err := testEnv.Start() @@ -53,8 +52,8 @@ func TestSubnetCreateDeleteConcurrency(t *testing.T) { t.Fatalf("Failed to start test env: %v", err) } - // Create client - k8sClient, err := createK8sClient(cfg) + // Create client with our private scheme + k8sClient, err := createK8sClient(cfg, testScheme) if err != nil { t.Fatalf("Failed to create k8s client: %v", err) } @@ -95,10 +94,10 @@ func TestSubnetCreateDeleteConcurrency(t *testing.T) { t.Fatalf("Failed to create pool: %v", err) } - // Setup manager + // Setup manager with our private scheme ctrl.SetLogger(zap.New(zap.UseDevMode(true))) mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, + Scheme: testScheme, Metrics: metricsserver.Options{ BindAddress: "0", // Disable metrics server }, diff --git a/internal/controller/envtest_helpers_race_test.go b/internal/controller/envtest_helpers_race_test.go index f32f26a..bab06b6 100644 --- a/internal/controller/envtest_helpers_race_test.go +++ b/internal/controller/envtest_helpers_race_test.go @@ -7,22 +7,25 @@ import ( "context" "testing" - "k8s.io/client-go/kubernetes/scheme" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - - ipamv1 "github.com/appthrust/plexaubnet/api/v1alpha1" ) // setupEnvTest sets up a controller-runtime envtest environment and registers the CRDs. // This duplicate implementation is compiled only when `-race` is enabled (build tag `race`). -func setupEnvTest(t *testing.T) *envtest.Environment { +func setupEnvTest(t *testing.T) (*envtest.Environment, *runtime.Scheme) { t.Helper() + + // Use the common helper function to create a private scheme + testScheme := NewTestScheme() + testEnv := &envtest.Environment{ CRDDirectoryPaths: []string{"../../config/crd/bases"}, + Scheme: testScheme, // Use our private scheme } cfg, err := testEnv.Start() @@ -34,11 +37,10 @@ func setupEnvTest(t *testing.T) *envtest.Environment { cfg.QPS = 200 // uplift default 5 cfg.Burst = 400 // uplift default 10 - if err := ipamv1.AddToScheme(scheme.Scheme); err != nil { - t.Fatalf("Error adding scheme: %v", err) - } + // Note: we don't need to call ipamv1.AddToScheme(scheme.Scheme) anymore + // since the types are already registered in our private scheme - return testEnv + return testEnv, testScheme } // stopEnvTest terminates the envtest environment. @@ -50,14 +52,14 @@ func stopEnvTest(t *testing.T, testEnv *envtest.Environment) { } // createK8sClient constructs a controller-runtime client using the provided rest.Config. -func createK8sClient(cfg *rest.Config) (client.Client, error) { - return client.New(cfg, client.Options{Scheme: scheme.Scheme}) +func createK8sClient(cfg *rest.Config, s *runtime.Scheme) (client.Client, error) { + return client.New(cfg, client.Options{Scheme: s}) } // setupManager creates a controller-runtime manager for tests with metrics & probes disabled. -func setupManager(cfg *rest.Config) (ctrl.Manager, error) { +func setupManager(cfg *rest.Config, s *runtime.Scheme) (ctrl.Manager, error) { mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, + Scheme: s, // Use the provided private scheme Metrics: metricsserver.Options{ BindAddress: "0", // Disable metrics server }, diff --git a/internal/controller/high_load_test.go b/internal/controller/high_load_test.go index 8429a83..d9c7693 100644 --- a/internal/controller/high_load_test.go +++ b/internal/controller/high_load_test.go @@ -30,8 +30,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -43,9 +43,13 @@ import ( ) // Helper function to set up the test environment -func setupEnvTest(t *testing.T) *envtest.Environment { +func setupEnvTest(t *testing.T) (*envtest.Environment, *runtime.Scheme) { + // Use helper to create private scheme with all required types + testScheme := NewTestScheme() + testEnv := &envtest.Environment{ CRDDirectoryPaths: []string{"../../config/crd/bases"}, + Scheme: testScheme, // Use our private scheme } cfg, err := testEnv.Start() @@ -57,12 +61,10 @@ func setupEnvTest(t *testing.T) *envtest.Environment { cfg.QPS = 200 // Significantly increased from default 5 cfg.Burst = 400 // Significantly increased from default 10 - err = ipamv1.AddToScheme(scheme.Scheme) - if err != nil { - t.Fatalf("Error adding scheme: %v", err) - } + // Note: we don't need to call ipamv1.AddToScheme(scheme.Scheme) anymore + // since the types are already registered in our private scheme - return testEnv + return testEnv, testScheme } // Helper function to stop the test environment @@ -73,16 +75,16 @@ func stopEnvTest(t *testing.T, testEnv *envtest.Environment) { } // Helper function to create a k8s client -func createK8sClient(cfg *rest.Config) (client.Client, error) { - // Use existing Scheme - return client.New(cfg, client.Options{Scheme: scheme.Scheme}) +func createK8sClient(cfg *rest.Config, s *runtime.Scheme) (client.Client, error) { + // Use the provided private scheme + return client.New(cfg, client.Options{Scheme: s}) } // Helper function to set up the manager -func setupManager(cfg *rest.Config) (ctrl.Manager, error) { +func setupManager(cfg *rest.Config, s *runtime.Scheme) (ctrl.Manager, error) { mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, + Scheme: s, // Use the provided private scheme Metrics: metricsserver.Options{ BindAddress: "0", // Disable metrics server }, @@ -112,8 +114,8 @@ func TestHighLoad1000SubnetClaimCreate(t *testing.T) { initialGoroutines := goruntime.NumGoroutine() t.Logf("Initial goroutines: %d", initialGoroutines) - // Setup test environment - testEnv := setupEnvTest(t) + // Setup test environment with private scheme + testEnv, testScheme := setupEnvTest(t) defer stopEnvTest(t, testEnv) // Get test environment configuration @@ -122,8 +124,8 @@ func TestHighLoad1000SubnetClaimCreate(t *testing.T) { t.Fatalf("Failed to start test env: %v", err) } - // Create client - k8sClient, err := createK8sClient(cfg) + // Create client using our private scheme + k8sClient, err := createK8sClient(cfg, testScheme) if err != nil { t.Fatalf("Failed to create k8s client: %v", err) } @@ -167,9 +169,9 @@ func TestHighLoad1000SubnetClaimCreate(t *testing.T) { startHeap := &goruntime.MemStats{} goruntime.ReadMemStats(startHeap) - // Start controller + // Start controller with our private scheme ctrl.SetLogger(zap.New(zap.UseDevMode(true))) - mgr, err := setupManager(cfg) + mgr, err := setupManager(cfg, testScheme) if err != nil { t.Fatalf("Failed to setup manager: %v", err) } @@ -379,8 +381,8 @@ func TestParentPoolContentionWithClaims(t *testing.T) { t.Skip("Skipping parent pool contention test in short mode") } - // Setup test environment - testEnv := setupEnvTest(t) + // Setup test environment with private scheme + testEnv, testScheme := setupEnvTest(t) defer stopEnvTest(t, testEnv) // Get test environment configuration @@ -389,8 +391,8 @@ func TestParentPoolContentionWithClaims(t *testing.T) { t.Fatalf("Failed to start test env: %v", err) } - // Create client - k8sClient, err := createK8sClient(cfg) + // Create client using our private scheme + k8sClient, err := createK8sClient(cfg, testScheme) if err != nil { t.Fatalf("Failed to create k8s client: %v", err) } @@ -438,9 +440,9 @@ func TestParentPoolContentionWithClaims(t *testing.T) { t.Fatalf("Failed to create pool: %v", err) } - // Start controller + // Start controller with our private scheme ctrl.SetLogger(zap.New(zap.UseDevMode(true))) - mgr, err := setupManager(cfg) + mgr, err := setupManager(cfg, testScheme) if err != nil { t.Fatalf("Failed to setup manager: %v", err) } diff --git a/internal/controller/parent_pool_integration_test.go b/internal/controller/parent_pool_integration_test.go index 09448dc..462160c 100644 --- a/internal/controller/parent_pool_integration_test.go +++ b/internal/controller/parent_pool_integration_test.go @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" @@ -92,9 +91,13 @@ func TestMain(m *testing.M) { // Start envtest environment fmt.Println("Parent Pool Integration Test: Setting up envtest environment...") + // Use common helper to create private scheme with all required types + testScheme := NewTestScheme() + parentPoolTestEnv = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, + Scheme: testScheme, // Use our private scheme } var err error @@ -109,23 +112,19 @@ func TestMain(m *testing.M) { parentPoolRestConfig.QPS = 100 parentPoolRestConfig.Burst = 200 - // Add IPAM CRD to schema - err = ipamv1.AddToScheme(scheme.Scheme) - if err != nil { - fmt.Printf("Parent Pool Integration Test: Schema registration failed: %v\n", err) - os.Exit(1) - } + // Note: we don't need to call ipamv1.AddToScheme(scheme.Scheme) anymore + // since the types are already registered in our private scheme - // Create client - parentPoolClient, err = client.New(parentPoolRestConfig, client.Options{Scheme: scheme.Scheme}) + // Create client with our private scheme + parentPoolClient, err = client.New(parentPoolRestConfig, client.Options{Scheme: testScheme}) if err != nil { fmt.Printf("Parent Pool Integration Test: Client creation failed: %v\n", err) os.Exit(1) } - // Setup controller manager + // Setup controller manager with our private scheme mgr, err := ctrl.NewManager(parentPoolRestConfig, ctrl.Options{ - Scheme: scheme.Scheme, + Scheme: testScheme, // Explicitly disable metrics server to avoid port conflicts Metrics: metricsserver.Options{ BindAddress: "0", // Let OS assign a free port diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index bda73ca..809cda9 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -25,7 +25,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -73,9 +72,15 @@ var _ = BeforeSuite(func() { Expect(envErr).NotTo(HaveOccurred(), "Failed to set KUBEBUILDER_KUBE_APISERVER_FLAGS") By("bootstrapping test environment") + + // Create a private scheme for this test suite to avoid race conditions + // with other test suites trying to register types to the global scheme + testScheme := NewTestScheme() + testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, + Scheme: testScheme, // Use our private scheme } var err error @@ -89,12 +94,12 @@ var _ = BeforeSuite(func() { cfg.Burst = 400 // Significantly increased from default 10 GinkgoWriter.Printf("Setting high QPS=%v, Burst=%v for parallel test\n", cfg.QPS, cfg.Burst) - err = ipamv1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) + // Note: we don't need to call ipamv1.AddToScheme(scheme.Scheme) anymore + // since the types are already registered in our private scheme via NewTestScheme() //+kubebuilder:scaffold:scheme - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + k8sClient, err = client.New(cfg, client.Options{Scheme: testScheme}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) @@ -104,9 +109,9 @@ var _ = BeforeSuite(func() { os.Setenv("KUBEBUILDER_CONTROLPLANE_START_PORT", "18200") mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, - HealthProbeBindAddress: "0", // Disable health check server - LeaderElection: false, // Also disable leader election + Scheme: testScheme, // Use our private scheme + HealthProbeBindAddress: "0", // Disable health check server + LeaderElection: false, // Also disable leader election Metrics: metricsserver.Options{ BindAddress: "0", // Disable metrics server }, diff --git a/internal/controller/test_helpers.go b/internal/controller/test_helpers.go new file mode 100644 index 0000000..abd1f31 --- /dev/null +++ b/internal/controller/test_helpers.go @@ -0,0 +1,36 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cidrallocator + +import ( + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + + ipamv1 "github.com/appthrust/plexaubnet/api/v1alpha1" +) + +// NewTestScheme creates a new scheme for testing purposes. +// This avoids race conditions where multiple test suites try to +// register types to the global scheme.Scheme concurrently. +func NewTestScheme() *runtime.Scheme { + s := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(s)) + utilruntime.Must(ipamv1.AddToScheme(s)) + // +kubebuilder:scaffold:scheme + return s +}