test(e2e+machine): Base structure for machine mode testing within E2E for oss

Makefile

@@ -51,7 +51,7 @@ e2etests: ## Run the e2e suite against your local cluster
 	# -count 1: prevents caching
 	# -timeout: If a test binary runs longer than TEST_TIMEOUT, panic
 	# -v: verbose output
-	cd test && AZURE_CLUSTER_NAME=${AZURE_CLUSTER_NAME} AZURE_ACR_NAME=${AZURE_ACR_NAME} AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP} AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID} AZURE_LOCATION=${AZURE_LOCATION} go test \
+	cd test && AZURE_CLUSTER_NAME=${AZURE_CLUSTER_NAME} AZURE_ACR_NAME=${AZURE_ACR_NAME} AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP} AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID} AZURE_LOCATION=${AZURE_LOCATION} PROVISION_MODE=${PROVISION_MODE} go test \
 		-p 1 \
 		-count 1 \
 		-timeout ${TEST_TIMEOUT} \

Makefile-az.mk

@@ -139,7 +139,7 @@ az-rmrg: ## Destroy test ACR and AKS cluster by deleting the resource group (use
 	az group delete --name $(AZURE_RESOURCE_GROUP)
 
 az-configure-values:  ## Generate cluster-related values for Karpenter Helm chart
-	hack/deploy/configure-values.sh $(AZURE_CLUSTER_NAME) $(AZURE_RESOURCE_GROUP) $(KARPENTER_SERVICE_ACCOUNT_NAME) $(AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME)
+	hack/deploy/configure-values.sh $(AZURE_CLUSTER_NAME) $(AZURE_RESOURCE_GROUP) $(KARPENTER_SERVICE_ACCOUNT_NAME) $(AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME) $(PROVISION_MODE)
 
 az-mkvmssflex: ## Create VMSS Flex (optional, only if creating VMs referencing this VMSS)
 	az vmss create --name $(AZURE_CLUSTER_NAME)-vmss --resource-group $(AZURE_RESOURCE_GROUP_MC) --location $(AZURE_LOCATION) \

pkg/consts/consts.go

@@ -39,4 +39,5 @@ const (
 
 	ProvisionModeAKSScriptless       = "aksscriptless"
 	ProvisionModeBootstrappingClient = "bootstrappingclient"
+	ProvisionModeAKSMachineAPI       = "aksmachineapi"
 )

test/pkg/environment/azure/environment.go

@@ -37,6 +37,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 	"github.com/Azure/karpenter-provider-azure/pkg/apis/v1beta1"
+	"github.com/Azure/karpenter-provider-azure/pkg/consts"
 	"github.com/Azure/karpenter-provider-azure/pkg/test"
 	"github.com/Azure/karpenter-provider-azure/pkg/test/azure"
 	"github.com/Azure/karpenter-provider-azure/test/pkg/environment/common"
@@ -61,7 +62,9 @@ type Environment struct {
 	VNETResourceGroup    string
 	ACRName              string
 	ClusterName          string
+	MachineAgentPoolName string
 	ClusterResourceGroup string
+	ProvisionMode        string
 
 	tracker *azure.Tracker
 
@@ -85,24 +88,30 @@ type Environment struct {
 	RBACManager *RBACManager
 }
 
-func readEnv(name string) string {
+func readEnv(name string, required bool) string {
 	value, exists := os.LookupEnv(name)
 	if !exists {
-		panic(fmt.Sprintf("Environment variable %s is not set", name))
+		if required {
+			panic(fmt.Sprintf("Environment variable %s is not set", name))
+		}
+		return ""
 	}
 	if value == "" {
-		panic(fmt.Sprintf("Environment variable %s is set to an empty string", name))
+		if required {
+			panic(fmt.Sprintf("Environment variable %s is set to an empty string", name))
+		}
 	}
 	return value
 }
 
 func NewEnvironment(t *testing.T) *Environment {
 	azureEnv := &Environment{
 		Environment:          common.NewEnvironment(t),
-		SubscriptionID:       readEnv("AZURE_SUBSCRIPTION_ID"),
-		ClusterName:          readEnv("AZURE_CLUSTER_NAME"),
-		ClusterResourceGroup: readEnv("AZURE_RESOURCE_GROUP"),
-		ACRName:              readEnv("AZURE_ACR_NAME"),
+		SubscriptionID:       readEnv("AZURE_SUBSCRIPTION_ID", true),
+		ClusterName:          readEnv("AZURE_CLUSTER_NAME", true),
+		ClusterResourceGroup: readEnv("AZURE_RESOURCE_GROUP", true),
+		ACRName:              readEnv("AZURE_ACR_NAME", true),
+		ProvisionMode:        readEnv("PROVISION_MODE", false),
 		Region:               lo.Ternary(os.Getenv("AZURE_LOCATION") == "", "westus2", os.Getenv("AZURE_LOCATION")),
 		tracker:              azure.NewTracker(),
 	}
@@ -124,6 +133,17 @@ func NewEnvironment(t *testing.T) *Environment {
 	azureEnv.KeyVaultClient = lo.Must(armkeyvault.NewVaultsClient(azureEnv.SubscriptionID, cred, byokRetryOptions))
 	azureEnv.DiskEncryptionSetClient = lo.Must(armcompute.NewDiskEncryptionSetsClient(azureEnv.SubscriptionID, cred, byokRetryOptions))
 	azureEnv.RBACManager = lo.Must(NewRBACManager(azureEnv.SubscriptionID, cred))
+	// Default to reserved managed machine agentpool name for NAP
+	azureEnv.MachineAgentPoolName = "aksmanagedap"
+	if azureEnv.Environment.InClusterController {
+		azureEnv.MachineAgentPoolName = "karp-e2e-byo-machine-ap"
+	}
+	// Create our BYO testing Machine Pool, if running self-hosted, with machine mode specified
+	// > Note: this only has to occur once per test, since its just a container for the machines
+	// > meaning that there is no risk of the tests modifying the Machine Pool itself.
+	if azureEnv.InClusterController && azureEnv.ProvisionMode == consts.ProvisionModeAKSMachineAPI {
+		azureEnv.ExpectRunInClusterControllerWithMachineMode()
+	}
 	return azureEnv
 }
 

test/pkg/environment/azure/expectations.go

@@ -25,6 +25,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/samber/lo"
+	v1 "k8s.io/api/core/v1"
 	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
@@ -34,6 +35,23 @@ import (
 	"github.com/Azure/karpenter-provider-azure/pkg/providers/instance"
 )
 
+// ExpectRunInClusterControllerWithMachineMode should only need to be called once, and only for InClusterController mode
+//
+// This is because:
+// - in running in NAP mode, the Machines AP will be created for us
+// - machine agentpool is just a container, so no risk/concern of tests modifying the AP.
+func (env *Environment) ExpectRunInClusterControllerWithMachineMode() containerservice.AgentPool {
+	Expect(env.InClusterController).To(BeTrue(), "Should only create a byo Machine Pool when running as an InClusterController")
+	By("Setup BYO Machine AgentPool for self-hosted testing")
+	byoMachineAP := env.ExpectCreatedMachineAgentPool()
+	env.ExpectSettingsOverridden([]v1.EnvVar{
+		{Name: "PROVISION_MODE", Value: "aksmachineapi"},
+		{Name: "MANAGE_EXISTING_AKS_MACHINES", Value: "true"},
+		{Name: "AKS_MACHINES_POOL_NAME", Value: *byoMachineAP.Name},
+	}...)
+	return byoMachineAP
+}
+
 func (env *Environment) EventuallyExpectKarpenterNicsToBeDeleted() {
 	GinkgoHelper()
 	Eventually(func() bool {

test/pkg/environment/azure/expectations_agentpools.go

@@ -0,0 +1,39 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/samber/lo"
+
+	containerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v7"
+)
+
+func (env *Environment) ExpectCreatedMachineAgentPool() containerservice.AgentPool {
+	GinkgoHelper()
+	byoTestMachineAP := containerservice.AgentPool{
+		Properties: &containerservice.ManagedClusterAgentPoolProfileProperties{
+			Mode: lo.ToPtr(containerservice.AgentPoolModeMachines),
+		},
+	}
+	poller, err := env.agentpoolsClient.BeginCreateOrUpdate(env.Context, env.ClusterResourceGroup, env.ClusterName, env.MachineAgentPoolName, byoTestMachineAP, nil)
+	Expect(err).ToNot(HaveOccurred())
+	res, err := poller.PollUntilDone(env.Context, nil)
+	Expect(err).ToNot(HaveOccurred())
+	return res.AgentPool
+}

test/pkg/environment/azure/expectations_machines.go

@@ -0,0 +1,43 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	containerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v7"
+)
+
+func (env *Environment) ExpectListMachines() []*containerservice.Machine {
+	GinkgoHelper()
+	var machines []*containerservice.Machine
+	pager := env.machinesClient.NewListPager(env.ClusterResourceGroup, env.ClusterName, env.MachineAgentPoolName, nil)
+	Expect(pager).ToNot(BeNil())
+	for pager.More() {
+		page, err := pager.NextPage(env.Context)
+		Expect(err).ToNot(HaveOccurred())
+		machines = append(machines, page.Value...)
+	}
+
+	return machines
+}
+
+func (env *Environment) ExpectNoMachines() {
+	GinkgoHelper()
+	Expect(len(env.ExpectListMachines())).To(Equal(0))
+}

test/pkg/environment/azure/setup.go

@@ -46,6 +46,9 @@ func (env *Environment) BeforeEach() {
 func (env *Environment) Cleanup() {
 	env.Environment.Cleanup()
 	env.Environment.CleanupObjects(CleanableObjects...)
+	// > Note: under current usage no machines should exist here,
+	// > as scaledown should ensure the machines are deleted
+	env.ExpectNoMachines()
 
 	err := env.tracker.Cleanup()
 	Expect(err).ToNot(HaveOccurred(), "Failed to clean up Azure resources")

test/suites/consolidation/suite_test.go

@@ -44,6 +44,7 @@ import (
 )
 
 var env *azure.Environment
+var nodeClass *v1beta1.AKSNodeClass
 
 func TestConsolidation(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -56,8 +57,6 @@ func TestConsolidation(t *testing.T) {
 	RunSpecs(t, "Consolidation")
 }
 
-var nodeClass *v1beta1.AKSNodeClass
-
 var _ = BeforeEach(func() {
 	nodeClass = env.DefaultAKSNodeClass()
 	env.BeforeEach()

test/suites/inplaceupdate/suite_test.go

@@ -32,6 +32,7 @@ import (
 	coretest "sigs.k8s.io/karpenter/pkg/test"
 
 	"github.com/Azure/karpenter-provider-azure/pkg/apis/v1beta1"
+	"github.com/Azure/karpenter-provider-azure/pkg/consts"
 	"github.com/Azure/karpenter-provider-azure/test/pkg/environment/azure"
 )
 
@@ -62,6 +63,12 @@ var _ = Describe("Inplace Update", func() {
 	var dep *appsv1.Deployment
 	var selector labels.Selector
 
+	BeforeEach(func() {
+		if env.ProvisionMode == consts.ProvisionModeAKSMachineAPI {
+			Skip("Skipping in Machine mode until expectatin functions are updated")
+		}
+	})
+
 	Context("Tags", func() {
 		It("should add tags in-place on all resources without drifting the nodeClaim", func() {
 			var numPods int32 = 3