feat: flag to enable/disable middleware logging (#1031)

Co-authored-by: Alex Leites <[email protected]>

Author: alimaazamat

Makefile-az.mk

@@ -1,6 +1,7 @@
 AZURE_LOCATION ?= westus2
 AZURE_VM_SIZE ?= ""
 COMMON_NAME ?= karpenter
+ENABLE_AZURE_SDK_LOGGING ?= true
 ifeq ($(CODESPACES),true)
   AZURE_RESOURCE_GROUP ?= $(CODESPACE_NAME)
   AZURE_ACR_NAME ?= $(subst -,,$(CODESPACE_NAME))
@@ -138,8 +139,8 @@ az-mkaks-savm: az-mkrg ## Create experimental cluster with standalone VMs (+ ACR
 az-rmrg: ## Destroy test ACR and AKS cluster by deleting the resource group (use with care!)
 	az group delete --name $(AZURE_RESOURCE_GROUP)
 
-az-configure-values:  ## Generate cluster-related values for Karpenter Helm chart
-	hack/deploy/configure-values.sh $(AZURE_CLUSTER_NAME) $(AZURE_RESOURCE_GROUP) $(KARPENTER_SERVICE_ACCOUNT_NAME) $(AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME)
+az-configure-values:  ## Generate cluster-related values for Karpenter Helm chart and set middleware logging flag
+	hack/deploy/configure-values.sh $(AZURE_CLUSTER_NAME) $(AZURE_RESOURCE_GROUP) $(KARPENTER_SERVICE_ACCOUNT_NAME) $(AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME) $(ENABLE_AZURE_SDK_LOGGING)
 
 az-mkvmssflex: ## Create VMSS Flex (optional, only if creating VMs referencing this VMSS)
 	az vmss create --name $(AZURE_CLUSTER_NAME)-vmss --resource-group $(AZURE_RESOURCE_GROUP_MC) --location $(AZURE_LOCATION) \

hack/deploy/configure-values.sh

@@ -3,8 +3,8 @@ set -euo pipefail
 # This script interrogates the AKS cluster and Azure resources to generate
 # the karpenter-values.yaml file using the karpenter-values-template.yaml file as a template.
 
-if [ "$#" -ne 4 ]; then
-    echo "Usage: $0 <cluster-name> <resource-group> <karpenter-service-account-name> <karpenter-user-assigned-identity-name>"
+if [ "$#" -ne 5 ]; then
+    echo "Usage: $0 <cluster-name> <resource-group> <karpenter-service-account-name> <karpenter-user-assigned-identity-name> <enable-azure-sdk-logging>"
     exit 1
 fi
 
@@ -14,6 +14,7 @@ CLUSTER_NAME=$1
 AZURE_RESOURCE_GROUP=$2
 KARPENTER_SERVICE_ACCOUNT_NAME=$3
 AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME=$4
+ENABLE_AZURE_SDK_LOGGING=$5
 
 # Optional values through env vars:
 LOG_LEVEL=${LOG_LEVEL:-"info"}
@@ -70,7 +71,7 @@ KUBELET_IDENTITY_CLIENT_ID=$(jq -r ".identityProfile.kubeletidentity.clientId //
 
 export CLUSTER_NAME AZURE_LOCATION AZURE_RESOURCE_GROUP_MC KARPENTER_SERVICE_ACCOUNT_NAME \
     CLUSTER_ENDPOINT BOOTSTRAP_TOKEN SSH_PUBLIC_KEY VNET_SUBNET_ID KARPENTER_USER_ASSIGNED_CLIENT_ID NODE_IDENTITIES AZURE_SUBSCRIPTION_ID NETWORK_PLUGIN NETWORK_PLUGIN_MODE NETWORK_POLICY \
-    LOG_LEVEL VNET_GUID KUBELET_IDENTITY_CLIENT_ID
+    LOG_LEVEL VNET_GUID KUBELET_IDENTITY_CLIENT_ID ENABLE_AZURE_SDK_LOGGING
 
 # get karpenter-values-template.yaml, if not already present (e.g. outside of repo context)
 if [ ! -f karpenter-values-template.yaml ]; then

karpenter-values-template.yaml

@@ -1,4 +1,3 @@
-
 replicas: 1 # for better debugging experience
 controller:
   env:
@@ -39,6 +38,8 @@ controller:
       value: ${KUBELET_IDENTITY_CLIENT_ID}
     - name: AZURE_NODE_RESOURCE_GROUP
       value: ${AZURE_RESOURCE_GROUP_MC}
+    - name: ENABLE_AZURE_SDK_LOGGING
+      value: ${ENABLE_AZURE_SDK_LOGGING:-false}
 
     # managed karpenter settings
     - name: USE_SIG

pkg/operator/operator.go

@@ -108,7 +108,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 	azClient, err := instance.NewAZClient(ctx, azConfig, env, cred)
 	lo.Must0(err, "creating Azure client")
 	if options.FromContext(ctx).VnetGUID == "" && options.FromContext(ctx).NetworkPluginMode == consts.NetworkPluginModeOverlay {
-		vnetGUID, err := getVnetGUID(cred, azConfig, options.FromContext(ctx).SubnetID)
+		vnetGUID, err := getVnetGUID(ctx, cred, azConfig, options.FromContext(ctx).SubnetID)
 		lo.Must0(err, "getting VNET GUID")
 		options.FromContext(ctx).VnetGUID = vnetGUID
 	}
@@ -232,7 +232,7 @@ func getCABundle(restConfig *rest.Config) (*string, error) {
 	return lo.ToPtr(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)), nil
 }
 
-func getVnetGUID(creds azcore.TokenCredential, cfg *auth.Config, subnetID string) (string, error) {
+func getVnetGUID(ctx context.Context, creds azcore.TokenCredential, cfg *auth.Config, subnetID string) (string, error) {
 	// TODO: Current the VNET client isn't used anywhere but this method. As such, it is not
 	// held on azclient like the other clients.
 	// We should possibly just put the vnet client on azclient, and then pass azclient in here, rather than
@@ -242,7 +242,8 @@ func getVnetGUID(creds azcore.TokenCredential, cfg *auth.Config, subnetID string
 		return "", err
 	}
 
-	opts := armopts.DefaultARMOpts(env.Cloud)
+	o := options.FromContext(ctx)
+	opts := armopts.DefaultARMOpts(env.Cloud, o.EnableAzureSDKLogging)
 	vnetClient, err := armnetwork.NewVirtualNetworksClient(cfg.SubscriptionID, creds, opts)
 	if err != nil {
 		return "", err

pkg/operator/options/options.go

@@ -89,6 +89,7 @@ type Options struct {
 	SIGSubscriptionID          string            `json:"sigSubscriptionId,omitempty"`
 	NodeResourceGroup          string            `json:"nodeResourceGroup,omitempty"`
 	AdditionalTags             map[string]string `json:"additionalTags,omitempty"`
+	EnableAzureSDKLogging      bool              `json:"enableAzureSDKLogging,omitempty"` // Controls whether Azure SDK middleware logging is enabled
 	DiskEncryptionSetID        string            `json:"diskEncryptionSetId,omitempty"`
 }
 
@@ -121,6 +122,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
 	}
 	// See https://github.com/Azure/karpenter-provider-azure/issues/1042 for issue discussing improvements around this
 	fs.Var(additionalTagsFlag, "additional-tags", "Additional tags to apply to the resources in Azure. Format is key1=value1,key2=value2. These tags will be merged with the tags specified on the NodePool. In the case of a tag collision, the NodePool tag wins. These tags only apply to new nodes and do not trigger drift, which means that adding tags to this collection will not update existing nodes until drift triggers for some other reason.")
+	fs.BoolVar(&o.EnableAzureSDKLogging, "enable-azure-sdk-logging", env.WithDefaultBool("ENABLE_AZURE_SDK_LOGGING", true), "If set to false then Azure SDK middleware logging is disabled for debugging, and won't be logging all HTTP requests/responses to Azure APIs.")
 }
 
 func (o *Options) GetAPIServerName() string {

pkg/operator/options/suite_test.go

@@ -66,6 +66,7 @@ var _ = Describe("Options", func() {
 		"KUBELET_IDENTITY_CLIENT_ID",
 		"LINUX_ADMIN_USERNAME",
 		"ADDITIONAL_TAGS",
+		"ENABLE_AZURE_SDK_LOGGING",
 	}
 
 	var fs *coreoptions.FlagSet

pkg/providers/imagefamily/nodebootstrappingclient.go

@@ -68,6 +68,7 @@ type NodeBootstrappingClient struct {
 	resourceName      string
 	credential        azcore.TokenCredential
 	tokenProvider     *tokenProvider
+	enableLogging     bool
 }
 
 // NewNodeBootstrappingClient creates a new NodeBootstrappingClient with token caching enabled.
@@ -79,6 +80,7 @@ func NewNodeBootstrappingClient(
 	resourceName string,
 	credential azcore.TokenCredential,
 	serverURL string,
+	enableLogging bool,
 ) (*NodeBootstrappingClient, error) {
 	return &NodeBootstrappingClient{
 		serverURL:         serverURL,
@@ -89,6 +91,7 @@ func NewNodeBootstrappingClient(
 		tokenProvider: &tokenProvider{
 			cloud: cloud,
 		},
+		enableLogging: enableLogging,
 	}, nil
 }
 
@@ -107,10 +110,12 @@ func (c *NodeBootstrappingClient) Get(
 	}
 	transport.DefaultAuthentication = httptransport.BearerToken(token.Token)
 
-	// Middleware logging
-	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
-	loggingClient := restlogger.NewLoggingClient(logger)
-	transport.Transport = loggingClient.Transport
+	// Middleware logging only if ENABLE_AZURE_SDK_LOGGING flag is enabled
+	if c.enableLogging {
+		logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+		loggingClient := restlogger.NewLoggingClient(logger)
+		transport.Transport = loggingClient.Transport
+	}
 
 	// Create the client
 	client := client.New(transport, strfmt.Default)

pkg/providers/instance/azure_client.go

@@ -123,7 +123,7 @@ func NewAZClientFromAPI(
 // nolint: gocyclo
 func NewAZClient(ctx context.Context, cfg *auth.Config, env *auth.Environment, cred azcore.TokenCredential) (*AZClient, error) {
 	o := options.FromContext(ctx)
-	opts := armopts.DefaultARMOpts(env.Cloud)
+	opts := armopts.DefaultARMOpts(env.Cloud, o.EnableAzureSDKLogging)
 	extensionsClient, err := armcompute.NewVirtualMachineExtensionsClient(cfg.SubscriptionID, cred, opts)
 	if err != nil {
 		return nil, err
@@ -193,7 +193,8 @@ func NewAZClient(ctx context.Context, cfg *auth.Config, env *auth.Environment, c
 			cfg.ResourceGroup,
 			o.ClusterName,
 			cred,
-			o.NodeBootstrappingServerURL)
+			o.NodeBootstrappingServerURL,
+			o.EnableAzureSDKLogging)
 		if err != nil {
 			return nil, err
 		}

pkg/test/options.go

@@ -45,6 +45,7 @@ type OptionsFields struct {
 	VnetGUID                       *string
 	KubeletIdentityClientID        *string
 	AdditionalTags                 map[string]string
+	EnableAzureSDKLogging          *bool
 	DiskEncryptionSetID            *string
 
 	// SIG Flags not required by the self hosted offering
@@ -79,6 +80,7 @@ func Options(overrides ...OptionsFields) *azoptions.Options {
 		NodeResourceGroup:              lo.FromPtrOr(options.NodeResourceGroup, "test-resourceGroup"),
 		ProvisionMode:                  lo.FromPtrOr(options.ProvisionMode, "aksscriptless"),
 		NodeBootstrappingServerURL:     lo.FromPtrOr(options.NodeBootstrappingServerURL, ""),
+		EnableAzureSDKLogging:          lo.FromPtrOr(options.EnableAzureSDKLogging, true),
 		UseSIG:                         lo.FromPtrOr(options.UseSIG, false),
 		SIGSubscriptionID:              lo.FromPtrOr(options.SIGSubscriptionID, "12345678-1234-1234-1234-123456789012"),
 		SIGAccessTokenServerURL:        lo.FromPtrOr(options.SIGAccessTokenServerURL, "https://test-sig-access-token-server.com"),

pkg/utils/opts/armopts.go

@@ -29,15 +29,17 @@ import (
 	"github.com/Azure/karpenter-provider-azure/pkg/auth"
 )
 
-func DefaultARMOpts(cloudConfig cloud.Configuration) *arm.ClientOptions {
+func DefaultARMOpts(cloudConfig cloud.Configuration, enableLogging bool) *arm.ClientOptions {
 	opts := &arm.ClientOptions{}
 	opts.Telemetry = DefaultTelemetryOpts()
 	opts.Retry = DefaultRetryOpts()
 	opts.Transport = defaultHTTPClient
 	opts.Cloud = cloudConfig
 
-	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
-	opts.PerCallPolicies = append(opts.PerCallPolicies, shPolicy.NewLoggingPolicy(*logger))
+	if enableLogging {
+		logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+		opts.PerCallPolicies = append(opts.PerCallPolicies, shPolicy.NewLoggingPolicy(*logger))
+	}
 	return opts
 }