Skip to content

Commit 57bfda5

Browse files
hakmanCiprian Hacman
authored andcommitted
feat: Add NodeOverlay support
Signed-off-by: Ciprian Hacman <[email protected]> Signed-off-by: Ciprian Hacman <[email protected]>
1 parent bdfd6e3 commit 57bfda5

File tree

16 files changed

+519
-13
lines changed

16 files changed

+519
-13
lines changed
Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
---
2+
apiVersion: apiextensions.k8s.io/v1
3+
kind: CustomResourceDefinition
4+
metadata:
5+
annotations:
6+
controller-gen.kubebuilder.io/version: v0.19.0
7+
name: nodeoverlays.karpenter.sh
8+
spec:
9+
group: karpenter.sh
10+
names:
11+
categories:
12+
- karpenter
13+
kind: NodeOverlay
14+
listKind: NodeOverlayList
15+
plural: nodeoverlays
16+
shortNames:
17+
- overlays
18+
singular: nodeoverlay
19+
scope: Cluster
20+
versions:
21+
- additionalPrinterColumns:
22+
- jsonPath: .status.conditions[?(@.type=="Ready")].status
23+
name: Ready
24+
type: string
25+
- jsonPath: .metadata.creationTimestamp
26+
name: Age
27+
type: date
28+
- jsonPath: .spec.weight
29+
name: Weight
30+
priority: 1
31+
type: integer
32+
name: v1alpha1
33+
schema:
34+
openAPIV3Schema:
35+
properties:
36+
apiVersion:
37+
description: |-
38+
APIVersion defines the versioned schema of this representation of an object.
39+
Servers should convert recognized schemas to the latest internal value, and
40+
may reject unrecognized values.
41+
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
42+
type: string
43+
kind:
44+
description: |-
45+
Kind is a string value representing the REST resource this object represents.
46+
Servers may infer this from the endpoint the client submits requests to.
47+
Cannot be updated.
48+
In CamelCase.
49+
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
50+
type: string
51+
metadata:
52+
type: object
53+
spec:
54+
properties:
55+
capacity:
56+
additionalProperties:
57+
anyOf:
58+
- type: integer
59+
- type: string
60+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
61+
x-kubernetes-int-or-string: true
62+
description: |-
63+
Capacity adds extended resources only, and does not replace any existing resources.
64+
These extended resources are appended to the node's existing resource list.
65+
Note: This field does not modify or override standard resources like cpu, memory, ephemeral-storage, or pods.
66+
type: object
67+
x-kubernetes-validations:
68+
- message: invalid resource restricted
69+
rule: self.all(x, !(x in ['cpu', 'memory', 'ephemeral-storage', 'pods']))
70+
price:
71+
description: Price specifies amount for an instance types that match the specified labels. Users can override prices using a signed float representing the price override
72+
pattern: ^\d+(\.\d+)?$
73+
type: string
74+
priceAdjustment:
75+
description: |-
76+
PriceAdjustment specifies the price change for matching instance types. Accepts either:
77+
- A fixed price modifier (e.g., -0.5, 1.2)
78+
- A percentage modifier (e.g., +10% for increase, -15% for decrees)
79+
pattern: ^(([+-]{1}(\d*\.?\d+))|(\+{1}\d*\.?\d+%)|(^(-\d{1,2}(\.\d+)?%)$)|(-100%))$
80+
type: string
81+
requirements:
82+
description: |-
83+
Requirements constrain when this NodeOverlay is applied during scheduling simulations.
84+
These requirements can match:
85+
- Well-known labels (e.g., node.kubernetes.io/instance-type, karpenter.sh/nodepool)
86+
- Custom labels from NodePool's spec.template.labels
87+
items:
88+
description: |-
89+
A node selector requirement is a selector that contains values, a key, and an operator
90+
that relates the key and values.
91+
properties:
92+
key:
93+
description: The label key that the selector applies to.
94+
type: string
95+
maxLength: 316
96+
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$
97+
x-kubernetes-validations:
98+
- message: label domain "kubernetes.io" is restricted
99+
rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io")
100+
- message: label domain "k8s.io" is restricted
101+
rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io")
102+
- message: label domain "karpenter.sh" is restricted
103+
rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh")
104+
- message: label "kubernetes.io/hostname" is restricted
105+
rule: self != "kubernetes.io/hostname"
106+
- message: label domain "karpenter.azure.com" is restricted
107+
rule: self in [ "karpenter.azure.com/aksnodeclass", "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com")
108+
operator:
109+
description: |-
110+
Represents a key's relationship to a set of values.
111+
Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
112+
type: string
113+
enum:
114+
- In
115+
- NotIn
116+
- Exists
117+
- DoesNotExist
118+
- Gt
119+
- Lt
120+
values:
121+
description: |-
122+
An array of string values. If the operator is In or NotIn,
123+
the values array must be non-empty. If the operator is Exists or DoesNotExist,
124+
the values array must be empty. If the operator is Gt or Lt, the values
125+
array must have a single element, which will be interpreted as an integer.
126+
This array is replaced during a strategic merge patch.
127+
items:
128+
type: string
129+
type: array
130+
x-kubernetes-list-type: atomic
131+
maxLength: 63
132+
pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$
133+
required:
134+
- key
135+
- operator
136+
type: object
137+
maxItems: 100
138+
type: array
139+
x-kubernetes-validations:
140+
- message: requirements with operator 'NotIn' must have a value defined
141+
rule: 'self.all(x, x.operator == ''NotIn'' ? x.values.size() != 0 : true)'
142+
- message: requirements with operator 'In' must have a value defined
143+
rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)'
144+
- message: requirements operator 'Gt' or 'Lt' must have a single positive integer value
145+
rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)'
146+
weight:
147+
description: |-
148+
Weight defines the priority of this NodeOverlay when overriding node attributes.
149+
NodeOverlays with higher numerical weights take precedence over those with lower weights.
150+
If no weight is specified, the NodeOverlay is treated as having a weight of 0.
151+
When multiple NodeOverlays have identical weights, they are merged in alphabetical order.
152+
format: int32
153+
maximum: 10000
154+
minimum: 1
155+
type: integer
156+
required:
157+
- requirements
158+
type: object
159+
x-kubernetes-validations:
160+
- message: cannot set both 'price' and 'priceAdjustment'
161+
rule: '!has(self.price) || !has(self.priceAdjustment)'
162+
status:
163+
description: NodeOverlayStatus defines the observed state of NodeOverlay
164+
properties:
165+
conditions:
166+
description: Conditions contains signals for health and readiness
167+
items:
168+
description: Condition aliases the upstream type and adds additional helper methods
169+
properties:
170+
lastTransitionTime:
171+
description: |-
172+
lastTransitionTime is the last time the condition transitioned from one status to another.
173+
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
174+
format: date-time
175+
type: string
176+
message:
177+
description: |-
178+
message is a human readable message indicating details about the transition.
179+
This may be an empty string.
180+
maxLength: 32768
181+
type: string
182+
observedGeneration:
183+
description: |-
184+
observedGeneration represents the .metadata.generation that the condition was set based upon.
185+
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
186+
with respect to the current state of the instance.
187+
format: int64
188+
minimum: 0
189+
type: integer
190+
reason:
191+
description: |-
192+
reason contains a programmatic identifier indicating the reason for the condition's last transition.
193+
Producers of specific condition types may define expected values and meanings for this field,
194+
and whether the values are considered a guaranteed API.
195+
The value should be a CamelCase string.
196+
This field may not be empty.
197+
maxLength: 1024
198+
minLength: 1
199+
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
200+
type: string
201+
status:
202+
description: status of the condition, one of True, False, Unknown.
203+
enum:
204+
- "True"
205+
- "False"
206+
- Unknown
207+
type: string
208+
type:
209+
description: type of condition in CamelCase or in foo.example.com/CamelCase.
210+
maxLength: 316
211+
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
212+
type: string
213+
required:
214+
- lastTransitionTime
215+
- message
216+
- reason
217+
- status
218+
- type
219+
type: object
220+
type: array
221+
type: object
222+
required:
223+
- spec
224+
type: object
225+
served: true
226+
storage: true
227+
subresources:
228+
status: {}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../pkg/apis/crds/karpenter.sh_nodeoverlays.yaml

charts/karpenter/templates/clusterrole-core.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ metadata:
3030
rules:
3131
# Read
3232
- apiGroups: ["karpenter.sh"]
33-
resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"]
33+
resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status", "nodeoverlays", "nodeoverlays/status"]
3434
verbs: ["get", "list", "watch"]
3535
- apiGroups: [""]
3636
resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"]
@@ -54,7 +54,7 @@ rules:
5454
resources: ["nodeclaims", "nodeclaims/status"]
5555
verbs: ["create", "delete", "update", "patch"]
5656
- apiGroups: ["karpenter.sh"]
57-
resources: ["nodepools", "nodepools/status"]
57+
resources: ["nodepools", "nodepools/status", "nodeoverlays/status"]
5858
verbs: ["update", "patch"]
5959
- apiGroups: [""]
6060
resources: ["events"]

charts/karpenter/templates/deployment.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ spec:
9999
divisor: "0"
100100
resource: limits.memory
101101
- name: FEATURE_GATES
102-
value: "SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }},NodeRepair={{ .Values.settings.featureGates.nodeRepair }}"
102+
value: "SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }},NodeRepair={{ .Values.settings.featureGates.nodeRepair }},NodeOverlay={{ .Values.settings.featureGates.nodeOverlay }}"
103103
{{- with .Values.settings.batchMaxDuration }}
104104
- name: BATCH_MAX_DURATION
105105
value: "{{ . }}"

charts/karpenter/values.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,3 +177,6 @@ settings:
177177
# -- nodeRepair is ALPHA and is disabled by default.
178178
# Setting this to true will enable node repair.
179179
nodeRepair: false
180+
# -- nodeOverlay is ALPHA and is disabled by default.
181+
# Setting this will allow the use of node overlay to impact scheduling decisions.
182+
nodeOverlay: false

cmd/controller/main.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232

3333
"github.com/Azure/karpenter-provider-azure/pkg/operator/options"
3434
"sigs.k8s.io/karpenter/pkg/cloudprovider/metrics"
35+
"sigs.k8s.io/karpenter/pkg/cloudprovider/overlay"
3536
corecontrollers "sigs.k8s.io/karpenter/pkg/controllers"
3637
"sigs.k8s.io/karpenter/pkg/controllers/state"
3738
coreoperator "sigs.k8s.io/karpenter/pkg/operator"
@@ -56,11 +57,13 @@ func main() {
5657
op.EventRecorder,
5758
op.GetClient(),
5859
op.ImageProvider,
60+
op.InstanceTypeStore,
5961
)
6062

6163
lo.Must0(op.AddHealthzCheck("cloud-provider", aksCloudProvider.LivenessProbe))
6264

63-
cloudProvider := metrics.Decorate(aksCloudProvider)
65+
overlayUndecoratedCloudProvider := metrics.Decorate(aksCloudProvider)
66+
cloudProvider := overlay.Decorate(overlayUndecoratedCloudProvider, op.GetClient(), op.InstanceTypeStore)
6467
clusterState := state.NewCluster(op.Clock, op.GetClient(), cloudProvider)
6568

6669
op.
@@ -71,7 +74,9 @@ func main() {
7174
op.GetClient(),
7275
op.EventRecorder,
7376
cloudProvider,
77+
overlayUndecoratedCloudProvider,
7478
clusterState,
79+
op.InstanceTypeStore,
7580
)...).
7681
WithControllers(ctx, controllers.NewControllers(
7782
ctx,

cmd/controller/main_ccp.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232

3333
"github.com/Azure/karpenter-provider-azure/pkg/operator/options"
3434
"sigs.k8s.io/karpenter/pkg/cloudprovider/metrics"
35+
"sigs.k8s.io/karpenter/pkg/cloudprovider/overlay"
3536
corecontrollers "sigs.k8s.io/karpenter/pkg/controllers"
3637
"sigs.k8s.io/karpenter/pkg/controllers/state"
3738
coreoperator "sigs.k8s.io/karpenter/pkg/operator"
@@ -56,11 +57,13 @@ func main() {
5657
op.EventRecorder,
5758
op.GetClient(),
5859
op.ImageProvider,
60+
op.InstanceTypeStore,
5961
)
6062

6163
lo.Must0(op.AddHealthzCheck("cloud-provider", aksCloudProvider.LivenessProbe))
6264

63-
cloudProvider := metrics.Decorate(aksCloudProvider)
65+
overlayUndecoratedCloudProvider := metrics.Decorate(aksCloudProvider)
66+
cloudProvider := overlay.Decorate(overlayUndecoratedCloudProvider, op.GetClient(), op.InstanceTypeStore)
6467
clusterState := state.NewCluster(op.Clock, op.GetClient(), cloudProvider)
6568

6669
op.
@@ -71,7 +74,9 @@ func main() {
7174
op.GetClient(),
7275
op.EventRecorder,
7376
cloudProvider,
77+
overlayUndecoratedCloudProvider,
7478
clusterState,
79+
op.InstanceTypeStore,
7580
)...).
7681
WithControllers(ctx, controllers.NewControllers(
7782
ctx,

hack/validation/requirements.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ rule=$(echo "$rule" | tr -s ' ') # remove extra spaces
3030
# check that .spec.versions has 1 entry
3131
[[ $(yq e '.spec.versions | length' pkg/apis/crds/karpenter.sh_nodepools.yaml) -eq 1 ]] || { echo "expected one version"; exit 1; }
3232
[[ $(yq e '.spec.versions | length' pkg/apis/crds/karpenter.sh_nodeclaims.yaml) -eq 1 ]] || { echo "expected one version"; exit 1; }
33+
[[ $(yq e '.spec.versions | length' pkg/apis/crds/karpenter.sh_nodeoverlays.yaml) -eq 1 ]] || { echo "expected one version"; exit 1; }
3334

3435
# nodeclaim
3536
printf -v expr '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations +=
@@ -40,3 +41,8 @@ yq eval "${expr}" -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml
4041
printf -v expr '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations +=
4142
[{"message": "label domain \\"karpenter.azure.com\\" is restricted", "rule": "%s"}]' "$rule"
4243
yq eval "${expr}" -i pkg/apis/crds/karpenter.sh_nodepools.yaml
44+
45+
# overlays
46+
printf -v expr '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations +=
47+
[{"message": "label domain \\"karpenter.azure.com\\" is restricted", "rule": "%s"}]' "$rule"
48+
yq eval "${expr}" -i pkg/apis/crds/karpenter.sh_nodeoverlays.yaml

pkg/apis/apis.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,12 @@ var (
3434
NodePoolCRD []byte
3535
//go:embed crds/karpenter.sh_nodeclaims.yaml
3636
NodeClaimCRD []byte
37-
CRDs = []*apiextensionsv1.CustomResourceDefinition{
37+
//go:embed crds/karpenter.sh_nodeoverlays.yaml
38+
NodeOverlayCRD []byte
39+
CRDs = []*apiextensionsv1.CustomResourceDefinition{
3840
object.Unmarshal[apiextensionsv1.CustomResourceDefinition](AKSNodeClassCRD),
3941
object.Unmarshal[apiextensionsv1.CustomResourceDefinition](NodePoolCRD),
4042
object.Unmarshal[apiextensionsv1.CustomResourceDefinition](NodeClaimCRD),
43+
object.Unmarshal[apiextensionsv1.CustomResourceDefinition](NodeOverlayCRD),
4144
}
4245
)

0 commit comments

Comments
 (0)