Update to Karpenter v1.8.0

https://github.com/kubernetes-sigs/karpenter/releases/tag/v1.8.0

Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]>

Author: mikkeloscar

cluster/manifests/z-karpenter/07-karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -262,6 +262,12 @@ spec:
                         description: ID is the capacity reservation id in EC2
                         pattern: ^cr-[0-9a-z]+$
                         type: string
+                      instanceMatchCriteria:
+                        description: InstanceMatchCriteria specifies how instances are matched to capacity reservations.
+                        enum:
+                          - open
+                          - targeted
+                        type: string
                       ownerID:
                         description: Owner is the owner id for the ami.
                         pattern: ^[0-9]{12}$
@@ -281,10 +287,10 @@ spec:
                   maxItems: 30
                   type: array
                   x-kubernetes-validations:
-                    - message: expected at least one, got none, ['tags', 'id']
-                      rule: self.all(x, has(x.tags) || has(x.id))
-                    - message: '''id'' is mutually exclusive, cannot be set along with tags in a capacity reservation selector term'
-                      rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))'
+                    - message: expected at least one, got none, ['tags', 'id', 'instanceMatchCriteria']
+                      rule: self.all(x, has(x.tags) || has(x.id) || has(x.instanceMatchCriteria))
+                    - message: '''id'' is mutually exclusive, cannot be set along with other fields in a capacity reservation selector term'
+                      rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID) || has(x.instanceMatchCriteria)))'
                 context:
                   description: |-
                     Context is a Reserved field in EC2 APIs
@@ -308,6 +314,11 @@ spec:
                   enum:
                     - RAID0
                   type: string
+                ipPrefixCount:
+                  description: IPPrefixCount sets the number of IPv4 prefixes to be automatically assigned to the network interface.
+                  format: int32
+                  minimum: 0
+                  type: integer
                 kubelet:
                   description: |-
                     Kubelet defines args to be used when configuring kubelet on provisioned nodes.

cluster/manifests/z-karpenter/09-karpenter.sh_nodepools.yaml

@@ -20,7 +20,7 @@ spec:
         - jsonPath: .spec.template.spec.nodeClassRef.name
           name: NodeClass
           type: string
-        - jsonPath: .status.resources.nodes
+        - jsonPath: .status.nodes
           name: Nodes
           type: string
         - jsonPath: .status.conditions[?(@.type=="Ready")].status
@@ -122,6 +122,7 @@ spec:
                                 - Empty
                                 - Drifted
                               type: string
+                            maxItems: 50
                             type: array
                           schedule:
                             description: |-
@@ -144,13 +145,15 @@ spec:
                         ConsolidateAfter is the duration the controller will wait
                         before attempting to terminate nodes that are underutilized.
                         Refer to ConsolidationPolicy for how underutilization is considered.
+                        When replicas is set, ConsolidateAfter is simply ignored
                       pattern: ^(([0-9]+(s|m|h))+|Never)$
                       type: string
                     consolidationPolicy:
                       default: WhenEmptyOrUnderutilized
                       description: |-
                         ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation
                         algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified
+                        When replicas is set, ConsolidationPolicy is simply ignored
                       enum:
                         - WhenEmpty
                         - WhenEmptyOrUnderutilized
@@ -165,8 +168,24 @@ spec:
                       - type: string
                     pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                     x-kubernetes-int-or-string: true
-                  description: Limits define a set of bounds for provisioning capacity.
+                  description: |-
+                    Limits define a set of bounds for provisioning capacity.
+                    Limits other than limits.nodes is not supported when replicas is set.
                   type: object
+                replicas:
+                  description: |-
+                    Replicas is the desired number of nodes for the NodePool. When specified, the NodePool will
+                    maintain this fixed number of replicas rather than scaling based on pod demand.
+                    When replicas is set:
+                      - The following fields are ignored:
+                          * disruption.consolidationPolicy
+                          * disruption.consolidateAfter
+                      - Only limits.nodes is supported; other resource limits (e.g., CPU, memory) must not be specified.
+                      - Weight is not supported.
+                    Note: This field is alpha.
+                  format: int64
+                  minimum: 0
+                  type: integer
                 template:
                   description: |-
                     Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with.
@@ -430,13 +449,21 @@ spec:
                     numerical weight indicates that this nodepool will be ordered
                     ahead of other nodepools with lower weights. A nodepool with no weight
                     will be treated as if it is a nodepool with a weight of 0.
+                    Weight is not supported when replicas is set.
                   format: int32
                   maximum: 100
                   minimum: 1
                   type: integer
               required:
                 - template
               type: object
+              x-kubernetes-validations:
+                - message: Cannot transition NodePool between static (replicas set) and dynamic (replicas unset) provisioning modes
+                  rule: has(self.replicas) == has(oldSelf.replicas)
+                - message: only 'limits.nodes' is supported on static NodePools
+                  rule: '!has(self.replicas) || (!has(self.limits) || size(self.limits) == 0 || (size(self.limits) == 1 && ''nodes'' in self.limits))'
+                - message: '''weight'' is not supported on static NodePools'
+                  rule: '!has(self.replicas) || !has(self.weight)'
             status:
               description: NodePoolStatus defines the observed state of NodePool
               properties:
@@ -502,6 +529,11 @@ spec:
                     the actual NodeClass Generation, NodeRegistrationHealthy status condition on the NodePool will be reset
                   format: int64
                   type: integer
+                nodes:
+                  default: 0
+                  description: Nodes is the count of nodes associated with this NodePool
+                  format: int64
+                  type: integer
                 resources:
                   additionalProperties:
                     anyOf:
@@ -518,4 +550,7 @@ spec:
       served: true
       storage: true
       subresources:
+        scale:
+          specReplicasPath: .spec.replicas
+          statusReplicasPath: .status.nodes
         status: {}

cluster/manifests/z-karpenter/deployment.yaml

@@ -51,7 +51,7 @@ spec:
             capabilities:
               drop:
                 - ALL
-          image: "container-registry.zalando.net/teapot/karpenter:1.7.1-main-42.patched"
+          image: "container-registry-test.zalando.net/teapot/karpenter:1.8.0-pr-45-3.patched"
           imagePullPolicy: IfNotPresent
           env:
             - name: KUBERNETES_MIN_VERSION
@@ -95,7 +95,7 @@ spec:
                   divisor: "0"
                   resource: limits.memory
             - name: FEATURE_GATES
-              value: "ReservedCapacity=false,SpotToSpotConsolidation=true,NodeRepair=false,NodeOverlay=false"
+              value: "ReservedCapacity=true,SpotToSpotConsolidation=true,NodeRepair=false,NodeOverlay=false,StaticCapacity=false"
             - name: BATCH_MAX_DURATION
               value: "10s"
             - name: BATCH_IDLE_DURATION
@@ -110,6 +110,8 @@ spec:
               value: "0.075"
             - name: RESERVED_ENIS
               value: "0"
+            - name: IGNORE_DRA_REQUESTS
+              value: "true"
             {{ $pod_cidr_size := "15" }}
             - name: MAX_NODE_CLAIMS
               value: "{{ nodeCIDRMaxNodesPodCIDR (parseInt64 $pod_cidr_size) (parseInt64 .Cluster.ConfigItems.node_cidr_mask_size) (parseInt64 .Cluster.ConfigItems.reserved_nodes) }}"