Feat/inference operator v3.7 logging priority

helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.lock

@@ -7,7 +7,7 @@ dependencies:
   version: 3.12.2
 - name: aws-fsx-csi-driver
   repository: https://kubernetes-sigs.github.io/aws-fsx-csi-driver
-  version: 1.11.0
+  version: 1.17.0
 - name: aws-load-balancer-controller
   repository: https://aws.github.io/eks-charts
   version: 1.13.2
@@ -17,5 +17,5 @@ dependencies:
 - name: keda
   repository: https://kedacore.github.io/charts
   version: 2.17.1
-digest: sha256:f54ece80a00cb4da98440551765d9c660a0704d6b59f4f9030a5a9e86eab4eea
-generated: "2025-10-27T17:20:29.746399171Z"
+digest: sha256:534d93414e9a3a600b9272ce30f04c84ad88af1dea0b4be37e309b35b9982992
+generated: "2026-09-08T14:54:38.719678-07:00"

helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml

@@ -15,11 +15,11 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 2.6.0
+version: 2.7.0
 
-# This is the version number of the application being deployed. Keep this aligned 
+# This is the version number of the application being deployed. Keep this aligned
 # with operator image MAJOR.MINOR version.
-appVersion: "3.6"
+appVersion: "3.7"
 
 dependencies:
 - name: aws-mountpoint-s3-csi-driver
@@ -33,7 +33,7 @@ dependencies:
   repository: "https://kubernetes-sigs.github.io/metrics-server/"
 - name: aws-fsx-csi-driver
   alias: fsx
-  version: 1.11.0
+  version: 1.17.0
   repository: "https://kubernetes-sigs.github.io/aws-fsx-csi-driver"
   condition: fsx.enabled
 - name: aws-load-balancer-controller

helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml

@@ -2539,6 +2539,51 @@ spec:
                     maxLength: 253
                     pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
                     type: string
+                  tolerations:
+                    description: |-
+                      Tolerations allow the inference pod to schedule onto nodes with matching taints.
+                      This is useful when HyperPod nodes are tainted to keep other workloads off of them:
+                      set the corresponding tolerations here so the inference pods can still be placed on
+                      those nodes. Each entry is a standard Kubernetes toleration and is applied to the
+                      Deployment's pod template (spec.template.spec.tolerations).
+                    items:
+                      description: |-
+                        The pod this Toleration is attached to tolerates any taint that matches
+                        the triple <key,value,effect> using the matching operator <operator>.
+                      properties:
+                        effect:
+                          description: |-
+                            Effect indicates the taint effect to match. Empty means match all taint effects.
+                            When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                          type: string
+                        key:
+                          description: |-
+                            Key is the taint key that the toleration applies to. Empty means match all taint keys.
+                            If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                          type: string
+                        operator:
+                          description: |-
+                            Operator represents a key's relationship to the value.
+                            Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal.
+                            Exists is equivalent to wildcard for value, so that a pod can
+                            tolerate all taints of a particular category.
+                            Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators).
+                          type: string
+                        tolerationSeconds:
+                          description: |-
+                            TolerationSeconds represents the period of time the toleration (which must be
+                            of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+                            it is not set, which means tolerate the taint forever (do not evict). Zero and
+                            negative values will be treated as 0 (evict immediately) by the system.
+                          format: int64
+                          type: integer
+                        value:
+                          description: |-
+                            Value is the taint value the toleration matches to.
+                            If the operator is Exists, the value should be empty, otherwise just a regular string.
+                          type: string
+                      type: object
+                    type: array
                   volumes:
                     description: |-
                       Additional volumes to add to the pod spec.
@@ -4479,6 +4524,17 @@ spec:
                         description: Provide the L2 cache URL to local storage
                         type: string
                     type: object
+                  mode:
+                    default: InProcess
+                    description: |-
+                      Mode selects how the LMCache cache engine is deployed: InProcess (default,
+                      legacy in-process connector) or MultiProcess (co-located lmcache server
+                      sidecar). MultiProcess requires an image with LMCache >= 0.5.0 and is
+                      incompatible with KV-aware intelligent routing.
+                    enum:
+                    - InProcess
+                    - MultiProcess
+                    type: string
                 type: object
               loadBalancer:
                 description: Configuration for Application Load Balancer

helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml

@@ -980,6 +980,17 @@ spec:
                         description: Provide the L2 cache URL to local storage
                         type: string
                     type: object
+                  mode:
+                    default: InProcess
+                    description: |-
+                      Mode selects how the LMCache cache engine is deployed: InProcess (default,
+                      legacy in-process connector) or MultiProcess (co-located lmcache server
+                      sidecar). MultiProcess requires an image with LMCache >= 0.5.0 and is
+                      incompatible with KV-aware intelligent routing.
+                    enum:
+                    - InProcess
+                    - MultiProcess
+                    type: string
                 type: object
               loadBalancer:
                 description: Configuration for Application Load Balancer
@@ -1535,6 +1546,51 @@ spec:
                     pattern: ^s3://([^/]+)/?(.*)$
                     type: string
                 type: object
+              tolerations:
+                description: |-
+                  Tolerations allow the inference pod to schedule onto nodes with matching taints.
+                  This is useful when HyperPod nodes are tainted to keep other workloads off of them:
+                  set the corresponding tolerations here so the inference pods can still be placed on
+                  those nodes. Each entry is a standard Kubernetes toleration and is applied to the
+                  Deployment's pod template (spec.template.spec.tolerations).
+                items:
+                  description: |-
+                    The pod this Toleration is attached to tolerates any taint that matches
+                    the triple <key,value,effect> using the matching operator <operator>.
+                  properties:
+                    effect:
+                      description: |-
+                        Effect indicates the taint effect to match. Empty means match all taint effects.
+                        When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                      type: string
+                    key:
+                      description: |-
+                        Key is the taint key that the toleration applies to. Empty means match all taint keys.
+                        If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                      type: string
+                    operator:
+                      description: |-
+                        Operator represents a key's relationship to the value.
+                        Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal.
+                        Exists is equivalent to wildcard for value, so that a pod can
+                        tolerate all taints of a particular category.
+                        Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators).
+                      type: string
+                    tolerationSeconds:
+                      description: |-
+                        TolerationSeconds represents the period of time the toleration (which must be
+                        of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+                        it is not set, which means tolerate the taint forever (do not evict). Zero and
+                        negative values will be treated as 0 (evict immediately) by the system.
+                      format: int64
+                      type: integer
+                    value:
+                      description: |-
+                        Value is the taint value the toleration matches to.
+                        If the operator is Exists, the value should be empty, otherwise just a regular string.
+                      type: string
+                  type: object
+                type: array
             required:
             - model
             - server

helm_chart/HyperPodHelmChart/charts/inference-operator/config/manager/manager.yaml

@@ -31,6 +31,9 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.priorityClassName }}
+      priorityClassName: {{ . | quote }}
+      {{- end }}
       securityContext:
         runAsNonRoot: true
         # TODO(user): For common cases that do not require escalating privileges
@@ -181,6 +184,10 @@ spec:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: webhook-certs
           readOnly: true
+        {{- if .Values.logging.cloudWatch.enabled }}
+        - mountPath: /logs
+          name: logs
+        {{- end }}
         env:
           - name: AWS_REGION
             value: {{ .Values.region }}
@@ -208,9 +215,60 @@ spec:
             value: {{ .Chart.AppVersion | quote }}
           - name: HYPERPOD_CLUSTER_ARN
             value: {{ .Values.hyperpodClusterArn | default "" }}
+          - name: CONSOLE_LOG_LEVEL
+            value: {{ .Values.logging.console.logLevel | default "debug" | quote }}
+          {{- if .Values.logging.cloudWatch.enabled }}
+          - name: LOG_FILE_PATH
+            value: /logs/operator.log
+          - name: CLOUDWATCH_LOG_LEVEL
+            value: {{ .Values.logging.cloudWatch.logLevel | default "info" | quote }}
+          {{- end }}
+      {{- if .Values.logging.cloudWatch.enabled }}
+      - name: fluent-bit
+        image: "{{ .Values.logging.cloudWatch.fluentBit.registryDomain }}/{{ .Values.logging.cloudWatch.fluentBit.repository }}:{{ .Values.logging.cloudWatch.fluentBit.tag }}"
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          runAsUser: 65532
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - "ALL"
+          readOnlyRootFilesystem: true
+        livenessProbe:
+          httpGet:
+            path: /api/v1/uptime
+            port: 2020
+          initialDelaySeconds: 10
+        resources:
+          limits:
+            cpu: 50m
+            memory: 64Mi
+          requests:
+            cpu: 5m
+            memory: 30Mi
+        env:
+          - name: AWS_REGION
+            value: {{ .Values.region }}
+          - name: EKS_CLUSTER_NAME
+            value: {{ .Values.eksClusterName }}
+        volumeMounts:
+          - mountPath: /logs
+            name: logs
+          - mountPath: /fluent-bit/etc
+            name: fluentbit-config
+            readOnly: true
+      {{- end }}
       serviceAccountName: {{ .Values.namePrefix }}-controller-manager
       terminationGracePeriodSeconds: 10
       volumes:
       - name: webhook-certs
         secret:
           secretName: webhook-server-cert
+      {{- if .Values.logging.cloudWatch.enabled }}
+      - name: logs
+        emptyDir:
+          sizeLimit: 200Mi
+      - name: fluentbit-config
+        configMap:
+          name: {{ .Values.namePrefix }}-fluentbit
+      {{- end }}

helm_chart/HyperPodHelmChart/charts/inference-operator/templates/_validateRequiredValues.tpl

@@ -56,4 +56,11 @@
 {{- $_ := set .Values.image "repository" (index .Values.image.repositoryDomainMap $region) -}}
 {{- end -}}
 
+{{- if .Values.logging.cloudWatch.enabled -}}
+{{- if not (hasKey .Values.logging.cloudWatch.fluentBit.registryAccountMap $region) -}}
+{{- fail "Unsupported AWS Region" -}}
+{{- end -}}
+{{- $_ := set .Values.logging.cloudWatch.fluentBit "registryDomain" (printf "%s.dkr.ecr.%s.amazonaws.com" (index .Values.logging.cloudWatch.fluentBit.registryAccountMap $region) $region) -}}
+{{- end -}}
+
 {{- end -}}

helm_chart/HyperPodHelmChart/charts/inference-operator/templates/fluentbit_configmap.yaml

@@ -0,0 +1,49 @@
+{{- if .Values.logging.cloudWatch.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.namePrefix }}-fluentbit
+  namespace: {{ .Values.shortPrefix }}-system
+data:
+  fluent-bit.conf: |
+    [SERVICE]
+        Flush         5
+        Log_Level     warn
+        Daemon        off
+        HTTP_Server   On
+        HTTP_Port     2020
+        Parsers_File  /fluent-bit/etc/parsers.conf
+
+    [INPUT]
+        Name              tail
+        Path              /logs/operator*.log
+        Tag               operator.*
+        Refresh_Interval  5
+        Read_from_Head    On
+        Mem_Buf_Limit     10MB
+        DB                /logs/fb.db
+        Parser            json
+
+    [FILTER]
+        Name   modify
+        Match  operator.*
+        Add    cluster   ${EKS_CLUSTER_NAME}
+        Add    namespace {{ .Values.shortPrefix }}-system
+
+    [OUTPUT]
+        Name               cloudwatch_logs
+        Match              operator.*
+        region             ${AWS_REGION}
+        log_group_name     /aws/hyperpod/${EKS_CLUSTER_NAME}/inference-operator
+        log_stream_prefix  operator-
+        auto_create_group  true
+        retry_limit        3
+
+  parsers.conf: |
+    [PARSER]
+        Name        json
+        Format      json
+        Time_Key    ts
+        Time_Format %Y-%m-%dT%H:%M:%S.%LZ
+        Time_Keep   On
+{{- end }}

helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml

@@ -25,7 +25,7 @@ image:
     ap-southeast-3: 158128612970.dkr.ecr.ap-southeast-3.amazonaws.com
     ap-south-2: 680458885894.dkr.ecr.ap-south-2.amazonaws.com
     eu-south-2: 025050981094.dkr.ecr.eu-south-2.amazonaws.com
-  tag: v3.6
+  tag: v3.7
   pullPolicy: Always
   repository:
   initContainer:
@@ -60,6 +60,8 @@ nodeSelector: {}
 
 tolerations: []
 
+priorityClassName: ""
+
 s3:
   enabled: true
   # IAM role ARN used for S3 CSI driver k8s service account
@@ -107,6 +109,35 @@ cert-manager:
   crds:
     enabled: true
 
+logging:
+  cloudWatch:
+    enabled: false
+    logLevel: info
+    fluentBit:
+      repository: eks/observability/aws-for-fluent-bit
+      tag: "3.4.6"
+      registryDomain: ""
+      registryAccountMap:
+        us-east-1: "602401143452"
+        us-east-2: "602401143452"
+        us-west-1: "602401143452"
+        us-west-2: "602401143452"
+        eu-west-1: "602401143452"
+        eu-west-2: "602401143452"
+        eu-central-1: "602401143452"
+        eu-north-1: "602401143452"
+        ap-northeast-1: "602401143452"
+        ap-southeast-1: "602401143452"
+        ap-southeast-2: "602401143452"
+        ap-south-1: "602401143452"
+        sa-east-1: "602401143452"
+        ap-southeast-3: "296578399912"
+        ap-southeast-4: "491585149902"
+        eu-south-2: "455263428931"
+        ap-south-2: "900889452093"
+  console:
+    logLevel: debug
+
 keda:
   enabled: true
   namespace: keda