From 37e552374e147e7a13e0ba8e9720360abe4fe659 Mon Sep 17 00:00:00 2001 From: Richa Shalom Gadagotti Date: Thu, 17 Sep 2026 20:25:25 +0000 Subject: [PATCH 1/3] feat(helm): bump inference-operator to v3.7, add priorityClassName, CloudWatch logging, tolerations, and MP mode --- ...s.amazon.com_inferenceendpointconfigs.yaml | 56 ++++++++++++++++++ ...emaker.aws.amazon.com_jumpstartmodels.yaml | 56 ++++++++++++++++++ .../config/manager/manager.yaml | 58 +++++++++++++++++++ .../charts/inference-operator/values.yaml | 35 ++++++++++- 4 files changed, 203 insertions(+), 2 deletions(-) diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml index a5bcb224..12bfa178 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml @@ -2539,6 +2539,51 @@ spec: maxLength: 253 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string + tolerations: + description: |- + Tolerations allow the inference pod to schedule onto nodes with matching taints. + This is useful when HyperPod nodes are tainted to keep other workloads off of them: + set the corresponding tolerations here so the inference pods can still be placed on + those nodes. Each entry is a standard Kubernetes toleration and is applied to the + Deployment's pod template (spec.template.spec.tolerations). + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array volumes: description: |- Additional volumes to add to the pod spec. @@ -4479,6 +4524,17 @@ spec: description: Provide the L2 cache URL to local storage type: string type: object + mode: + default: InProcess + description: |- + Mode selects how the LMCache cache engine is deployed: InProcess (default, + legacy in-process connector) or MultiProcess (co-located lmcache server + sidecar). MultiProcess requires an image with LMCache >= 0.5.0 and is + incompatible with KV-aware intelligent routing. + enum: + - InProcess + - MultiProcess + type: string type: object loadBalancer: description: Configuration for Application Load Balancer diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml index d05ebae5..156949b5 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml @@ -980,6 +980,17 @@ spec: description: Provide the L2 cache URL to local storage type: string type: object + mode: + default: InProcess + description: |- + Mode selects how the LMCache cache engine is deployed: InProcess (default, + legacy in-process connector) or MultiProcess (co-located lmcache server + sidecar). MultiProcess requires an image with LMCache >= 0.5.0 and is + incompatible with KV-aware intelligent routing. + enum: + - InProcess + - MultiProcess + type: string type: object loadBalancer: description: Configuration for Application Load Balancer @@ -1535,6 +1546,51 @@ spec: pattern: ^s3://([^/]+)/?(.*)$ type: string type: object + tolerations: + description: |- + Tolerations allow the inference pod to schedule onto nodes with matching taints. + This is useful when HyperPod nodes are tainted to keep other workloads off of them: + set the corresponding tolerations here so the inference pods can still be placed on + those nodes. Each entry is a standard Kubernetes toleration and is applied to the + Deployment's pod template (spec.template.spec.tolerations). + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array required: - model - server diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/manager/manager.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/manager/manager.yaml index f13bc4b4..30077bcb 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/config/manager/manager.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/config/manager/manager.yaml @@ -31,6 +31,9 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} securityContext: runAsNonRoot: true # TODO(user): For common cases that do not require escalating privileges @@ -181,6 +184,10 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: webhook-certs readOnly: true + {{- if .Values.logging.cloudWatch.enabled }} + - mountPath: /logs + name: logs + {{- end }} env: - name: AWS_REGION value: {{ .Values.region }} @@ -208,9 +215,60 @@ spec: value: {{ .Chart.AppVersion | quote }} - name: HYPERPOD_CLUSTER_ARN value: {{ .Values.hyperpodClusterArn | default "" }} + - name: CONSOLE_LOG_LEVEL + value: {{ .Values.logging.console.logLevel | default "debug" | quote }} + {{- if .Values.logging.cloudWatch.enabled }} + - name: LOG_FILE_PATH + value: /logs/operator.log + - name: CLOUDWATCH_LOG_LEVEL + value: {{ .Values.logging.cloudWatch.logLevel | default "info" | quote }} + {{- end }} + {{- if .Values.logging.cloudWatch.enabled }} + - name: fluent-bit + image: "{{ .Values.logging.cloudWatch.fluentBit.registryDomain }}/{{ .Values.logging.cloudWatch.fluentBit.repository }}:{{ .Values.logging.cloudWatch.fluentBit.tag }}" + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 65532 + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /api/v1/uptime + port: 2020 + initialDelaySeconds: 10 + resources: + limits: + cpu: 50m + memory: 64Mi + requests: + cpu: 5m + memory: 30Mi + env: + - name: AWS_REGION + value: {{ .Values.region }} + - name: EKS_CLUSTER_NAME + value: {{ .Values.eksClusterName }} + volumeMounts: + - mountPath: /logs + name: logs + - mountPath: /fluent-bit/etc + name: fluentbit-config + readOnly: true + {{- end }} serviceAccountName: {{ .Values.namePrefix }}-controller-manager terminationGracePeriodSeconds: 10 volumes: - name: webhook-certs secret: secretName: webhook-server-cert + {{- if .Values.logging.cloudWatch.enabled }} + - name: logs + emptyDir: + sizeLimit: 200Mi + - name: fluentbit-config + configMap: + name: {{ .Values.namePrefix }}-fluentbit + {{- end }} diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml index 4d0994cf..42bec83f 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml @@ -25,7 +25,7 @@ image: ap-southeast-3: 158128612970.dkr.ecr.ap-southeast-3.amazonaws.com ap-south-2: 680458885894.dkr.ecr.ap-south-2.amazonaws.com eu-south-2: 025050981094.dkr.ecr.eu-south-2.amazonaws.com - tag: v3.6 + tag: v3.7 pullPolicy: Always repository: initContainer: @@ -60,6 +60,8 @@ nodeSelector: {} tolerations: [] +priorityClassName: "" + s3: enabled: true # IAM role ARN used for S3 CSI driver k8s service account @@ -107,6 +109,35 @@ cert-manager: crds: enabled: true +logging: + cloudWatch: + enabled: false + logLevel: info + fluentBit: + repository: eks/observability/aws-for-fluent-bit + tag: "3.4.6" + registryDomain: "" + registryAccountMap: + us-east-1: "602401143452" + us-east-2: "602401143452" + us-west-1: "602401143452" + us-west-2: "602401143452" + eu-west-1: "602401143452" + eu-west-2: "602401143452" + eu-central-1: "602401143452" + eu-north-1: "602401143452" + ap-northeast-1: "602401143452" + ap-southeast-1: "602401143452" + ap-southeast-2: "602401143452" + ap-south-1: "602401143452" + sa-east-1: "602401143452" + ap-southeast-3: "296578399912" + ap-southeast-4: "491585149902" + eu-south-2: "455263428931" + ap-south-2: "900889452093" + console: + logLevel: debug + keda: enabled: true namespace: keda @@ -132,4 +163,4 @@ components: path: "config/metrics" webhook: enabled: true - path: "config/webhook" + path: "config/webhook" \ No newline at end of file From 1ead0e819a14d528205efbb900e9a9bd79badd97 Mon Sep 17 00:00:00 2001 From: Richa Shalom Gadagotti Date: Thu, 17 Sep 2026 20:29:56 +0000 Subject: [PATCH 2/3] fix: restore trailing newline in values.yaml --- .../HyperPodHelmChart/charts/inference-operator/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml index 42bec83f..2979eff3 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/values.yaml @@ -163,4 +163,4 @@ components: path: "config/metrics" webhook: enabled: true - path: "config/webhook" \ No newline at end of file + path: "config/webhook" From 99a989d4a4d8ab1aa885ab42e986c8759e69aecf Mon Sep 17 00:00:00 2001 From: Richa Shalom Gadagotti Date: Thu, 17 Sep 2026 21:31:07 +0000 Subject: [PATCH 3/3] feat(helm): add fluentbit ConfigMap, registryDomain resolver, bump Chart to v2.7.0/appVersion 3.7, bump fsx to 1.17.0 --- .../charts/inference-operator/Chart.lock | 6 +-- .../charts/inference-operator/Chart.yaml | 8 +-- .../templates/_validateRequiredValues.tpl | 7 +++ .../templates/fluentbit_configmap.yaml | 49 +++++++++++++++++++ 4 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 helm_chart/HyperPodHelmChart/charts/inference-operator/templates/fluentbit_configmap.yaml diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.lock b/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.lock index 8c71acfd..022cc7a6 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.lock +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.lock @@ -7,7 +7,7 @@ dependencies: version: 3.12.2 - name: aws-fsx-csi-driver repository: https://kubernetes-sigs.github.io/aws-fsx-csi-driver - version: 1.11.0 + version: 1.17.0 - name: aws-load-balancer-controller repository: https://aws.github.io/eks-charts version: 1.13.2 @@ -17,5 +17,5 @@ dependencies: - name: keda repository: https://kedacore.github.io/charts version: 2.17.1 -digest: sha256:f54ece80a00cb4da98440551765d9c660a0704d6b59f4f9030a5a9e86eab4eea -generated: "2025-10-27T17:20:29.746399171Z" +digest: sha256:534d93414e9a3a600b9272ce30f04c84ad88af1dea0b4be37e309b35b9982992 +generated: "2026-09-08T14:54:38.719678-07:00" diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml index 43da4435..47628cbd 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml @@ -15,11 +15,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 2.6.0 +version: 2.7.0 -# This is the version number of the application being deployed. Keep this aligned +# This is the version number of the application being deployed. Keep this aligned # with operator image MAJOR.MINOR version. -appVersion: "3.6" +appVersion: "3.7" dependencies: - name: aws-mountpoint-s3-csi-driver @@ -33,7 +33,7 @@ dependencies: repository: "https://kubernetes-sigs.github.io/metrics-server/" - name: aws-fsx-csi-driver alias: fsx - version: 1.11.0 + version: 1.17.0 repository: "https://kubernetes-sigs.github.io/aws-fsx-csi-driver" condition: fsx.enabled - name: aws-load-balancer-controller diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/templates/_validateRequiredValues.tpl b/helm_chart/HyperPodHelmChart/charts/inference-operator/templates/_validateRequiredValues.tpl index 0bd291f8..c2678304 100644 --- a/helm_chart/HyperPodHelmChart/charts/inference-operator/templates/_validateRequiredValues.tpl +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/templates/_validateRequiredValues.tpl @@ -56,4 +56,11 @@ {{- $_ := set .Values.image "repository" (index .Values.image.repositoryDomainMap $region) -}} {{- end -}} +{{- if .Values.logging.cloudWatch.enabled -}} +{{- if not (hasKey .Values.logging.cloudWatch.fluentBit.registryAccountMap $region) -}} +{{- fail "Unsupported AWS Region" -}} +{{- end -}} +{{- $_ := set .Values.logging.cloudWatch.fluentBit "registryDomain" (printf "%s.dkr.ecr.%s.amazonaws.com" (index .Values.logging.cloudWatch.fluentBit.registryAccountMap $region) $region) -}} +{{- end -}} + {{- end -}} diff --git a/helm_chart/HyperPodHelmChart/charts/inference-operator/templates/fluentbit_configmap.yaml b/helm_chart/HyperPodHelmChart/charts/inference-operator/templates/fluentbit_configmap.yaml new file mode 100644 index 00000000..7eba3a24 --- /dev/null +++ b/helm_chart/HyperPodHelmChart/charts/inference-operator/templates/fluentbit_configmap.yaml @@ -0,0 +1,49 @@ +{{- if .Values.logging.cloudWatch.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.namePrefix }}-fluentbit + namespace: {{ .Values.shortPrefix }}-system +data: + fluent-bit.conf: | + [SERVICE] + Flush 5 + Log_Level warn + Daemon off + HTTP_Server On + HTTP_Port 2020 + Parsers_File /fluent-bit/etc/parsers.conf + + [INPUT] + Name tail + Path /logs/operator*.log + Tag operator.* + Refresh_Interval 5 + Read_from_Head On + Mem_Buf_Limit 10MB + DB /logs/fb.db + Parser json + + [FILTER] + Name modify + Match operator.* + Add cluster ${EKS_CLUSTER_NAME} + Add namespace {{ .Values.shortPrefix }}-system + + [OUTPUT] + Name cloudwatch_logs + Match operator.* + region ${AWS_REGION} + log_group_name /aws/hyperpod/${EKS_CLUSTER_NAME}/inference-operator + log_stream_prefix operator- + auto_create_group true + retry_limit 3 + + parsers.conf: | + [PARSER] + Name json + Format json + Time_Key ts + Time_Format %Y-%m-%dT%H:%M:%S.%LZ + Time_Keep On +{{- end }}