@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da855b42d2

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 17:00 UTC

[Doc]: Make extraInit containers fully configurable in helm chart (#27497)

Signed-off-by: Fang Han <fhan0520@gmail.com>

Fang Han committed Nov 6, 2025 at 12:27 UTC da855b42d2005d9b701758e9d59836131ee8c6fb

10 files changed +439 -27

docs/deployment/frameworks/helm.md

+44 -5

@@ -13,7 +13,7 @@ Before you begin, ensure that you have the following:

13 - A running Kubernetes cluster

14 - NVIDIA Kubernetes Device Plugin (`k8s-device-plugin`): This can be found at [https://github.com/NVIDIA/k8s-device-plugin](https://github.com/NVIDIA/k8s-device-plugin)

15 - Available GPU resources in your cluster

16 -- An S3 with the model which will be deployed

16 +- (Optional) An S3 bucket or other storage with the model weights, if using automatic model download

18 ## Installing the chart

@@ -61,10 +61,16 @@ The following table describes configurable parameters of the chart in `values.ya

61 | deploymentStrategy | object | {} | Deployment strategy configuration |

64 -| extraInit | object | {"pvcStorage":"1Gi","s3modelpath":"relative_s3_model_path/opt-125m", "awsEc2MetadataDisabled": true} | Additional configuration for the init container |

64 +| extraInit | object | {"modelDownload":{"enabled":true},"initContainers":[],"pvcStorage":"1Gi"} | Additional configuration for init containers |

65 +| extraInit.modelDownload | object | {"enabled":true} | Model download functionality configuration |

67 +| extraInit.modelDownload.image | object | {"repository":"amazon/aws-cli","tag":"2.6.4","pullPolicy":"IfNotPresent"} | Image for model download operations |

68 +| extraInit.modelDownload.waitContainer | object | {} | Wait container configuration (command, args, env) |

69 +| extraInit.modelDownload.downloadJob | object | {} | Download job configuration (command, args, env) |

76 | image | object | {"command":["vllm","serve","/data/","--served-model-name","opt-125m","--host","0.0.0.0","--port","8000"],"repository":"vllm/vllm-openai","tag":"latest"} | Image configuration |

@@ -98,3 +104,36 @@ The following table describes configurable parameters of the chart in `values.ya

105 | servicePort | int | 80 | Service port |

107 +

108 +## Configuration Examples

109 +

110 +### Using S3 Model Download (Default)

111 +

112 +```yaml

113 +extraInit:

114 + modelDownload:

115 + enabled: true

116 + pvcStorage: "10Gi"

117 + s3modelpath: "models/llama-7b"

118 +```

119 +

120 +### Using Custom Init Containers Only

121 +

122 +For use cases like llm-d where you need custom sidecars without model download:

123 +

124 +```yaml

125 +extraInit:

126 + modelDownload:

127 + enabled: false

128 + initContainers:

129 + - name: llm-d-routing-proxy

130 + image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0

131 + imagePullPolicy: IfNotPresent

132 + ports:

133 + - containerPort: 8080

134 + name: proxy

135 + securityContext:

136 + runAsUser: 1000

137 + restartPolicy: Always

138 + pvcStorage: "10Gi"

139 +```

examples/online_serving/chart-helm/README.md

+12

@@ -19,3 +19,15 @@ This directory contains a Helm chart for deploying the vllm application. The cha

19 - templates/pvc.yaml: Template for Persistent Volume Claims.

20 - templates/secrets.yaml: Template for Kubernetes Secrets.

21 - templates/service.yaml: Template for creating Services.

22 +

23 +## Running Tests

24 +

25 +This chart includes unit tests using [helm-unittest](https://github.com/helm-unittest/helm-unittest). Install the plugin and run tests:

26 +

27 +```bash

28 +# Install plugin

29 +helm plugin install https://github.com/helm-unittest/helm-unittest

30 +

31 +# Run tests

32 +helm unittest .

33 +```

examples/online_serving/chart-helm/templates/_helpers.tpl

+4 -3

@@ -123,9 +123,6 @@ runAsUser:

123 {{- end }}

124 {{- end }}

125

126 -{{- define "chart.extraInitImage" -}}

127 -"amazon/aws-cli:2.6.4"

128 -{{- end }}

126

127 {{- define "chart.extraInitEnv" -}}

128 - name: S3_ENDPOINT_URL

@@ -148,11 +145,15 @@ runAsUser:

145 secretKeyRef:

146 name: {{ .Release.Name }}-secrets

147 key: s3accesskey

148 +{{- if .Values.extraInit.s3modelpath }}

149 - name: S3_PATH

150 value: "{{ .Values.extraInit.s3modelpath }}"

151 +{{- end }}

152 +{{- if hasKey .Values.extraInit "awsEc2MetadataDisabled" }}

153 - name: AWS_EC2_METADATA_DISABLED

154 value: "{{ .Values.extraInit.awsEc2MetadataDisabled }}"

155 {{- end }}

156 +{{- end }}

157

158 {{/*

159 Define chart labels

examples/online_serving/chart-helm/templates/deployment.yaml

+16 -7

@@ -72,16 +72,21 @@ spec:

72 {{ toYaml . | nindent 8 }}

73 {{- end }}

75 - {{- if .Values.extraInit }}

75 + {{- if and .Values.extraInit (or .Values.extraInit.modelDownload.enabled .Values.extraInit.initContainers) }}

76 initContainers:

77 + {{- if .Values.extraInit.modelDownload.enabled }}

78 - name: wait-download-model

78 - image: {{ include "chart.extraInitImage" . }}

79 - command:

80 - - /bin/bash

79 + image: {{ .Values.extraInit.modelDownload.image.repository }}:{{ .Values.extraInit.modelDownload.image.tag }}

80 + imagePullPolicy: {{ .Values.extraInit.modelDownload.image.pullPolicy }}

81 + command: {{ .Values.extraInit.modelDownload.waitContainer.command | toJson }}

82 args:

82 - - -eucx

83 - - while aws --endpoint-url $S3_ENDPOINT_URL s3 sync --dryrun s3://$S3_BUCKET_NAME/$S3_PATH /data | grep -q download; do sleep 10; done

84 - env: {{- include "chart.extraInitEnv" . | nindent 10 }}

83 + {{- toYaml .Values.extraInit.modelDownload.waitContainer.args | nindent 10 }}

84 + env:

85 + {{- if .Values.extraInit.modelDownload.waitContainer.env }}

86 + {{- toYaml .Values.extraInit.modelDownload.waitContainer.env | nindent 10 }}

87 + {{- else }}

88 + {{- include "chart.extraInitEnv" . | nindent 10 }}

89 + {{- end }}

90 resources:

91 requests:

92 cpu: 200m

@@ -93,6 +98,10 @@ spec:

98 - name: {{ .Release.Name }}-storage

99 mountPath: /data

100 {{- end }}

101 + {{- with .Values.extraInit.initContainers }}

102 + {{- toYaml . | nindent 6 }}

103 + {{- end }}

104 + {{- end }}

105 volumes:

106 - name: {{ .Release.Name }}-storage

107 persistentVolumeClaim:

examples/online_serving/chart-helm/templates/job.yaml

+11 -7

@@ -1,4 +1,4 @@

1 -{{- if .Values.extraInit }}

1 +{{- if and .Values.extraInit .Values.extraInit.modelDownload.enabled }}

2 apiVersion: batch/v1

3 kind: Job

4 metadata:

@@ -12,13 +12,17 @@ spec:

12 spec:

13 containers:

14 - name: job-download-model

15 - image: {{ include "chart.extraInitImage" . }}

16 - command:

17 - - /bin/bash

15 + image: {{ .Values.extraInit.modelDownload.image.repository }}:{{ .Values.extraInit.modelDownload.image.tag }}

16 + imagePullPolicy: {{ .Values.extraInit.modelDownload.image.pullPolicy }}

17 + command: {{ .Values.extraInit.modelDownload.downloadJob.command | toJson }}

18 args:

19 - - -eucx

20 - - aws --endpoint-url $S3_ENDPOINT_URL s3 sync s3://$S3_BUCKET_NAME/$S3_PATH /data

21 - env: {{- include "chart.extraInitEnv" . | nindent 8 }}

19 + {{- toYaml .Values.extraInit.modelDownload.downloadJob.args | nindent 8 }}

20 + env:

21 + {{- if .Values.extraInit.modelDownload.downloadJob.env }}

22 + {{- toYaml .Values.extraInit.modelDownload.downloadJob.env | nindent 8 }}

23 + {{- else }}

24 + {{- include "chart.extraInitEnv" . | nindent 8 }}

25 + {{- end }}

26 volumeMounts:

27 - name: {{ .Release.Name }}-storage

28 mountPath: /data

examples/online_serving/chart-helm/tests/deployment_test.yaml new

+135

@@ -0,0 +1,135 @@

1 +suite: test deployment

2 +templates:

3 + - deployment.yaml

4 +tests:

5 + - it: should create wait-download-model init container when modelDownload is enabled

6 + set:

7 + extraInit:

8 + modelDownload:

9 + enabled: true

10 + image:

11 + repository: "amazon/aws-cli"

12 + tag: "2.6.4"

13 + pullPolicy: "IfNotPresent"

14 + waitContainer:

15 + command: [ "/bin/bash" ]

16 + args:

17 + - "-eucx"

18 + - "while aws --endpoint-url $S3_ENDPOINT_URL s3 sync --dryrun s3://$S3_BUCKET_NAME/$S3_PATH /data | grep -q download; do sleep 10; done"

19 + downloadJob:

20 + command: [ "/bin/bash" ]

21 + args:

22 + - "-eucx"

23 + - "aws --endpoint-url $S3_ENDPOINT_URL s3 sync s3://$S3_BUCKET_NAME/$S3_PATH /data"

24 + initContainers: [ ]

25 + pvcStorage: "1Gi"

26 + s3modelpath: "relative_s3_model_path/opt-125m"

27 + awsEc2MetadataDisabled: true

28 + asserts:

29 + - hasDocuments:

30 + count: 1

31 + - isKind:

32 + of: Deployment

33 + - isNotEmpty:

34 + path: spec.template.spec.initContainers

35 + - equal:

36 + path: spec.template.spec.initContainers[0].name

37 + value: wait-download-model

38 + - equal:

39 + path: spec.template.spec.initContainers[0].image

40 + value: amazon/aws-cli:2.6.4

41 + - equal:

42 + path: spec.template.spec.initContainers[0].imagePullPolicy

43 + value: IfNotPresent

44 +

45 + - it: should only create custom init containers when modelDownload is disabled

46 + set:

47 + extraInit:

48 + modelDownload:

49 + enabled: false

50 + image:

51 + repository: "amazon/aws-cli"

52 + tag: "2.6.4"

53 + pullPolicy: "IfNotPresent"

54 + waitContainer:

55 + command: [ "/bin/bash" ]

56 + args: [ "-c", "echo test" ]

57 + downloadJob:

58 + command: [ "/bin/bash" ]

59 + args: [ "-c", "echo test" ]

60 + initContainers:

61 + - name: llm-d-routing-proxy

62 + image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0

63 + imagePullPolicy: IfNotPresent

64 + ports:

65 + - containerPort: 8080

66 + name: proxy

67 + pvcStorage: "10Gi"

68 + asserts:

69 + - hasDocuments:

70 + count: 1

71 + - isKind:

72 + of: Deployment

73 + - lengthEqual:

74 + path: spec.template.spec.initContainers

75 + count: 1

76 + - equal:

77 + path: spec.template.spec.initContainers[0].name

78 + value: llm-d-routing-proxy

79 + - equal:

80 + path: spec.template.spec.initContainers[0].image

81 + value: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0

82 + - equal:

83 + path: spec.template.spec.initContainers[0].ports[0].containerPort

84 + value: 8080

85 +

86 + - it: should create both wait-download-model and custom init containers when both are enabled

87 + set:

88 + extraInit:

89 + modelDownload:

90 + enabled: true

91 + image:

92 + repository: "amazon/aws-cli"

93 + tag: "2.6.4"

94 + pullPolicy: "IfNotPresent"

95 + waitContainer:

96 + command: [ "/bin/bash" ]

97 + args:

98 + - "-eucx"

99 + - "while aws --endpoint-url $S3_ENDPOINT_URL s3 sync --dryrun s3://$S3_BUCKET_NAME/$S3_PATH /data | grep -q download; do sleep 10; done"

100 + downloadJob:

101 + command: [ "/bin/bash" ]

102 + args:

103 + - "-eucx"

104 + - "aws --endpoint-url $S3_ENDPOINT_URL s3 sync s3://$S3_BUCKET_NAME/$S3_PATH /data"

105 + initContainers:

106 + - name: llm-d-routing-proxy

107 + image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0

108 + imagePullPolicy: IfNotPresent

109 + ports:

110 + - containerPort: 8080

111 + name: proxy

112 + pvcStorage: "10Gi"

113 + asserts:

114 + - hasDocuments:

115 + count: 1

116 + - isKind:

117 + of: Deployment

118 + - lengthEqual:

119 + path: spec.template.spec.initContainers

120 + count: 2

121 + - equal:

122 + path: spec.template.spec.initContainers[0].name

123 + value: wait-download-model

124 + - equal:

125 + path: spec.template.spec.initContainers[0].image

126 + value: amazon/aws-cli:2.6.4

127 + - equal:

128 + path: spec.template.spec.initContainers[1].name

129 + value: llm-d-routing-proxy

130 + - equal:

131 + path: spec.template.spec.initContainers[1].image

132 + value: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0

133 + - equal:

134 + path: spec.template.spec.initContainers[1].ports[0].containerPort

135 + value: 8080

\ No newline at end of file

examples/online_serving/chart-helm/tests/job_test.yaml new

+61

@@ -0,0 +1,61 @@

1 +suite: test job

2 +templates:

3 + - job.yaml

4 +tests:

5 + - it: should create job when modelDownload is enabled

6 + set:

7 + extraInit:

8 + modelDownload:

9 + enabled: true

10 + image:

11 + repository: "amazon/aws-cli"

12 + tag: "2.6.4"

13 + pullPolicy: "IfNotPresent"

14 + waitContainer:

15 + command: [ "/bin/bash" ]

16 + args: [ "-c", "wait" ]

17 + downloadJob:

18 + command: [ "/bin/bash" ]

19 + args:

20 + - "-eucx"

21 + - "aws --endpoint-url $S3_ENDPOINT_URL s3 sync s3://$S3_BUCKET_NAME/$S3_PATH /data"

22 + pvcStorage: "1Gi"

23 + s3modelpath: "relative_s3_model_path/opt-125m"

24 + awsEc2MetadataDisabled: true

25 + asserts:

26 + - hasDocuments:

27 + count: 1

28 + - isKind:

29 + of: Job

30 + - equal:

31 + path: spec.template.spec.containers[0].name

32 + value: job-download-model

33 + - equal:

34 + path: spec.template.spec.containers[0].image

35 + value: amazon/aws-cli:2.6.4

36 + - equal:

37 + path: spec.template.spec.restartPolicy

38 + value: OnFailure

39 +

40 + - it: should not create job when modelDownload is disabled

41 + set:

42 + extraInit:

43 + modelDownload:

44 + enabled: false

45 + image:

46 + repository: "amazon/aws-cli"

47 + tag: "2.6.4"

48 + pullPolicy: "IfNotPresent"

49 + waitContainer:

50 + command: [ "/bin/bash" ]

51 + args: [ "-c", "wait" ]

52 + downloadJob:

53 + command: [ "/bin/bash" ]

54 + args: [ "-c", "download" ]

55 + initContainers:

56 + - name: llm-d-routing-proxy

57 + image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0

58 + pvcStorage: "10Gi"

59 + asserts:

60 + - hasDocuments:

61 + count: 0

examples/online_serving/chart-helm/tests/pvc_test.yaml new

+32

@@ -0,0 +1,32 @@

1 +suite: test pvc

2 +templates:

3 + - pvc.yaml

4 +tests:

5 + # Test Case: PVC Created When extraInit Defined

6 + - it: should create pvc when extraInit is defined

7 + set:

8 + extraInit:

9 + modelDownload:

10 + enabled: true

11 + image:

12 + repository: "amazon/aws-cli"

13 + tag: "2.6.4"

14 + pullPolicy: "IfNotPresent"

15 + waitContainer:

16 + command: ["/bin/bash"]

17 + args: ["-c", "wait"]

18 + downloadJob:

19 + command: ["/bin/bash"]

20 + args: ["-c", "download"]

21 + pvcStorage: "10Gi"

22 + asserts:

23 + - hasDocuments:

24 + count: 1

25 + - isKind:

26 + of: PersistentVolumeClaim

27 + - equal:

28 + path: spec.accessModes[0]

29 + value: ReadWriteOnce

30 + - equal:

31 + path: spec.resources.requests.storage

32 + value: 10Gi

\ No newline at end of file

examples/online_serving/chart-helm/values.schema.json

+67 -3

@@ -136,6 +136,70 @@

136 "extraInit": {

137 "type": "object",

138 "properties": {

139 + "modelDownload": {

140 + "type": "object",

141 + "properties": {

142 + "enabled": {

143 + "type": "boolean"

144 + },

145 + "image": {

146 + "type": "object",

147 + "properties": {

148 + "repository": {

149 + "type": "string"

150 + },

151 + "tag": {

152 + "type": "string"

153 + },

154 + "pullPolicy": {

155 + "type": "string"

156 + }

157 + },

158 + "required": ["repository", "tag", "pullPolicy"]

159 + },

160 + "waitContainer": {

161 + "type": "object",

162 + "properties": {

163 + "command": {

164 + "type": "array",

165 + "items": {"type": "string"}

166 + },

167 + "args": {

168 + "type": "array",

169 + "items": {"type": "string"}

170 + },

171 + "env": {

172 + "type": "array",

173 + "items": {"type": "object"}

174 + }

175 + },

176 + "required": ["command", "args"]

177 + },

178 + "downloadJob": {

179 + "type": "object",

180 + "properties": {

181 + "command": {

182 + "type": "array",

183 + "items": {"type": "string"}

184 + },

185 + "args": {

186 + "type": "array",

187 + "items": {"type": "string"}

188 + },

189 + "env": {

190 + "type": "array",

191 + "items": {"type": "object"}

192 + }

193 + },

194 + "required": ["command", "args"]

195 + }

196 + },

197 + "required": ["enabled", "image", "waitContainer", "downloadJob"]

198 + },

199 + "initContainers": {

200 + "type": "array",

201 + "items": {"type": "object"}

202 + },

203 "s3modelpath": {

204 "type": "string"

205 },

@@ -147,9 +211,9 @@

211 }

212 },

213 "required": [

150 - "pvcStorage",

151 - "s3modelpath",

152 - "awsEc2MetadataDisabled"

214 + "modelDownload",

215 + "initContainers",

216 + "pvcStorage"

217 ]

218 },

219 "extraContainers": {

examples/online_serving/chart-helm/values.yaml

+57 -2

@@ -75,10 +75,65 @@ maxUnavailablePodDisruptionBudget: ""

76 # -- Additional configuration for the init container

77 extraInit:

78 - # -- Path of the model on the s3 which hosts model weights and config files

78 + # -- Model download functionality (optional)

79 + modelDownload:

80 + # -- Enable model download job and wait container

81 + enabled: true

82 + # -- Image configuration for model download operations

83 + image:

84 + # -- Image repository

85 + repository: "amazon/aws-cli"

86 + # -- Image tag

87 + tag: "2.6.4"

88 + # -- Image pull policy

89 + pullPolicy: "IfNotPresent"

90 + # -- Wait container configuration (init container that waits for model to be ready)

91 + waitContainer:

92 + # -- Command to execute

93 + command: ["/bin/bash"]

94 + # -- Arguments for the wait container

95 + args:

96 + - "-eucx"

97 + - "while aws --endpoint-url $S3_ENDPOINT_URL s3 sync --dryrun s3://$S3_BUCKET_NAME/$S3_PATH /data | grep -q download; do sleep 10; done"

98 + # -- Environment variables (optional, overrides S3 defaults entirely if specified)

99 + # env:

100 + # - name: HUGGING_FACE_HUB_TOKEN

101 + # value: "your-token"

102 + # - name: MODEL_ID

103 + # value: "meta-llama/Llama-2-7b"

104 + # -- Download job configuration (job that actually downloads the model)

105 + downloadJob:

106 + # -- Command to execute

107 + command: ["/bin/bash"]

108 + # -- Arguments for the download job

109 + args:

110 + - "-eucx"

111 + - "aws --endpoint-url $S3_ENDPOINT_URL s3 sync s3://$S3_BUCKET_NAME/$S3_PATH /data"

112 + # -- Environment variables (optional, overrides S3 defaults entirely if specified)

113 + # env:

114 + # - name: HUGGING_FACE_HUB_TOKEN

115 + # value: "your-token"

116 + # - name: MODEL_ID

117 + # value: "meta-llama/Llama-2-7b"

118 +

119 + # -- Custom init containers (appended after wait-download-model if modelDownload is enabled)

120 + initContainers: []

121 + # Example for llm-d sidecar:

122 + # initContainers:

123 + # - name: llm-d-routing-proxy

124 + # image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0

125 + # imagePullPolicy: IfNotPresent

126 + # ports:

127 + # - containerPort: 8080

128 + # name: proxy

129 + # securityContext:

130 + # runAsUser: 1000

131 +

132 + # -- Path of the model on the s3 which hosts model weights and config files

133 s3modelpath: "relative_s3_model_path/opt-125m"

80 - # -- Storage size of the s3

134 + # -- Storage size for the PVC

135 pvcStorage: "1Gi"

136 + # -- Disable AWS EC2 metadata service

137 awsEc2MetadataDisabled: true

138

139 # -- Additional containers configuration