@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3d300aecb1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Doc] Switch K8S examples to default MP mode (#39400)

Signed-off-by: Peter Pan <Peter.Pan@daocloud.io> Signed-off-by: Peter Pan <peter.pan@daocloud.io> Signed-off-by: Kyle Sayers <kylesayrs@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Kyle Sayers <kylesayrs@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>

Peter Pan committed Jun 11, 2026 at 02:17 UTC 3d300aecb1e6639872b698bd74ed38fb81d9603e
3 files changed +472 -224
docs/deployment/frameworks/lws.md
+222 -108
@@ -7,108 +7,202 @@ vLLM can be deployed with [LWS](https://github.com/kubernetes-sigs/lws) on Kuber
7
8 ## Prerequisites
9
10 -* At least two Kubernetes nodes, each with 8 GPUs, are required.
11 -* Install LWS by following the instructions found [here](https://lws.sigs.k8s.io/docs/installation/).
10 +- At least two Kubernetes nodes, each with 8 GPUs, are required.
11 +- Install LWS by following the instructions found [here](https://lws.sigs.k8s.io/docs/installation/).
12
13 ## Deploy and Serve
14
15 -Deploy the following yaml file `lws.yaml`
16 -
17 -??? code "Yaml"
18 -
19 - ```yaml
20 - apiVersion: leaderworkerset.x-k8s.io/v1
21 - kind: LeaderWorkerSet
22 - metadata:
23 - name: vllm
24 - spec:
25 - replicas: 1
26 - leaderWorkerTemplate:
27 - size: 2
28 - restartPolicy: RecreateGroupOnPodRestart
29 - leaderTemplate:
30 - metadata:
31 - labels:
32 - role: leader
33 - spec:
34 - containers:
35 - - name: vllm-leader
36 - image: docker.io/vllm/vllm-openai:latest
37 - env:
38 - - name: HF_TOKEN
39 - value: <your-hf-token>
40 - command:
41 - - sh
42 - - -c
43 - - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=$(LWS_GROUP_SIZE);
44 - vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline_parallel_size 2"
45 - resources:
46 - limits:
47 - nvidia.com/gpu: "8"
48 - memory: 1124Gi
49 - ephemeral-storage: 800Gi
50 - requests:
51 - ephemeral-storage: 800Gi
52 - cpu: 125
53 - ports:
54 - - containerPort: 8080
55 - readinessProbe:
56 - tcpSocket:
57 - port: 8080
58 - initialDelaySeconds: 15
59 - periodSeconds: 10
60 - volumeMounts:
61 - - mountPath: /dev/shm
62 - name: dshm
63 - volumes:
64 - - name: dshm
65 - emptyDir:
66 - medium: Memory
67 - sizeLimit: 15Gi
68 - workerTemplate:
69 - spec:
70 - containers:
71 - - name: vllm-worker
72 - image: docker.io/vllm/vllm-openai:latest
73 - command:
74 - - sh
75 - - -c
76 - - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(LWS_LEADER_ADDRESS)"
77 - resources:
78 - limits:
79 - nvidia.com/gpu: "8"
80 - memory: 1124Gi
81 - ephemeral-storage: 800Gi
82 - requests:
83 - ephemeral-storage: 800Gi
84 - cpu: 125
85 - env:
86 - - name: HF_TOKEN
87 - value: <your-hf-token>
88 - volumeMounts:
89 - - mountPath: /dev/shm
90 - name: dshm
91 - volumes:
92 - - name: dshm
93 - emptyDir:
94 - medium: Memory
95 - sizeLimit: 15Gi
96 - ---
97 - apiVersion: v1
98 - kind: Service
99 - metadata:
100 - name: vllm-leader
101 - spec:
102 - ports:
103 - - name: http
104 - port: 8080
105 - protocol: TCP
106 - targetPort: 8080
107 - selector:
108 - leaderworkerset.sigs.k8s.io/name: vllm
109 - role: leader
110 - type: ClusterIP
111 - ```
15 +Deploy the following yaml file `lws.yaml` (we have examples that use multiprocessing or Ray):
16 +
17 +??? code "lws.yaml"
18 + === "Multiprocessing (default)"
19 + ```yaml
20 + apiVersion: leaderworkerset.x-k8s.io/v1
21 + kind: LeaderWorkerSet
22 + metadata:
23 + name: vllm
24 + spec:
25 + replicas: 1
26 + leaderWorkerTemplate:
27 + size: 2
28 + restartPolicy: RecreateGroupOnPodRestart
29 + leaderTemplate:
30 + metadata:
31 + labels:
32 + role: leader
33 + spec:
34 + containers:
35 + - name: vllm-leader
36 + image: docker.io/vllm/vllm-openai:latest
37 + env:
38 + - name: HF_TOKEN
39 + value: <your-hf-token>
40 + command:
41 + - sh
42 + - -c
43 + - "vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size $(LWS_GROUP_SIZE) --nnodes $(LWS_GROUP_SIZE) --node-rank $(LWS_WORKER_INDEX) --master-addr $(LWS_LEADER_ADDRESS) --port 8080"
44 + resources:
45 + limits:
46 + nvidia.com/gpu: "8"
47 + memory: 1124Gi
48 + ephemeral-storage: 800Gi
49 + requests:
50 + ephemeral-storage: 800Gi
51 + cpu: 125
52 + ports:
53 + - containerPort: 8080
54 + readinessProbe:
55 + tcpSocket:
56 + port: 8080
57 + initialDelaySeconds: 15
58 + periodSeconds: 10
59 + volumeMounts:
60 + - mountPath: /dev/shm
61 + name: dshm
62 + volumes:
63 + - name: dshm
64 + emptyDir:
65 + medium: Memory
66 + sizeLimit: 15Gi
67 + workerTemplate:
68 + spec:
69 + containers:
70 + - name: vllm-worker
71 + image: docker.io/vllm/vllm-openai:latest
72 + command:
73 + - sh
74 + - -c
75 + - "vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size $(LWS_GROUP_SIZE) --nnodes $(LWS_GROUP_SIZE) --node-rank $(LWS_WORKER_INDEX) --master-addr $(LWS_LEADER_ADDRESS) --headless"
76 + resources:
77 + limits:
78 + nvidia.com/gpu: "8"
79 + memory: 1124Gi
80 + ephemeral-storage: 800Gi
81 + requests:
82 + ephemeral-storage: 800Gi
83 + cpu: 125
84 + env:
85 + - name: HF_TOKEN
86 + value: <your-hf-token>
87 + volumeMounts:
88 + - mountPath: /dev/shm
89 + name: dshm
90 + volumes:
91 + - name: dshm
92 + emptyDir:
93 + medium: Memory
94 + sizeLimit: 15Gi
95 + ---
96 + apiVersion: v1
97 + kind: Service
98 + metadata:
99 + name: vllm-leader
100 + spec:
101 + ports:
102 + - name: http
103 + port: 8080
104 + protocol: TCP
105 + targetPort: 8080
106 + selector:
107 + leaderworkerset.sigs.k8s.io/name: vllm
108 + role: leader
109 + type: ClusterIP
110 + ```
111 +
112 + === "Ray"
113 + ```yaml
114 + apiVersion: leaderworkerset.x-k8s.io/v1
115 + kind: LeaderWorkerSet
116 + metadata:
117 + name: vllm
118 + spec:
119 + replicas: 1
120 + leaderWorkerTemplate:
121 + size: 2
122 + restartPolicy: RecreateGroupOnPodRestart
123 + leaderTemplate:
124 + metadata:
125 + labels:
126 + role: leader
127 + spec:
128 + containers:
129 + - name: vllm-leader
130 + image: docker.io/vllm/vllm-openai:latest
131 + env:
132 + - name: HF_TOKEN
133 + value: <your-hf-token>
134 + command:
135 + - sh
136 + - -c
137 + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=$(LWS_GROUP_SIZE);
138 + vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline-parallel-size 2 --distributed-executor-backend ray"
139 + resources:
140 + limits:
141 + nvidia.com/gpu: "8"
142 + memory: 1124Gi
143 + ephemeral-storage: 800Gi
144 + requests:
145 + ephemeral-storage: 800Gi
146 + cpu: 125
147 + ports:
148 + - containerPort: 8080
149 + readinessProbe:
150 + tcpSocket:
151 + port: 8080
152 + initialDelaySeconds: 15
153 + periodSeconds: 10
154 + volumeMounts:
155 + - mountPath: /dev/shm
156 + name: dshm
157 + volumes:
158 + - name: dshm
159 + emptyDir:
160 + medium: Memory
161 + sizeLimit: 15Gi
162 + workerTemplate:
163 + spec:
164 + containers:
165 + - name: vllm-worker
166 + image: docker.io/vllm/vllm-openai:latest
167 + command:
168 + - sh
169 + - -c
170 + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(LWS_LEADER_ADDRESS)"
171 + resources:
172 + limits:
173 + nvidia.com/gpu: "8"
174 + memory: 1124Gi
175 + ephemeral-storage: 800Gi
176 + requests:
177 + ephemeral-storage: 800Gi
178 + cpu: 125
179 + env:
180 + - name: HF_TOKEN
181 + value: <your-hf-token>
182 + volumeMounts:
183 + - mountPath: /dev/shm
184 + name: dshm
185 + volumes:
186 + - name: dshm
187 + emptyDir:
188 + medium: Memory
189 + sizeLimit: 15Gi
190 + ---
191 + apiVersion: v1
192 + kind: Service
193 + metadata:
194 + name: vllm-leader
195 + spec:
196 + ports:
197 + - name: http
198 + port: 8080
199 + protocol: TCP
200 + targetPort: 8080
201 + selector:
202 + leaderworkerset.sigs.k8s.io/name: vllm
203 + role: leader
204 + type: ClusterIP
205 + ```
206
207 ```bash
208 kubectl apply -f lws.yaml
@@ -130,16 +224,37 @@ vllm-0-1 1/1 Running 0 2s
224
225 Verify that the distributed tensor-parallel inference works:
226
133 -```bash
134 -kubectl logs vllm-0 |grep -i "Loading model weights took"
135 -```
227 +=== "Multiprocessing (default)"
228 + ```bash
229 + kubectl logs vllm-0 | grep -i "Model loading"
230 + kubectl logs vllm-0-1 | grep -i "Model loading"
231 + ```
232
137 -Should get something similar to this:
233 + Should get something similar to this:
234
139 -```text
140 -INFO 05-08 03:20:24 model_runner.py:173] Loading model weights took 0.1189 GB
141 -(RayWorkerWrapper pid=169, ip=10.20.0.197) INFO 05-08 03:20:28 model_runner.py:173] Loading model weights took 0.1189 GB
142 -```
235 + POD 0 (PP Rank 0)
236 +
237 + ```text
238 + (Worker_PP0_TP0 pid=601) INFO 04-28 08:16:58 [gpu_model_runner.py:4820] Model loading took 3.82 GiB memory and 157.996399 seconds
239 + ```
240 +
241 + POD 1 (PP Rank 1)
242 +
243 + ```text
244 + (Worker_PP1_TP0 pid=396) INFO 04-28 08:17:09 [gpu_model_runner.py:4820] Model loading took 3.82 GiB memory and 168.878781 seconds
245 + ```
246 +
247 +=== "Ray"
248 + ```bash
249 + kubectl logs vllm-0 | grep -i "Loading model weights took"
250 + ```
251 +
252 + Should get something similar to this:
253 +
254 + ```text
255 + INFO 05-08 03:20:24 model_runner.py:173] Loading model weights took 0.1189 GB
256 + (RayWorkerWrapper pid=169, ip=10.20.0.197) INFO 05-08 03:20:28 model_runner.py:173] Loading model weights took 0.1189 GB
257 + ```
258
259 ## Access ClusterIP service
260
@@ -173,7 +288,6 @@ curl http://localhost:8080/v1/completions \
288 The output should be similar to the following
289
290 ??? console "Output"
176 -
291 ```text
292 {
293 "id": "cmpl-1bb34faba88b43f9862cfbfb2200949d",
docs/deployment/integrations/kthena.md
+249 -115
@@ -64,36 +64,74 @@ A simplified version of the example (`llama-multinode`) looks like:
64 - `spec.replicas: 1` – one `ServingGroup` (one logical model deployment).
65 - `roles`:
66 - `entryTemplate` – defines **leader** pods that run:
67 - - vLLM’s **multi-node cluster bootstrap script** (Ray cluster).
67 + - vLLM’s **multi-node cluster bootstrap script**.
68 - vLLM **OpenAI-compatible API server**.
69 - - `workerTemplate` – defines **worker** pods that join the leader’s Ray cluster.
69 + - `workerTemplate` – defines **worker** pods to join the leader’s Ray cluster (Ray backend) or to join same distributed process group (multiprocessing backend).
70
71 Key points from the example YAML:
72
73 -- **Image**: `vllm/vllm-openai:latest` (matches upstream vLLM images).
74 -- **Command** (leader):
75 -
76 - ```yaml
77 - command:
78 - - sh
79 - - -c
80 - - >
81 - bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2;
82 - vllm serve meta-llama/Llama-3.1-405B-Instruct
83 - --port 8080
84 - --tensor-parallel-size 8
85 - --pipeline-parallel-size 2
86 - ```
87 -
88 -- **Command** (worker):
89 -
90 - ```yaml
91 - command:
92 - - sh
93 - - -c
94 - - >
95 - bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)
96 - ```
73 +Image: `vllm/vllm-openai:latest` (matches upstream vLLM images).
74 +Commands:
75 +
76 +??? code "Yaml"
77 + === "Multiprocessing (default)"
78 + Leader:
79 +
80 + ```yaml
81 + command:
82 + - sh
83 + - -c
84 + - >
85 + vllm serve meta-llama/Llama-3.1-405B-Instruct
86 + --tensor-parallel-size 8
87 + --pipeline-parallel-size 2
88 + --nnodes=2
89 + --node-rank=0
90 + --master-addr=$(ENTRY_ADDRESS)
91 + --port 8080
92 + ```
93 +
94 + Worker:
95 +
96 + ```yaml
97 + command:
98 + - sh
99 + - -c
100 + - >
101 + vllm serve meta-llama/Llama-3.1-405B-Instruct
102 + --tensor-parallel-size 8
103 + --pipeline-parallel-size 2
104 + --nnodes=2
105 + --node-rank=1
106 + --master-addr=$(ENTRY_ADDRESS)
107 + --headless
108 + ```
109 +
110 + === "Ray"
111 + Leader:
112 +
113 + ```yaml
114 + command:
115 + - sh
116 + - -c
117 + - >
118 + bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh
119 + leader --ray_cluster_size=2; python3 -m
120 + vllm.entrypoints.openai.api_server --port 8080 --model
121 + meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8
122 + --pipeline-parallel-size 2
123 + ```
124 +
125 + Worker:
126 +
127 + ```yaml
128 + command:
129 + - sh
130 + - -c
131 + - >
132 + bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh
133 + worker --ray_address=$(ENTRY_ADDRESS)
134 + ```
135
136 ---
137
@@ -111,96 +149,192 @@ kubectl create secret generic hf-token \
149
150 ### 3.2 Apply the `ModelServing`
151
152 +Save one of the following manifests to `modelserving.yaml`:
153 +
154 +??? code "modelserving.yaml"
155 + === "Multiprocessing (default)"
156 + ```yaml
157 + apiVersion: workload.serving.volcano.sh/v1alpha1
158 + kind: ModelServing
159 + metadata:
160 + name: llama-multinode
161 + namespace: default
162 + spec:
163 + schedulerName: volcano
164 + replicas: 1 # group replicas
165 + template:
166 + restartGracePeriodSeconds: 60
167 + gangPolicy:
168 + minRoleReplicas:
169 + 405b: 1
170 + roles:
171 + - name: 405b
172 + replicas: 2
173 + entryTemplate:
174 + spec:
175 + containers:
176 + - name: leader
177 + image: vllm/vllm-openai:latest
178 + env:
179 + - name: HUGGING_FACE_HUB_TOKEN
180 + valueFrom:
181 + secretKeyRef:
182 + name: hf-token
183 + key: HUGGING_FACE_HUB_TOKEN
184 + command:
185 + - sh
186 + - -c
187 + - "vllm serve meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size 2 --nnodes 2 --node-rank 0 --master-addr $(ENTRY_ADDRESS) --distributed-executor-backend mp --port 8080"
188 + resources:
189 + limits:
190 + nvidia.com/gpu: "8"
191 + memory: 1124Gi
192 + ephemeral-storage: 800Gi
193 + requests:
194 + ephemeral-storage: 800Gi
195 + cpu: 125
196 + ports:
197 + - containerPort: 8080
198 + readinessProbe:
199 + tcpSocket:
200 + port: 8080
201 + initialDelaySeconds: 15
202 + periodSeconds: 10
203 + volumeMounts:
204 + - mountPath: /dev/shm
205 + name: dshm
206 + volumes:
207 + - name: dshm
208 + emptyDir:
209 + medium: Memory
210 + sizeLimit: 15Gi
211 + workerReplicas: 1
212 + workerTemplate:
213 + spec:
214 + containers:
215 + - name: worker
216 + image: vllm/vllm-openai:latest
217 + command:
218 + - sh
219 + - -c
220 + - "vllm serve meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size 2 --nnodes 2 --node-rank 1 --master-addr $(ENTRY_ADDRESS) --distributed-executor-backend mp --headless"
221 + resources:
222 + limits:
223 + nvidia.com/gpu: "8"
224 + memory: 1124Gi
225 + ephemeral-storage: 800Gi
226 + requests:
227 + ephemeral-storage: 800Gi
228 + cpu: 125
229 + env:
230 + - name: HUGGING_FACE_HUB_TOKEN
231 + valueFrom:
232 + secretKeyRef:
233 + name: hf-token
234 + key: HUGGING_FACE_HUB_TOKEN
235 + volumeMounts:
236 + - mountPath: /dev/shm
237 + name: dshm
238 + volumes:
239 + - name: dshm
240 + emptyDir:
241 + medium: Memory
242 + sizeLimit: 15Gi
243 + ```
244 +
245 + === "Ray"
246 + ```yaml
247 + apiVersion: workload.serving.volcano.sh/v1alpha1
248 + kind: ModelServing
249 + metadata:
250 + name: llama-multinode
251 + namespace: default
252 + spec:
253 + schedulerName: volcano
254 + replicas: 1 # group replicas
255 + template:
256 + restartGracePeriodSeconds: 60
257 + gangPolicy:
258 + minRoleReplicas:
259 + 405b: 1
260 + roles:
261 + - name: 405b
262 + replicas: 2
263 + entryTemplate:
264 + spec:
265 + containers:
266 + - name: leader
267 + image: vllm/vllm-openai:latest
268 + env:
269 + - name: HUGGING_FACE_HUB_TOKEN
270 + valueFrom:
271 + secretKeyRef:
272 + name: hf-token
273 + key: HUGGING_FACE_HUB_TOKEN
274 + command:
275 + - sh
276 + - -c
277 + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2;
278 + vllm serve meta-llama/Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline-parallel-size 2"
279 + resources:
280 + limits:
281 + nvidia.com/gpu: "8"
282 + memory: 1124Gi
283 + ephemeral-storage: 800Gi
284 + requests:
285 + ephemeral-storage: 800Gi
286 + cpu: 125
287 + ports:
288 + - containerPort: 8080
289 + readinessProbe:
290 + tcpSocket:
291 + port: 8080
292 + initialDelaySeconds: 15
293 + periodSeconds: 10
294 + volumeMounts:
295 + - mountPath: /dev/shm
296 + name: dshm
297 + volumes:
298 + - name: dshm
299 + emptyDir:
300 + medium: Memory
301 + sizeLimit: 15Gi
302 + workerReplicas: 1
303 + workerTemplate:
304 + spec:
305 + containers:
306 + - name: worker
307 + image: vllm/vllm-openai:latest
308 + command:
309 + - sh
310 + - -c
311 + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)"
312 + resources:
313 + limits:
314 + nvidia.com/gpu: "8"
315 + memory: 1124Gi
316 + ephemeral-storage: 800Gi
317 + requests:
318 + ephemeral-storage: 800Gi
319 + cpu: 125
320 + env:
321 + - name: HUGGING_FACE_HUB_TOKEN
322 + valueFrom:
323 + secretKeyRef:
324 + name: hf-token
325 + key: HUGGING_FACE_HUB_TOKEN
326 + volumeMounts:
327 + - mountPath: /dev/shm
328 + name: dshm
329 + volumes:
330 + - name: dshm
331 + emptyDir:
332 + medium: Memory
333 + sizeLimit: 15Gi
334 + ```
335 +
336 ```bash
115 -cat <<EOF | kubectl apply -f -
116 -apiVersion: workload.serving.volcano.sh/v1alpha1
117 -kind: ModelServing
118 -metadata:
119 - name: llama-multinode
120 - namespace: default
121 -spec:
122 - schedulerName: volcano
123 - replicas: 1 # group replicas
124 - template:
125 - restartGracePeriodSeconds: 60
126 - gangPolicy:
127 - minRoleReplicas:
128 - 405b: 1
129 - roles:
130 - - name: 405b
131 - replicas: 2
132 - entryTemplate:
133 - spec:
134 - containers:
135 - - name: leader
136 - image: vllm/vllm-openai:latest
137 - env:
138 - - name: HUGGING_FACE_HUB_TOKEN
139 - valueFrom:
140 - secretKeyRef:
141 - name: hf-token
142 - key: HUGGING_FACE_HUB_TOKEN
143 - command:
144 - - sh
145 - - -c
146 - - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2;
147 - vllm serve meta-llama/Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline-parallel-size 2"
148 - resources:
149 - limits:
150 - nvidia.com/gpu: "8"
151 - memory: 1124Gi
152 - ephemeral-storage: 800Gi
153 - requests:
154 - ephemeral-storage: 800Gi
155 - cpu: 125
156 - ports:
157 - - containerPort: 8080
158 - readinessProbe:
159 - tcpSocket:
160 - port: 8080
161 - initialDelaySeconds: 15
162 - periodSeconds: 10
163 - volumeMounts:
164 - - mountPath: /dev/shm
165 - name: dshm
166 - volumes:
167 - - name: dshm
168 - emptyDir:
169 - medium: Memory
170 - sizeLimit: 15Gi
171 - workerReplicas: 1
172 - workerTemplate:
173 - spec:
174 - containers:
175 - - name: worker
176 - image: vllm/vllm-openai:latest
177 - command:
178 - - sh
179 - - -c
180 - - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)"
181 - resources:
182 - limits:
183 - nvidia.com/gpu: "8"
184 - memory: 1124Gi
185 - ephemeral-storage: 800Gi
186 - requests:
187 - ephemeral-storage: 800Gi
188 - cpu: 125
189 - env:
190 - - name: HUGGING_FACE_HUB_TOKEN
191 - valueFrom:
192 - secretKeyRef:
193 - name: hf-token
194 - key: HUGGING_FACE_HUB_TOKEN
195 - volumeMounts:
196 - - mountPath: /dev/shm
197 - name: dshm
198 - volumes:
199 - - name: dshm
200 - emptyDir:
201 - medium: Memory
202 - sizeLimit: 15Gi
203 -EOF
337 +kubectl apply -f modelserving.yaml
338 ```
339
340 Kthena will:
examples/ray_serving/multi-node-serving.sh
+1 -1
@@ -11,7 +11,7 @@
11 # Example usage:
12 # On the head node machine, start the Ray head node process and run a vLLM server.
13 # ./multi-node-serving.sh leader --ray_port=6379 --ray_cluster_size=<SIZE> [<extra ray args>] && \
14 -# vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline_parallel_size 2
14 +# vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline-parallel-size 2 --distributed-executor-backend ray
15 #
16 # On each worker node, start the Ray worker node process.
17 # ./multi-node-serving.sh worker --ray_address=<HEAD_NODE_IP> --ray_port=6379 [<extra ray args>]