A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 18:30 UTC
[EPD] update EPD script arguments (#36742)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
liuzhenwei committed
Mar 31, 2026 at 20:02 UTC
0c637391359e53c7b41e99ebe7188bd6dd097b8f
3 files changed
+58
-15
examples/online_serving/disaggregated_encoder/README.md
+5
@@ -26,8 +26,13 @@ MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash disagg_1e1p1d_example.sh
26
27
# Use specific storage path
28
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash disagg_1e1p1d_example.sh
29
+
30
+# Run on XPU; scripts switch from CUDA_VISIBLE_DEVICES to ZE_AFFINITY_MASK
31
+DEVICE_PLATFORM=xpu GPU_E=0 GPU_PD=1 bash disagg_1e1pd_example.sh
32
```
33
34
+`DEVICE_PLATFORM` defaults to `cuda`. Set `DEVICE_PLATFORM=xpu` when running these examples on Intel GPUs so the scripts use `ZE_AFFINITY_MASK` instead of `CUDA_VISIBLE_DEVICES` for device selection.
35
+
36
## Encoder Instances
37
38
Encoder engines should be launched with the following flags:
examples/online_serving/disaggregated_encoder/disagg_1e1p1d_example.sh
+29
-9
@@ -19,11 +19,29 @@ GPU_E="${GPU_E:-2}"
19
GPU_P="${GPU_P:-2}"
20
GPU_D="${GPU_D:-3}"
21
22
+# Device platform and affinity env name.
23
+# DEVICE_PLATFORM supports: cuda, xpu
24
+DEVICE_PLATFORM="${DEVICE_PLATFORM:-cuda}"
25
+if [[ -z "${DEVICE_AFFINITY_ENV:-}" ]]; then
26
+ if [[ "${DEVICE_PLATFORM,,}" == "xpu" ]]; then
27
+ DEVICE_AFFINITY_ENV="ZE_AFFINITY_MASK"
28
+ else
29
+ DEVICE_AFFINITY_ENV="CUDA_VISIBLE_DEVICES"
30
+ fi
31
+fi
32
+
33
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_cache}"
34
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-12000}" # wait_for_server timeout
35
36
NUM_PROMPTS="${NUM_PROMPTS:-100}" # number of prompts to send in benchmark
37
38
+# Serve args
39
+GPU_MEMORY_UTILIZATION_E="${GPU_MEMORY_UTILIZATION_E:-0.01}"
40
+GPU_MEMORY_UTILIZATION_P="${GPU_MEMORY_UTILIZATION_P:-0.7}"
41
+GPU_MEMORY_UTILIZATION_D="${GPU_MEMORY_UTILIZATION_D:-0.7}"
42
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
43
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}"
44
+
45
export UCX_TLS=all
46
export UCX_NET_DEVICES=all
47
@@ -92,14 +110,14 @@ mkdir -p "$EC_SHARED_STORAGE_PATH"
110
###############################################################################
111
# Encoder worker
112
###############################################################################
95
-CUDA_VISIBLE_DEVICES="$GPU_E" vllm serve "$MODEL" \
96
- --gpu-memory-utilization 0.01 \
113
+env "$DEVICE_AFFINITY_ENV=$GPU_E" vllm serve "$MODEL" \
114
+ --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION_E" \
115
--port "$ENCODE_PORT" \
116
--enforce-eager \
117
--enable-request-id-headers \
118
--no-enable-prefix-caching \
119
--max-num-batched-tokens 114688 \
102
- --max-num-seqs 128 \
120
+ --max-num-seqs "$MAX_NUM_SEQS" \
121
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
122
--ec-transfer-config '{
123
"ec_connector": "ECExampleConnector",
@@ -115,15 +133,16 @@ PIDS+=($!)
133
###############################################################################
134
# Prefill worker
135
###############################################################################
118
-CUDA_VISIBLE_DEVICES="$GPU_P" \
136
+env "$DEVICE_AFFINITY_ENV=$GPU_P" \
137
UCX_NET_DEVICES=all \
138
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
139
vllm serve "$MODEL" \
122
- --gpu-memory-utilization 0.7 \
140
+ --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION_P" \
141
--port "$PREFILL_PORT" \
142
--enforce-eager \
143
--enable-request-id-headers \
126
- --max-num-seqs 128 \
144
+ --max-num-seqs "$MAX_NUM_SEQS" \
145
+ --max-model-len "$MAX_MODEL_LEN" \
146
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
147
--ec-transfer-config '{
148
"ec_connector": "ECExampleConnector",
@@ -143,15 +162,16 @@ PIDS+=($!)
162
###############################################################################
163
# Decode worker
164
###############################################################################
146
-CUDA_VISIBLE_DEVICES="$GPU_D" \
165
+env "$DEVICE_AFFINITY_ENV=$GPU_D" \
166
UCX_NET_DEVICES=all \
167
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
168
vllm serve "$MODEL" \
150
- --gpu-memory-utilization 0.7 \
169
+ --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION_D" \
170
--port "$DECODE_PORT" \
171
--enforce-eager \
172
--enable-request-id-headers \
154
- --max-num-seqs 128 \
173
+ --max-num-seqs "$MAX_NUM_SEQS" \
174
+ --max-model-len "$MAX_MODEL_LEN" \
175
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
176
--kv-transfer-config '{
177
"kv_connector": "NixlConnector",
examples/online_serving/disaggregated_encoder/disagg_1e1pd_example.sh
+24
-6
@@ -17,11 +17,28 @@ PROXY_PORT="${PROXY_PORT:-10001}"
17
GPU_E="${GPU_E:-0}"
18
GPU_PD="${GPU_PD:-1}"
19
20
+# Device platform and affinity env name.
21
+# DEVICE_PLATFORM supports: cuda, xpu
22
+DEVICE_PLATFORM="${DEVICE_PLATFORM:-cuda}"
23
+if [[ -z "${DEVICE_AFFINITY_ENV:-}" ]]; then
24
+ if [[ "${DEVICE_PLATFORM,,}" == "xpu" ]]; then
25
+ DEVICE_AFFINITY_ENV="ZE_AFFINITY_MASK"
26
+ else
27
+ DEVICE_AFFINITY_ENV="CUDA_VISIBLE_DEVICES"
28
+ fi
29
+fi
30
+
31
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_cache}"
32
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-12000}" # wait_for_server timeout
33
34
NUM_PROMPTS="${NUM_PROMPTS:-100}" # number of prompts to send in benchmark
35
36
+# Serve args
37
+GPU_MEMORY_UTILIZATION_E="${GPU_MEMORY_UTILIZATION_E:-0.01}"
38
+GPU_MEMORY_UTILIZATION_PD="${GPU_MEMORY_UTILIZATION_PD:-0.7}"
39
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
40
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}"
41
+
42
###############################################################################
43
# Helpers
44
###############################################################################
@@ -86,14 +103,14 @@ mkdir -p "$EC_SHARED_STORAGE_PATH"
103
###############################################################################
104
# Encoder worker
105
###############################################################################
89
-CUDA_VISIBLE_DEVICES="$GPU_E" vllm serve "$MODEL" \
90
- --gpu-memory-utilization 0.01 \
106
+env "$DEVICE_AFFINITY_ENV=$GPU_E" vllm serve "$MODEL" \
107
+ --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION_E" \
108
--port "$ENCODE_PORT" \
109
--enforce-eager \
110
--enable-request-id-headers \
111
--no-enable-prefix-caching \
112
--max-num-batched-tokens 114688 \
96
- --max-num-seqs 128 \
113
+ --max-num-seqs "$MAX_NUM_SEQS" \
114
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
115
--ec-transfer-config '{
116
"ec_connector": "ECExampleConnector",
@@ -109,12 +126,13 @@ PIDS+=($!)
126
###############################################################################
127
# Prefill+Decode worker
128
###############################################################################
112
-CUDA_VISIBLE_DEVICES="$GPU_PD" vllm serve "$MODEL" \
113
- --gpu-memory-utilization 0.7 \
129
+env "$DEVICE_AFFINITY_ENV=$GPU_PD" vllm serve "$MODEL" \
130
+ --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION_PD" \
131
--port "$PREFILL_DECODE_PORT" \
132
--enforce-eager \
133
--enable-request-id-headers \
117
- --max-num-seqs 128 \
134
+ --max-num-seqs "$MAX_NUM_SEQS" \
135
+ --max-model-len "$MAX_MODEL_LEN" \
136
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
137
--ec-transfer-config '{
138
"ec_connector": "ECExampleConnector",