A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 05:30 UTC
[Frontend] Forward X-data-parallel-rank header on /inference/v1/generate (#42330)
Signed-off-by: hallerite <git@hallerite.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
hallerite committed
May 20, 2026 at 10:58 UTC
cb600d1cdbb079ab9432348f128e71c4e2e0a373
1 file changed
+4
vllm/entrypoints/serve/disagg/serving.py
+4
@@ -197,6 +197,9 @@ class ServingTokens(OpenAIServing):
197
else await self._get_trace_headers(raw_request.headers)
198
)
199
200
+ # Extract data_parallel_rank from header (router can inject it)
201
+ data_parallel_rank = self._get_data_parallel_rank(raw_request)
202
+
203
result_generator = self.engine_client.generate(
204
engine_input,
205
sampling_params,
@@ -204,6 +207,7 @@ class ServingTokens(OpenAIServing):
207
lora_request=lora_request,
208
trace_headers=trace_headers,
209
priority=request.priority,
210
+ data_parallel_rank=data_parallel_rank,
211
)
212
213
assert result_generator is not None