A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Make handling of SamplingParams clearer in n>1 case (#26032)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Oct 1, 2025 at 19:31 UTC
169313b9f86ec469acbfdac89b9648b3f6513609
1 file changed
+11
-5
vllm/v1/engine/async_llm.py
+11
-5
@@ -289,13 +289,19 @@ class AsyncLLM(EngineClient):
289
await self._add_request(request, prompt_str, None, 0, queue)
290
return queue
291
292
+ # Get the updated SamplingParams from the request, which
293
+ # were cloned/updated in processor.process_inputs above.
294
+ parent_params = request.sampling_params
295
+ assert parent_params is not None
296
+
297
# Fan out child requests (for n>1).
293
- parent_request = ParentRequest(request_id, request.sampling_params)
294
- for idx in range(params.n):
295
- request_id, params = parent_request.get_child_info(idx)
296
- child_request = request if idx == params.n - 1 else copy(request)
298
+ parent_request = ParentRequest(request_id, parent_params)
299
+ for idx in range(parent_params.n):
300
+ request_id, child_params = parent_request.get_child_info(idx)
301
+ child_request = request if idx == parent_params.n - 1 else copy(
302
+ request)
303
child_request.request_id = request_id
298
- child_request.sampling_params = params
304
+ child_request.sampling_params = child_params
305
await self._add_request(child_request, prompt_str, parent_request,
306
idx, queue)
307
return queue