A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Defensively copy `sampling_params` (#2881)
If the SamplingParams object passed to LLMEngine.add_request() is mutated after it returns, it could affect the async sampling process for that request. Suggested by @Yard1 https://github.com/vllm-project/vllm/pull/2514#discussion_r1490106059
Nick Hill committed
Feb 17, 2024 at 11:18 UTC
185b2c29e241c864c9660ef0b69b5076a370f55e
1 file changed
+3
vllm/engine/llm_engine.py
+3
@@ -464,6 +464,9 @@ class LLMEngine:
464
prompt_token_ids[:prefix_pos], lora_request.lora_int_id
465
if lora_request else 0) if prefix_pos is not None else None
466
467
+ # Defensive copy of SamplingParams, which are used by the sampler
468
+ sampling_params = copy.deepcopy(sampling_params)
469
+
470
# Create the sequence group.
471
seq_group = SequenceGroup(request_id, [seq], sampling_params,
472
arrival_time, lora_request, prefix)