A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Fix] Don't deep-copy LogitsProcessors when copying SamplingParams (#3099)
Nick Hill committed
Feb 29, 2024 at 11:20 UTC
29a8d6a554a87292f05b62078976b43a899691e3
2 files changed
+18
-2
vllm/engine/llm_engine.py
+3
-2
@@ -484,8 +484,9 @@ class LLMEngine:
484
prompt_token_ids[:prefix_pos], lora_request.lora_int_id
485
if lora_request else 0) if prefix_pos is not None else None
486
487
- # Defensive copy of SamplingParams, which are used by the sampler
488
- sampling_params = copy.deepcopy(sampling_params)
487
+ # Defensive copy of SamplingParams, which are used by the sampler,
488
+ # this doesn't deep-copy LogitsProcessor objects
489
+ sampling_params = sampling_params.clone()
490
491
# Create the sequence group.
492
seq_group = SequenceGroup(request_id, [seq], sampling_params,
vllm/sampling_params.py
+15
@@ -1,4 +1,5 @@
1
"""Sampling parameters for text generation."""
2
+import copy
3
from enum import IntEnum
4
from functools import cached_property
5
from typing import Callable, List, Optional, Union
@@ -237,6 +238,20 @@ class SamplingParams:
238
return SamplingType.RANDOM_SEED
239
return SamplingType.RANDOM
240
241
+ def clone(self) -> "SamplingParams":
242
+ """Deep copy excluding LogitsProcessor objects.
243
+
244
+ LogitsProcessor objects are excluded because they may contain an
245
+ arbitrary, nontrivial amount of data.
246
+ See https://github.com/vllm-project/vllm/issues/3087
247
+ """
248
+
249
+ logit_processor_refs = None if self.logits_processors is None else {
250
+ id(lp): lp
251
+ for lp in self.logits_processors
252
+ }
253
+ return copy.deepcopy(self, memo=logit_processor_refs)
254
+
255
def __repr__(self) -> str:
256
return (
257
f"SamplingParams(n={self.n}, "