@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 29a8d6a554
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Fix] Don't deep-copy LogitsProcessors when copying SamplingParams (#3099)

Nick Hill committed Feb 29, 2024 at 11:20 UTC 29a8d6a554a87292f05b62078976b43a899691e3
2 files changed +18 -2
vllm/engine/llm_engine.py
+3 -2
@@ -484,8 +484,9 @@ class LLMEngine:
484 prompt_token_ids[:prefix_pos], lora_request.lora_int_id
485 if lora_request else 0) if prefix_pos is not None else None
486
487 - # Defensive copy of SamplingParams, which are used by the sampler
488 - sampling_params = copy.deepcopy(sampling_params)
487 + # Defensive copy of SamplingParams, which are used by the sampler,
488 + # this doesn't deep-copy LogitsProcessor objects
489 + sampling_params = sampling_params.clone()
490
491 # Create the sequence group.
492 seq_group = SequenceGroup(request_id, [seq], sampling_params,
vllm/sampling_params.py
+15
@@ -1,4 +1,5 @@
1 """Sampling parameters for text generation."""
2 +import copy
3 from enum import IntEnum
4 from functools import cached_property
5 from typing import Callable, List, Optional, Union
@@ -237,6 +238,20 @@ class SamplingParams:
238 return SamplingType.RANDOM_SEED
239 return SamplingType.RANDOM
240
241 + def clone(self) -> "SamplingParams":
242 + """Deep copy excluding LogitsProcessor objects.
243 +
244 + LogitsProcessor objects are excluded because they may contain an
245 + arbitrary, nontrivial amount of data.
246 + See https://github.com/vllm-project/vllm/issues/3087
247 + """
248 +
249 + logit_processor_refs = None if self.logits_processors is None else {
250 + id(lp): lp
251 + for lp in self.logits_processors
252 + }
253 + return copy.deepcopy(self, memo=logit_processor_refs)
254 +
255 def __repr__(self) -> str:
256 return (
257 f"SamplingParams(n={self.n}, "