@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc5ef333e0

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 22:45 UTC

[Perf] Add skip_clone to SamplingParams for internal request handling (#31041)

Signed-off-by: mgoin <mgoin64@gmail.com>

Michael Goin committed Dec 24, 2025 at 17:35 UTC bc5ef333e09166e26ada29121804a207758c102e

5 files changed +25 -2

vllm/entrypoints/api_server.py

+2 -1

@@ -60,7 +60,8 @@ async def generate(request: Request) -> Response:

60 async def _generate(request_dict: dict, raw_request: Request) -> Response:

61 prompt = request_dict.pop("prompt")

62 stream = request_dict.pop("stream", False)

63 - sampling_params = SamplingParams(**request_dict)

63 + # Since SamplingParams is created fresh per request, safe to skip clone

64 + sampling_params = SamplingParams(**request_dict, skip_clone=True)

65 request_id = random_uuid()

67 assert engine is not None

vllm/entrypoints/llm.py

+4 -1

@@ -642,7 +642,10 @@ class LLM:

642 # following the huggingface transformers implementation

643 # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa

644 beam_search_params = SamplingParams(

645 - logprobs=2 * beam_width, max_tokens=1, temperature=temperature

645 + logprobs=2 * beam_width,

646 + max_tokens=1,

647 + temperature=temperature,

648 + skip_clone=True, # Internal beam search, safe to skip clone

649 )

650 instances: list[BeamSearchInstance] = []

651

vllm/entrypoints/openai/protocol.py

@@ -474,6 +474,7 @@ class ResponsesRequest(OpenAIBaseModel):

474 ),

475 structured_outputs=structured_outputs,

476 logit_bias=self.logit_bias,

477 + skip_clone=True, # Created fresh per request, safe to skip clone

478 )

479

480 def is_include_output_logprobs(self) -> bool:

@@ -876,6 +877,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

877 bad_words=self.bad_words,

878 allowed_token_ids=self.allowed_token_ids,

879 extra_args=extra_args or None,

880 + skip_clone=True, # Created fresh per request, safe to skip clone

881 )

882

883 @model_validator(mode="before")

@@ -1316,6 +1318,7 @@ class CompletionRequest(OpenAIBaseModel):

1318 logit_bias=self.logit_bias,

1319 allowed_token_ids=self.allowed_token_ids,

1320 extra_args=extra_args or None,

1321 + skip_clone=True, # Created fresh per request, safe to skip clone

1322 )

1323

1324 @model_validator(mode="before")

@@ -2182,6 +2185,7 @@ class TranscriptionRequest(OpenAIBaseModel):

2185 if self.stream

2186 else RequestOutputKind.FINAL_ONLY,

2187 extra_args=self.vllm_xargs,

2188 + skip_clone=True, # Created fresh per request, safe to skip clone

2189 )

2190

2191 @model_validator(mode="before")

@@ -2409,6 +2413,7 @@ class TranslationRequest(OpenAIBaseModel):

2413 output_kind=RequestOutputKind.DELTA

2414 if self.stream

2415 else RequestOutputKind.FINAL_ONLY,

2416 + skip_clone=True, # Created fresh per request, safe to skip clone

2417 )

2418

2419 @model_validator(mode="before")

vllm/entrypoints/openai/speech_to_text.py

@@ -219,6 +219,7 @@ class OpenAISpeechToText(OpenAIServing):

219 dummy_params = SamplingParams(

220 max_tokens=1,

221 temperature=0.0,

222 + skip_clone=True, # Internal warmup, safe to skip clone

223 )

224

225 # Process the dummy input through the input processor

vllm/sampling_params.py

+13

@@ -211,6 +211,12 @@ class SamplingParams(

211 set to an integer k, will use only the last k tokens from the prompt

212 (i.e., left truncation). If set to `None`, truncation is disabled."""

213 output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE

214 + skip_clone: bool = False

215 + """Internal flag indicating that this SamplingParams instance is safe to

216 + reuse without cloning. When True, clone() will return self without

217 + performing a deep copy. This should only be set when the params object

218 + is guaranteed to be dedicated to a single request and won't be modified

219 + in ways that would affect other uses."""

220

221 # The below fields are not supposed to be used as an input.

222 # They are set in post_init.

@@ -270,6 +276,7 @@ class SamplingParams(

276 logit_bias: dict[int, float] | dict[str, float] | None = None,

277 allowed_token_ids: list[int] | None = None,

278 extra_args: dict[str, Any] | None = None,

279 + skip_clone: bool = False,

280 ) -> "SamplingParams":

281 if logit_bias is not None:

282 # Convert token_id to integer

@@ -310,6 +317,7 @@ class SamplingParams(

317 logit_bias=logit_bias,

318 allowed_token_ids=allowed_token_ids,

319 extra_args=extra_args,

320 + skip_clone=skip_clone,

321 )

322

323 def __post_init__(self) -> None:

@@ -540,8 +548,13 @@ class SamplingParams(

548 data that is expensive to copy. However, if not copied, the processor

549 needs to support parallel decoding for multiple sequences

550 See https://github.com/vllm-project/vllm/issues/3087

551 +

552 + If skip_clone is True, uses shallow copy instead of deep copy.

553 """

554

555 + if self.skip_clone:

556 + return copy.copy(self)

557 +

558 logit_processor_refs = (

559 None

560 if self.logits_processors is None