A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 10:45 UTC
[Performance] Cache loaded custom logitsprocs to avoid overheads (#28462)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Nov 12, 2025 at 08:49 UTC
3f770f4427cb926c24af540cc72d1b5901f7f702
1 file changed
+8
-2
vllm/v1/sample/logits_processor/__init__.py
+8
-2
@@ -5,7 +5,7 @@ import inspect
5
import itertools
6
from abc import abstractmethod
7
from collections.abc import Sequence
8
-from functools import partial
8
+from functools import lru_cache, partial
9
from typing import TYPE_CHECKING
10
11
import torch
@@ -216,11 +216,17 @@ def build_logitsprocs(
216
)
217
218
219
+cached_load_custom_logitsprocs = lru_cache(_load_custom_logitsprocs)
220
+
221
+
222
def validate_logits_processors_parameters(
223
logits_processors: Sequence[str | type[LogitsProcessor]] | None,
224
sampling_params: SamplingParams,
225
):
223
- for logits_procs in _load_custom_logitsprocs(logits_processors):
226
+ logits_processors = (
227
+ tuple(logits_processors) if logits_processors is not None else None
228
+ )
229
+ for logits_procs in cached_load_custom_logitsprocs(logits_processors):
230
logits_procs.validate_params(sampling_params)
231
232