@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f5a4b6473

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 18:15 UTC

[Bugfix] Validate custom logits processor xargs for online serving (#27560)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Isotr0py committed Nov 6, 2025 at 00:53 UTC 3f5a4b6473ad1948ec49d62abf2af48c7dc1c5c6

18 files changed +232 -49

docs/design/logits_processors.md

+13 -1

@@ -254,7 +254,15 @@ The previous sections alluded to the interfaces which vLLM logits processors mus

254 changes to the batch makeup.

255 """

256 raise NotImplementedError

257 -

257 +

258 + @classmethod

259 + def validate_params(cls, sampling_params: SamplingParams):

260 + """Validate sampling params for this logits processor.

261 +

262 + Raise ValueError for invalid ones.

263 + """

264 + return None

265 +

266 ```

267

268 A vLLM logits processor must subclass `LogitsProcessor` and define (at minimum) the following methods:

@@ -279,6 +287,10 @@ A vLLM logits processor must subclass `LogitsProcessor` and define (at minimum)

287 * Use the `BatchUpdate` members to update logits processor internal state

288 * **Note:** batch update data structure may be `None`, signaling no change to the batch constituents. In this case, the LogitsProcessor might still want to update its state based on the updated `output_token_ids` lists that it could have retained when they were added.

289

290 +* `validate_params(cls, sampling_params: SamplingParams)`:

291 + * Raise `ValueError` if `SamplingParams` has invalid arguments (especially custom arguments) used by logits processor.

292 + * When request is sent to entrypoint, `validate_params()` will validate `SamplingParams` and refuse request with invalid arguments.

293 +

294 ### `BatchUpdate` data structure

295

296 The `BatchUpdate` abstraction models the persistent batch as a list of requests, supporting the following operations to change batch state (note that the order in which the operations are mentioned below reflects the order in which they should be processed in `update_state()`):

docs/features/custom_arguments.md

@@ -4,6 +4,9 @@ You can use vLLM *custom arguments* to pass in arguments which are not part of t

5 Custom arguments can be useful if, for example, you want to use a [custom logits processor](./custom_logitsprocs.md) without modifying the vLLM source code.

7 +!!! note

8 + Make sure your custom logits processor have implemented `validate_params` for custom arguments. Otherwise invalid custom arguments can cause unexpected behaviour.

9 +

10 ## Offline Custom Arguments

12 Custom arguments passed to `SamplingParams.extra_args` as a `dict` will be visible to any code which has access to `SamplingParams`:

docs/features/custom_logitsprocs.md

+33 -9

@@ -18,6 +18,11 @@ In vLLM, logits processors operate at batch granularity. During a given engine s

19 Custom logits processors must subclass `vllm.v1.sample.logits_processor.LogitsProcessor` and define (at minimum) the following methods:

21 +* `validate_params(cls, sampling_params: SamplingParams)`:

22 + * Raise `ValueError` if `SamplingParams` has invalid arguments (especially custom arguments) used by logits processor.

23 + * When request is sent to entrypoint, `validate_params()` will validate `SamplingParams` and refuse request with invalid arguments.

24 + * **Note:** it's important to implement `validate_params()` to prevent invalid parameters for custom logits processor. Otherwise requests with invalid parameters can cause unexpected behaviour in custom logits processor.

25 +

26 * `__init__(self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool)`

27 * `vllm_config`: engine configuration data structure

28 * `device`: hardware accelerator device info

@@ -103,6 +108,14 @@ The contrived example below implements a custom logits processor which consumes

108 class DummyLogitsProcessor(LogitsProcessor):

109 """Fake logit processor to support unit testing and examples"""

110

111 + @classmethod

112 + def validate_params(cls, params: SamplingParams):

113 + target_token: int | None = params.extra_args and params.extra_args.get(

114 + "target_token"

115 + )

116 + if target_token is not None and not isinstance(target_token, int):

117 + raise ValueError(f"target_token value {target_token} is not int")

118 +

119 def __init__(self, vllm_config: "VllmConfig", device: torch.device,

120 is_pin_memory: bool):

121 self.req_info: dict[int, int] = {}

@@ -118,6 +131,7 @@ The contrived example below implements a custom logits processor which consumes

131 # Process added requests.

132 for index, params, _, _ in batch_update.added:

133 assert params is not None

134 + self.validate_params(params)

135 if params.extra_args and (target_token :=

136 params.extra_args.get("target_token")):

137 self.req_info[index] = target_token

@@ -157,6 +171,7 @@ The contrived example below implements a custom logits processor which consumes

171 logits[rows, cols] = values_to_keep

172

173 return logits

174 +

175 ```

176

177 In the rest of this document, we will use `DummyLogitsProcessor` as an example of a custom logits processor.

@@ -180,7 +195,13 @@ RequestLogitsProcessor = Union[

195

196 While request-level logits processors are explicitly *not* supported in the vLLM engine, vLLM *does* provide a convenient process to wrap an existing `Callable` request-level logits processor and create a batch-level logits processor that is compatible with vLLM. The `Callable` must conform to the type annotation above; if your request-level logits processor has a different interface, then in order to wrap it, you may need to modify it or implement an additional wrapper layer to comply with the interface specification above.

197

183 -You can wrap the request-level logits processor by subclassing `AdapterLogitsProcessor` as shown in the example below (in this example, `DummyPerReqLogitsProcessor` is a stand-in for your request-level logits processor which needs to be wrapped.) Override `AdapterLogitsProcessor.is_argmax_invariant(self)` to accurately reflect whether your request-level logits processor may impact which token has the highest-value logit. Override `AdapterLogitsProcessor.new_req_logits_processor(self,params)` to create a new request-level logits processor instance from a `SamplingParams` instance:

198 +You can wrap the request-level logits processor by subclassing `AdapterLogitsProcessor` as shown in the example below (in this example, `DummyPerReqLogitsProcessor` is a stand-in for your request-level logits processor which needs to be wrapped.):

199 +

200 +* Override `AdapterLogitsProcessor.validate_params(cls,params)` to validate request's sampling parameters.

201 +

202 +* Override `AdapterLogitsProcessor.is_argmax_invariant(self)` to accurately reflect whether your request-level logits processor may impact which token has the highest-value logit.

203 +

204 +* Override `AdapterLogitsProcessor.new_req_logits_processor(self,params)` to create a new request-level logits processor instance from a `SamplingParams` instance:

205

206 ??? code "Example of Wrapping a Request-Level Logits Processor"

207

@@ -220,6 +241,16 @@ You can wrap the request-level logits processor by subclassing `AdapterLogitsPro

241 """Example of wrapping a fake request-level logit processor to create a

242 batch-level logits processor"""

243

244 + @classmethod

245 + def validate_params(cls, params: SamplingParams):

246 + target_token: Any | None = params.extra_args and params.extra_args.get(

247 + "target_token"

248 + )

249 + if target_token is not None and not isinstance(target_token, int):

250 + raise ValueError(

251 + f"target_token value {target_token} is not int"

252 + )

253 +

254 def is_argmax_invariant(self) -> bool:

255 return False

256

@@ -240,18 +271,11 @@ You can wrap the request-level logits processor by subclassing `AdapterLogitsPro

271 Returns:

272 `Callable` request logits processor, or None

273 """

243 - target_token: Optional[Any] = params.extra_args and params.extra_args.get(

274 + target_token: Any | None = params.extra_args and params.extra_args.get(

275 "target_token"

276 )

277 if target_token is None:

278 return None

248 - if not isinstance(target_token, int):

249 - logger.warning(

250 - "target_token value %s is not int; not applying logits"

251 - " processor to request.",

252 - target_token,

253 - )

254 - return None

279 return DummyPerReqLogitsProcessor(target_token)

280 ```

281

examples/offline_inference/logits_processor/custom.py

+17 -2

@@ -33,6 +33,8 @@ Output: ' in the hands of the people.\n\nThe future of AI is in the'

33 ------------------------------------------------------------

34 """

36 +from typing import Any

37 +

38 import torch

40 from vllm import LLM, SamplingParams

@@ -48,6 +50,16 @@ from vllm.v1.sample.logits_processor.builtin import process_dict_updates

50 class DummyLogitsProcessor(LogitsProcessor):

51 """Fake logit processor to support unit testing and examples"""

53 + @classmethod

54 + def validate_params(cls, params: SamplingParams):

55 + target_token: Any | None = params.extra_args and params.extra_args.get(

56 + "target_token"

57 + )

58 + if target_token is not None and not isinstance(target_token, int):

59 + raise ValueError(

60 + f"target_token value {target_token} {type(target_token)} is not int"

61 + )

62 +

63 def __init__(

64 self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool

65 ):

@@ -57,14 +69,17 @@ class DummyLogitsProcessor(LogitsProcessor):

69 return False

71 def update_state(self, batch_update: BatchUpdate | None):

72 + def extract_extra_arg(params: SamplingParams) -> int | None:

73 + self.validate_params(params)

74 + return params.extra_args and params.extra_args.get("target_token")

75 +

76 process_dict_updates(

77 self.req_info,

78 batch_update,

79 # This function returns the LP's per-request state based on the

80 # request details, or None if this LP does not apply to the

81 # request.

66 - lambda params, _, __: params.extra_args

67 - and (params.extra_args.get("target_token")),

82 + lambda params, _, __: extract_extra_arg(params),

83 )

85 def apply(self, logits: torch.Tensor) -> torch.Tensor:

examples/offline_inference/logits_processor/custom_req.py

+8 -7

@@ -76,6 +76,14 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):

76 """Example of wrapping a fake request-level logit processor to create a

77 batch-level logits processor"""

79 + @classmethod

80 + def validate_params(cls, params: SamplingParams):

81 + target_token: Any | None = params.extra_args and params.extra_args.get(

82 + "target_token"

83 + )

84 + if target_token is not None and not isinstance(target_token, int):

85 + raise ValueError(f"target_token value {target_token} is not int")

86 +

87 def is_argmax_invariant(self) -> bool:

88 return False

@@ -101,13 +109,6 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):

109 )

110 if target_token is None:

111 return None

104 - if not isinstance(target_token, int):

105 - logger.warning(

106 - "target_token value %s is not int; not applying logits"

107 - " processor to request.",

108 - target_token,

109 - )

110 - return None

112 return DummyPerReqLogitsProcessor(target_token)

113

114

examples/offline_inference/logits_processor/custom_req_init.py

+8 -7

@@ -77,6 +77,14 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):

77 """Example of overriding the wrapper class `__init__()` in order to utilize

78 info about the device type"""

80 + @classmethod

81 + def validate_params(cls, params: SamplingParams):

82 + target_token = params.extra_args and params.extra_args.get("target_token")

83 + if target_token is not None and not isinstance(target_token, int):

84 + raise ValueError(

85 + f"`target_token` has to be an integer, got {target_token}."

86 + )

87 +

88 def __init__(

89 self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool

90 ):

@@ -113,13 +121,6 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):

121 is None

122 ):

123 return None

116 - if not isinstance(target_token, int):

117 - logger.warning(

118 - "target_token value %s is not int; not applying logits"

119 - " processor to request.",

120 - target_token,

121 - )

122 - return None

124 return DummyPerReqLogitsProcessor(target_token)

125

126

tests/entrypoints/openai/test_lora_resolvers.py

@@ -40,6 +40,7 @@ class MockModelConfig:

40 tokenizer_revision: str | None = None

41 multimodal_config: MultiModalConfig = field(default_factory=MultiModalConfig)

42 hf_config: MockHFConfig = field(default_factory=MockHFConfig)

43 + logits_processors: list[str] | None = None

44 logits_processor_pattern: str | None = None

45 diff_sampling_param: dict | None = None

46 allowed_local_media_path: str = ""

tests/entrypoints/openai/test_serving_chat.py

@@ -353,6 +353,7 @@ class MockModelConfig:

353 tokenizer_revision = None

354 multimodal_config = MultiModalConfig()

355 hf_config = MockHFConfig()

356 + logits_processors: list[str] | None = None

357 logits_processor_pattern = None

358 diff_sampling_param: dict | None = None

359 allowed_local_media_path: str = ""

tests/v1/logits_processors/test_custom_online.py

+29

@@ -177,3 +177,32 @@ async def test_custom_logitsprocs(client: openai.AsyncOpenAI, model_name: str):

177

178 # Alternate whether to activate dummy logitproc for each request

179 use_dummy_logitproc = not use_dummy_logitproc

180 +

181 +

182 +@pytest.mark.asyncio

183 +@pytest.mark.parametrize(

184 + "model_name",

185 + [MODEL_NAME],

186 +)

187 +async def test_invalid_custom_logitsproc_arg(

188 + client: openai.AsyncOpenAI, model_name: str

189 +):

190 + """Test that request with invalid custom logitsproc is rejected"""

191 +

192 + prompt = "Hello, my name is"

193 + # Pass invalid (non-int) target_token value to dummy logits processor

194 + request_keyword_args: dict[str, Any] = {

195 + **api_keyword_args,

196 + "extra_body": {

197 + "vllm_xargs": {DUMMY_LOGITPROC_ARG: "invalid_target_token_value"}

198 + },

199 + }

200 +

201 + with pytest.raises(openai.OpenAIError) as exc_info:

202 + await client.completions.create(

203 + model=model_name,

204 + prompt=prompt,

205 + **request_keyword_args,

206 + )

207 +

208 + assert "is not int" in str(exc_info.value)

tests/v1/logits_processors/utils.py

+15 -2

@@ -52,6 +52,16 @@ prompts = [

52 class DummyLogitsProcessor(LogitsProcessor):

53 """Fake logit processor to support unit testing and examples"""

55 + @classmethod

56 + def validate_params(cls, params: SamplingParams):

57 + target_token: int | None = params.extra_args and params.extra_args.get(

58 + "target_token"

59 + )

60 + if target_token is not None and not isinstance(target_token, int):

61 + raise ValueError(

62 + f"target_token value {target_token} {type(target_token)} is not int"

63 + )

64 +

65 def __init__(

66 self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool

67 ):

@@ -62,11 +72,14 @@ class DummyLogitsProcessor(LogitsProcessor):

72 return False

74 def update_state(self, batch_update: BatchUpdate | None):

75 + def extract_extra_arg(params: SamplingParams) -> int | None:

76 + self.validate_params(params)

77 + return params.extra_args and params.extra_args.get("target_token")

78 +

79 process_dict_updates(

80 self.req_info,

81 batch_update,

68 - lambda params, _, __: params.extra_args

69 - and (params.extra_args.get("target_token")),

82 + lambda params, _, __: extract_extra_arg(params),

83 )

85 def apply(self, logits: torch.Tensor) -> torch.Tensor:

vllm/entrypoints/openai/protocol.py

+2 -2

@@ -772,10 +772,10 @@ class ChatCompletionRequest(OpenAIBaseModel):

772 description="KVTransfer parameters used for disaggregated serving.",

773 )

774

775 - vllm_xargs: dict[str, str | int | float] | None = Field(

776 default=None,

777 description=(

778 - "Additional request parameters with string or "

778 + "Additional request parameters with (list of) string or "

779 "numeric values, used by custom extensions."

780 ),

781 )

vllm/entrypoints/openai/serving_chat.py

@@ -68,6 +68,7 @@ from vllm.transformers_utils.tokenizers import (

68 validate_request_params,

69 )

70 from vllm.utils.collection_utils import as_list

71 +from vllm.v1.sample.logits_processor import validate_logits_processors_parameters

73 logger = init_logger(__name__)

@@ -107,6 +108,9 @@ class OpenAIServingChat(OpenAIServing):

108 self.trust_request_chat_template = trust_request_chat_template

109 self.enable_log_outputs = enable_log_outputs

110

111 + # set up logits processors

112 + self.logits_processors = self.model_config.logits_processors

113 +

114 # set up reasoning parser

115 self.reasoning_parser = self._get_reasoning_parser(

116 reasoning_parser_name=reasoning_parser

@@ -291,6 +295,10 @@ class OpenAIServingChat(OpenAIServing):

295 self.model_config.logits_processor_pattern,

296 self.default_sampling_params,

297 )

298 + validate_logits_processors_parameters(

299 + self.logits_processors,

300 + sampling_params,

301 + )

302

303 self._log_inputs(

304 request_id,

vllm/entrypoints/openai/serving_completion.py

@@ -36,6 +36,7 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams

36 from vllm.transformers_utils.tokenizer import AnyTokenizer

37 from vllm.utils.async_utils import merge_async_iterators

38 from vllm.utils.collection_utils import as_list

39 +from vllm.v1.sample.logits_processor import validate_logits_processors_parameters

41 logger = init_logger(__name__)

@@ -59,6 +60,10 @@ class OpenAIServingCompletion(OpenAIServing):

60 return_tokens_as_token_ids=return_tokens_as_token_ids,

61 log_error_stack=log_error_stack,

62 )

63 +

64 + # set up logits processors

65 + self.logits_processors = self.model_config.logits_processors

66 +

67 self.enable_prompt_tokens_details = enable_prompt_tokens_details

68 self.default_sampling_params = self.model_config.get_diff_sampling_param()

69 self.enable_force_include_usage = enable_force_include_usage

@@ -181,6 +186,10 @@ class OpenAIServingCompletion(OpenAIServing):

186 self.model_config.logits_processor_pattern,

187 self.default_sampling_params,

188 )

189 + validate_logits_processors_parameters(

190 + self.logits_processors,

191 + sampling_params,

192 + )

193

194 request_id_item = f"{request_id}-{i}"

195

vllm/model_executor/models/deepseek_ocr.py

+23 -17

@@ -131,25 +131,18 @@ class NGramPerReqLogitsProcessor(AdapterLogitsProcessor):

131 """Example of overriding the wrapper class `__init__()` in order to utilize

132 info about the device type"""

133

134 - def __init__(

135 - self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool

136 - ):

137 - super().__init__(vllm_config, device, is_pin_memory)

138 -

139 - def is_argmax_invariant(self) -> bool:

140 - return True

141 -

142 - def new_req_logits_processor(

143 - self,

144 - params: SamplingParams,

145 - ) -> RequestLogitsProcessor | None:

134 + @classmethod

135 + def validate_params(cls, params: SamplingParams):

136 ngram_size = params.extra_args and params.extra_args.get("ngram_size")

137 window_size = params.extra_args and params.extra_args.get("window_size", 100)

138 whitelist_token_ids = params.extra_args and params.extra_args.get(

139 "whitelist_token_ids", None

140 )

141 + # if ngram_size is not provided, skip validation because the processor

142 + # will not be used.

143 if ngram_size is None:

144 return None

145 +

146 if not isinstance(ngram_size, int) or ngram_size <= 0:

147 raise ValueError(

148 f"`ngram_size` has to be a strictly positive integer, got {ngram_size}."

@@ -163,13 +156,26 @@ class NGramPerReqLogitsProcessor(AdapterLogitsProcessor):

156 whitelist_token_ids, Iterable

157 ):

158 raise ValueError(

166 - "`whitelist_token_ids` has to be a set of integers, "

159 + "`whitelist_token_ids` has to be a sequence of integers, "

160 f"got {whitelist_token_ids}."

161 )

169 - else:

170 - whitelist_token_ids = (

171 - set(whitelist_token_ids) if whitelist_token_ids else None

172 - )

162 +

163 + def is_argmax_invariant(self) -> bool:

164 + return True

165 +

166 + def new_req_logits_processor(

167 + self,

168 + params: SamplingParams,

169 + ) -> RequestLogitsProcessor | None:

170 + ngram_size = params.extra_args and params.extra_args.get("ngram_size")

171 + window_size = params.extra_args and params.extra_args.get("window_size", 100)

172 + whitelist_token_ids = params.extra_args and params.extra_args.get(

173 + "whitelist_token_ids", None

174 + )

175 + if ngram_size is None:

176 + return None

177 +

178 + whitelist_token_ids = set(whitelist_token_ids) if whitelist_token_ids else None

179 return NoRepeatNGramLogitsProcessor(

180 ngram_size=ngram_size,

181 window_size=window_size,

vllm/transformers_utils/configs/deepseek_vl2.py

@@ -218,3 +218,9 @@ class DeepseekVLV2Config(PretrainedConfig):

218 self.global_view_pos = global_view_pos

219 self.candidate_resolutions = candidate_resolutions

220 self.vocab_size = self.text_config.vocab_size

221 +

222 + # update model_type for OCR model

223 + if "DeepseekOCRForCausalLM" in (

224 + self.architectures or kwargs.get("architectures", [])

225 + ):

226 + self.model_type = "deepseek_ocr"

vllm/utils/torch_utils.py

+28

@@ -2,6 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import contextlib

4 import importlib.metadata

5 +import os

6 import threading

7 from collections.abc import Callable, Collection

8 from functools import lru_cache

@@ -68,6 +69,33 @@ def set_default_torch_num_threads(num_threads: int):

69 torch.set_num_threads(old_num_threads)

72 +@contextlib.contextmanager

73 +def guard_cuda_initialization():

74 + """Avoid unexpected CUDA initialization."""

75 + from vllm.platforms import current_platform

76 +

77 + if not current_platform.is_cuda():

78 + yield

79 + return

80 +

81 + had_key = "CUDA_VISIBLE_DEVICES" in os.environ

82 + old_value = os.environ.get("CUDA_VISIBLE_DEVICES")

83 + os.environ["CUDA_VISIBLE_DEVICES"] = ""

84 + try:

85 + yield

86 + except Exception as e:

87 + if "No CUDA GPUs are available" in str(e):

88 + err_msg = "CUDA initialization is blocked."

89 + else:

90 + err_msg = str(e)

91 + raise RuntimeError(err_msg) from e

92 + finally:

93 + if had_key:

94 + os.environ["CUDA_VISIBLE_DEVICES"] = old_value

95 + else:

96 + os.environ.pop("CUDA_VISIBLE_DEVICES")

97 +

98 +

99 def get_dtype_size(dtype: torch.dtype) -> int:

100 """Get the size of the data type in bytes."""

101 return torch.tensor([], dtype=dtype).element_size()

vllm/v1/sample/logits_processor/__init__.py

+20 -2

@@ -13,6 +13,7 @@ import torch

13 from vllm.logger import init_logger

14 from vllm.logits_process import LogitsProcessor as RequestLogitsProcessor

15 from vllm.sampling_params import SamplingParams

16 +from vllm.utils.torch_utils import guard_cuda_initialization

17 from vllm.v1.sample.logits_processor.builtin import (

18 LogitBiasLogitsProcessor,

19 MinPLogitsProcessor,

@@ -72,8 +73,10 @@ def _load_logitsprocs_plugins() -> list[type[LogitsProcessor]]:

73 entrypoint.name,

74 entrypoint.value,

75 )

75 - classes.append(entrypoint.load())

76 + with guard_cuda_initialization():

77 + classes.append(entrypoint.load())

78 except Exception as e:

79 + logger.error("Failed to load LogitsProcessor plugin %s: %s", entrypoint, e)

80 raise RuntimeError(

81 f"Failed to load LogitsProcessor plugin {entrypoint}"

82 ) from e

@@ -126,8 +129,15 @@ def _load_logitsprocs_by_fqcns(

129

130 try:

131 # Load module

129 - module = importlib.import_module(module_path)

132 + with guard_cuda_initialization():

133 + module = importlib.import_module(module_path)

134 except Exception as e:

135 + logger.error(

136 + "Failed to load %sth LogitsProcessor plugin %s: %s",

137 + ldx,

138 + logitproc,

139 + e,

140 + )

141 raise RuntimeError(

142 f"Failed to load {ldx}th LogitsProcessor plugin {logitproc}"

143 ) from e

@@ -206,6 +216,14 @@ def build_logitsprocs(

216 )

217

218

219 +def validate_logits_processors_parameters(

220 + logits_processors: Sequence[str | type[LogitsProcessor]] | None,

221 + sampling_params: SamplingParams,

222 +):

223 + for logits_procs in _load_custom_logitsprocs(logits_processors):

224 + logits_procs.validate_params(sampling_params)

225 +

226 +

227 class AdapterLogitsProcessor(LogitsProcessor):

228 """Wrapper for per-request logits processors

229

vllm/v1/sample/logits_processor/interface.py

@@ -58,6 +58,14 @@ class BatchUpdate:

60 class LogitsProcessor(ABC):

61 + @classmethod

62 + def validate_params(cls, sampling_params: SamplingParams):

63 + """Validate sampling params for this logits processor.

64 +

65 + Raise ValueError for invalid ones.

66 + """

67 + return None

68 +

69 @abstractmethod

70 def __init__(

71 self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool