@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8d9338fae4

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 09:45 UTC

[Chore] Rename `Processor` to `InputProcessor` (#29682)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 29, 2025 at 01:35 UTC 8d9338fae4ad2b9236daaa87d7c9fb6245b8755f

14 files changed +723 -693

tests/entrypoints/openai/test_lora_resolvers.py

+1 -1

@@ -114,7 +114,7 @@ def mock_serving_setup():

114 mock_engine.add_lora.reset_mock()

115

116 mock_engine.model_config = MockModelConfig()

117 - mock_engine.processor = MagicMock()

117 + mock_engine.input_processor = MagicMock()

118 mock_engine.io_processor = MagicMock()

119

120 models = OpenAIServingModels(

tests/entrypoints/openai/test_serving_chat.py

+7 -7

@@ -429,7 +429,7 @@ async def test_serving_chat_returns_correct_model_name():

429 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

430 mock_engine.errored = False

431 mock_engine.model_config = MockModelConfig()

432 - mock_engine.processor = MagicMock()

432 + mock_engine.input_processor = MagicMock()

433 mock_engine.io_processor = MagicMock()

434

435 serving_chat = _build_serving_chat(mock_engine)

@@ -459,7 +459,7 @@ async def test_serving_chat_should_set_correct_max_tokens():

459 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

460 mock_engine.errored = False

461 mock_engine.model_config = MockModelConfig()

462 - mock_engine.processor = MagicMock()

462 + mock_engine.input_processor = MagicMock()

463 mock_engine.io_processor = MagicMock()

464

465 serving_chat = _build_serving_chat(mock_engine)

@@ -492,7 +492,7 @@ async def test_serving_chat_should_set_correct_max_tokens():

492 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

493 mock_engine.errored = False

494 mock_engine.model_config = mock_model_config

495 - mock_engine.processor = MagicMock()

495 + mock_engine.input_processor = MagicMock()

496 mock_engine.io_processor = MagicMock()

497

498 # Initialize the serving chat

@@ -537,7 +537,7 @@ async def test_serving_chat_should_set_correct_max_tokens():

537 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

538 mock_engine.errored = False

539 mock_engine.model_config = mock_model_config

540 - mock_engine.processor = MagicMock()

540 + mock_engine.input_processor = MagicMock()

541 mock_engine.io_processor = MagicMock()

542

543 # Initialize the serving chat

@@ -583,7 +583,7 @@ async def test_serving_chat_could_load_correct_generation_config():

583 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

584 mock_engine.errored = False

585 mock_engine.model_config = mock_model_config

586 - mock_engine.processor = MagicMock()

586 + mock_engine.input_processor = MagicMock()

587 mock_engine.io_processor = MagicMock()

588

589 # Initialize the serving chat

@@ -629,7 +629,7 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):

629 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

630 mock_engine.errored = False

631 mock_engine.model_config = mock_model_config

632 - mock_engine.processor = MagicMock()

632 + mock_engine.input_processor = MagicMock()

633 mock_engine.io_processor = MagicMock()

634

635 serving_chat = _build_serving_chat(mock_engine)

@@ -662,7 +662,7 @@ async def test_serving_chat_data_parallel_rank_extraction():

662 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

663 mock_engine.errored = False

664 mock_engine.model_config = MockModelConfig()

665 - mock_engine.processor = MagicMock()

665 + mock_engine.input_processor = MagicMock()

666 mock_engine.io_processor = MagicMock()

667

668 # Mock the generate method to return an async generator

tests/entrypoints/openai/test_serving_engine.py

+1 -1

@@ -23,7 +23,7 @@ def serving() -> OpenAIServing:

23 model_config.max_model_len = 32768

24 models = Mock(spec=OpenAIServingModels)

25 models.model_config = model_config

26 - models.processor = Mock()

26 + models.input_processor = Mock()

27 models.io_processor = Mock()

29 serving = OpenAIServing(

tests/entrypoints/openai/test_serving_models.py

+1 -1

@@ -30,7 +30,7 @@ async def _async_serving_models_init() -> OpenAIServingModels:

30 mock_model_config = MagicMock(spec=ModelConfig)

31 mock_model_config.max_model_len = 2048

32 mock_engine_client.model_config = mock_model_config

33 - mock_engine_client.processor = MagicMock()

33 + mock_engine_client.input_processor = MagicMock()

34 mock_engine_client.io_processor = MagicMock()

36 serving_models = OpenAIServingModels(

tests/entrypoints/openai/test_serving_responses.py

+2 -2

@@ -127,7 +127,7 @@ class TestInitializeToolSessions:

127 model_config.get_diff_sampling_param.return_value = {}

128 engine_client.model_config = model_config

129

130 - engine_client.processor = MagicMock()

130 + engine_client.input_processor = MagicMock()

131 engine_client.io_processor = MagicMock()

132

133 models = MagicMock()

@@ -213,7 +213,7 @@ class TestValidateGeneratorInput:

213 model_config.get_diff_sampling_param.return_value = {}

214 engine_client.model_config = model_config

215

216 - engine_client.processor = MagicMock()

216 + engine_client.input_processor = MagicMock()

217 engine_client.io_processor = MagicMock()

218

219 models = MagicMock()

tests/v1/engine/test_process_multi_modal_uuids.py renamed

+18 -17

@@ -7,18 +7,17 @@ from vllm.assets.image import ImageAsset

7 from vllm.assets.video import VideoAsset

8 from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig

9 from vllm.sampling_params import SamplingParams

10 -from vllm.v1.engine import processor as processor_mod

11 -from vllm.v1.engine.processor import Processor

10 +from vllm.v1.engine import input_processor as input_processor_mod

11 +from vllm.v1.engine.input_processor import InputProcessor

13 cherry_pil_image = ImageAsset("cherry_blossom").pil_image

14 stop_pil_image = ImageAsset("stop_sign").pil_image

15 baby_reading_np_ndarrays = VideoAsset("baby_reading").np_ndarrays

18 -# Mock processor for testing

19 -def _mk_processor(

18 +def _mock_input_processor(

19 monkeypatch, *, mm_cache_gb: float = 4.0, enable_prefix_caching: bool = True

21 -) -> Processor:

20 +) -> InputProcessor:

21 """

22 Create a Processor instance with minimal configuration suitable for unit

23 tests without accessing external resources.

@@ -36,7 +35,7 @@ def _mk_processor(

35 raising=True,

36 )

37 monkeypatch.setattr(

39 - processor_mod,

38 + input_processor_mod,

39 "processor_cache_from_config",

40 lambda vllm_config, mm_registry: None,

41 raising=True,

@@ -65,11 +64,11 @@ def _mk_processor(

64 device_config=DeviceConfig(device="cpu"),

65 )

68 - return Processor(vllm_config, tokenizer=None)

67 + return InputProcessor(vllm_config, tokenizer=None)

70 def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):

72 - processor = _mk_processor(monkeypatch)

71 + input_processor = _mock_input_processor(monkeypatch)

73 prompt = {

74 "prompt": "USER: <image>\nDescribe\nASSISTANT:",

@@ -79,7 +78,7 @@ def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):

78 }

80 with pytest.raises(ValueError, match="must have same length as data"):

82 - processor.process_inputs(

81 + input_processor.process_inputs(

82 request_id="req-1",

83 prompt=prompt, # type: ignore[arg-type]

84 params=SamplingParams(),

@@ -87,7 +86,7 @@ def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):

88 def test_multi_modal_uuids_missing_modality_raises(monkeypatch):

90 - processor = _mk_processor(monkeypatch)

89 + input_processor = _mock_input_processor(monkeypatch)

91 prompt = {

92 "prompt": "USER: <image><video>\nDescribe\nASSISTANT:",

@@ -101,7 +100,7 @@ def test_multi_modal_uuids_missing_modality_raises(monkeypatch):

100 }

101

102 with pytest.raises(ValueError, match="must be provided if multi_modal_data"):

104 - processor.process_inputs(

103 + input_processor.process_inputs(

104 request_id="req-2",

105 prompt=prompt, # type: ignore[arg-type]

106 params=SamplingParams(),

@@ -119,7 +118,7 @@ def test_multi_modal_uuids_missing_modality_raises(monkeypatch):

118 def test_multi_modal_uuids_accepts_none_and_passes_through(

119 monkeypatch, mm_cache_gb: float, enable_prefix_caching: bool

120 ):

122 - processor = _mk_processor(

121 + input_processor = _mock_input_processor(

122 monkeypatch,

123 mm_cache_gb=mm_cache_gb,

124 enable_prefix_caching=enable_prefix_caching,

@@ -137,7 +136,7 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(

136

137 # Monkeypatch only the bound preprocess method on this instance

138 monkeypatch.setattr(

140 - processor.input_preprocessor, "preprocess", fake_preprocess, raising=True

139 + input_processor.input_preprocessor, "preprocess", fake_preprocess, raising=True

140 )

141

142 # Use a consistent two-image scenario across all configurations

@@ -151,7 +150,7 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(

150 "multi_modal_uuids": mm_uuids,

151 }

152

154 - processor.process_inputs(

153 + input_processor.process_inputs(

154 request_id="req-3",

155 prompt=prompt, # type: ignore[arg-type]

156 params=SamplingParams(),

@@ -163,7 +162,9 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(

162 def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):

163 # When both processor cache is 0 and prefix caching disabled, the

164 # processor builds overrides from request id instead of using user UUIDs.

166 - processor = _mk_processor(monkeypatch, mm_cache_gb=0.0, enable_prefix_caching=False)

165 + input_processor = _mock_input_processor(

166 + monkeypatch, mm_cache_gb=0.0, enable_prefix_caching=False

167 + )

168

169 captured: dict[str, object] = {}

170

@@ -174,7 +175,7 @@ def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):

175 return {"type": "token", "prompt_token_ids": [1]}

176

177 monkeypatch.setattr(

177 - processor.input_preprocessor, "preprocess", fake_preprocess, raising=True

178 + input_processor.input_preprocessor, "preprocess", fake_preprocess, raising=True

179 )

180

181 request_id = "req-42"

@@ -188,7 +189,7 @@ def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):

189 "multi_modal_uuids": mm_uuids,

190 }

191

191 - processor.process_inputs(

192 + input_processor.process_inputs(

193 request_id=request_id,

194 prompt=prompt, # type: ignore[arg-type]

195 params=SamplingParams(),

vllm/engine/protocol.py

+2 -2

@@ -15,7 +15,7 @@ from vllm.sampling_params import SamplingParams

15 from vllm.tasks import SupportedTask

16 from vllm.transformers_utils.tokenizer import AnyTokenizer

17 from vllm.v1.engine import EngineCoreRequest

18 -from vllm.v1.engine.processor import Processor

18 +from vllm.v1.engine.input_processor import InputProcessor

21 class EngineClient(ABC):

@@ -23,7 +23,7 @@ class EngineClient(ABC):

24 vllm_config: VllmConfig

25 model_config: ModelConfig

26 - processor: Processor

26 + input_processor: InputProcessor

27 io_processor: IOProcessor | None

29 @property

vllm/entrypoints/llm.py

+3 -3

@@ -347,7 +347,7 @@ class LLM:

347 self.supported_tasks = supported_tasks

348

349 self.model_config = self.llm_engine.model_config

350 - self.processor = self.llm_engine.processor

350 + self.input_processor = self.llm_engine.input_processor

351 self.io_processor = self.llm_engine.io_processor

352

353 def get_tokenizer(self) -> AnyTokenizer:

@@ -364,7 +364,7 @@ class LLM:

364 self.llm_engine.tokenizer = get_cached_tokenizer(tokenizer)

365

366 def reset_mm_cache(self) -> None:

367 - self.processor.clear_mm_cache()

367 + self.input_processor.clear_mm_cache()

368 self.llm_engine.reset_mm_cache()

369

370 def get_default_sampling_params(self) -> SamplingParams:

@@ -1674,7 +1674,7 @@ class LLM:

1674 tokenization_kwargs,

1675 )

1676

1677 - engine_request = self.processor.process_inputs(

1677 + engine_request = self.input_processor.process_inputs(

1678 request_id,

1679 engine_prompt,

1680 params,

vllm/entrypoints/openai/serving_engine.py

+5 -5

@@ -284,7 +284,7 @@ class OpenAIServing:

284 self._async_tokenizer_pool: dict[AnyTokenizer, AsyncMicrobatchTokenizer] = {}

285 self.log_error_stack = log_error_stack

286

287 - self.processor = self.models.processor

287 + self.input_processor = self.models.input_processor

288 self.io_processor = self.models.io_processor

289 self.model_config = self.models.model_config

290 self.max_model_len = self.model_config.max_model_len

@@ -330,7 +330,7 @@ class OpenAIServing:

330 return parser

331

332 async def reset_mm_cache(self) -> None:

333 - self.processor.clear_mm_cache()

333 + self.input_processor.clear_mm_cache()

334 await self.engine_client.reset_mm_cache()

335

336 async def beam_search(

@@ -348,8 +348,8 @@ class OpenAIServing:

348 length_penalty = params.length_penalty

349 include_stop_str_in_output = params.include_stop_str_in_output

350

351 - processor = self.processor

352 - tokenizer = processor.tokenizer

351 + input_processor = self.input_processor

352 + tokenizer = input_processor.tokenizer

353 if tokenizer is None:

354 raise ValueError(

355 "You cannot use beam search when `skip_tokenizer_init` is True"

@@ -1214,7 +1214,7 @@ class OpenAIServing:

1214 self.max_model_len, params.truncate_prompt_tokens, tokenization_kwargs

1215 )

1216

1217 - engine_request = self.processor.process_inputs(

1217 + engine_request = self.input_processor.process_inputs(

1218 request_id,

1219 engine_prompt,

1220 params,

vllm/entrypoints/openai/serving_models.py

+1 -1

@@ -69,7 +69,7 @@ class OpenAIServingModels:

69 )

70 self.lora_resolver_lock: dict[str, Lock] = defaultdict(Lock)

72 - self.processor = self.engine_client.processor

72 + self.input_processor = self.engine_client.input_processor

73 self.io_processor = self.engine_client.io_processor

74 self.model_config = self.engine_client.model_config

75 self.max_model_len = self.model_config.max_model_len

vllm/v1/engine/async_llm.py

+17 -12

@@ -10,6 +10,7 @@ from typing import Any, cast

11 import numpy as np

12 import torch

13 +from typing_extensions import deprecated

15 import vllm.envs as envs

16 from vllm.config import VllmConfig

@@ -35,9 +36,9 @@ from vllm.utils.math_utils import cdiv

36 from vllm.v1.engine import EngineCoreRequest

37 from vllm.v1.engine.core_client import EngineCoreClient

38 from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError

39 +from vllm.v1.engine.input_processor import InputProcessor

40 from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollector

41 from vllm.v1.engine.parallel_sampling import ParentRequest

40 -from vllm.v1.engine.processor import Processor

42 from vllm.v1.executor import Executor

43 from vllm.v1.metrics.loggers import (

44 StatLoggerFactory,

@@ -112,7 +113,7 @@ class AsyncLLM(EngineClient):

113 else:

114 tokenizer = init_tokenizer_from_configs(self.model_config)

115

115 - self.processor = Processor(self.vllm_config, tokenizer)

116 + self.input_processor = InputProcessor(self.vllm_config, tokenizer)

117 self.io_processor = get_io_processor(

118 self.vllm_config,

119 self.model_config.io_processor_plugin,

@@ -193,6 +194,14 @@ class AsyncLLM(EngineClient):

194 else:

195 self.profiler = None

196

197 + @property

198 + @deprecated(

199 + "`AsyncLLM.processor` has been renamed to `AsyncLLM.input_processor`. "

200 + "The old name will be removed in v0.13."

201 + )

202 + def processor(self):

203 + return self.input_processor

204 +

205 @classmethod

206 def from_vllm_config(

207 cls,

@@ -293,11 +302,7 @@ class AsyncLLM(EngineClient):

302 request = prompt

303 else:

304 assert prompt_text is None

296 - logger.warning_once(

297 - "Processor has been moved under OpenAIServing and will "

298 - "be removed from AsyncLLM in v0.13."

299 - )

300 - request = self.processor.process_inputs(

305 + request = self.input_processor.process_inputs(

306 request_id,

307 prompt,

308 params,

@@ -481,7 +486,7 @@ class AsyncLLM(EngineClient):

486 output_processor = self.output_processor

487 log_stats = self.log_stats

488 logger_manager = self.logger_manager

484 - processor = self.processor

489 + input_processor = self.input_processor

490

491 async def output_handler():

492 try:

@@ -532,7 +537,7 @@ class AsyncLLM(EngineClient):

537 engine_idx=outputs.engine_index,

538 scheduler_stats=outputs.scheduler_stats,

539 iteration_stats=iteration_stats,

535 - mm_cache_stats=processor.stat_mm_cache(),

540 + mm_cache_stats=input_processor.stat_mm_cache(),

541 )

542 except Exception as e:

543 logger.exception("AsyncLLM output_handler failed.")

@@ -699,11 +704,11 @@ class AsyncLLM(EngineClient):

704

705 @property

706 def tokenizer(self) -> AnyTokenizer | None:

702 - return self.processor.tokenizer

707 + return self.input_processor.tokenizer

708

709 @tokenizer.setter

710 def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:

706 - self.processor.tokenizer = tokenizer

711 + self.input_processor.tokenizer = tokenizer

712

713 async def get_tokenizer(self) -> AnyTokenizer:

714 if self.tokenizer is None:

@@ -738,7 +743,7 @@ class AsyncLLM(EngineClient):

743 await asyncio.gather(*coros)

744

745 async def reset_mm_cache(self) -> None:

741 - self.processor.clear_mm_cache()

746 + self.input_processor.clear_mm_cache()

747 await self.engine_core.reset_mm_cache_async()

748

749 async def reset_prefix_cache(self) -> None:

vllm/v1/engine/input_processor.py new

+637

@@ -0,0 +1,637 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import time

5 +from collections.abc import Mapping

6 +from typing import Any, Literal, cast

7 +

8 +from vllm.config import VllmConfig

9 +from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs

10 +from vllm.inputs.parse import split_enc_dec_inputs

11 +from vllm.inputs.preprocess import InputPreprocessor

12 +from vllm.logger import init_logger

13 +from vllm.lora.request import LoRARequest

14 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

15 +from vllm.multimodal.cache import processor_cache_from_config

16 +from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict

17 +from vllm.multimodal.parse import MultiModalDataParser

18 +from vllm.multimodal.processing import EncDecMultiModalProcessor

19 +from vllm.multimodal.utils import argsort_mm_positions

20 +from vllm.pooling_params import PoolingParams

21 +from vllm.sampling_params import SamplingParams

22 +from vllm.transformers_utils.tokenizer import AnyTokenizer

23 +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

24 +from vllm.utils import length_from_prompt_token_ids_or_embeds

25 +from vllm.v1.engine import EngineCoreRequest

26 +from vllm.v1.metrics.stats import MultiModalCacheStats

27 +from vllm.v1.structured_output.backend_guidance import validate_guidance_grammar

28 +from vllm.v1.structured_output.backend_lm_format_enforcer import (

29 + validate_structured_output_request_lm_format_enforcer,

30 +)

31 +from vllm.v1.structured_output.backend_outlines import (

32 + validate_structured_output_request_outlines,

33 +)

34 +from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar

35 +

36 +logger = init_logger(__name__)

37 +

38 +

39 +class InputProcessor:

40 + def __init__(

41 + self,

42 + vllm_config: VllmConfig,

43 + tokenizer: AnyTokenizer | None,

44 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

45 + ) -> None:

46 + self.vllm_config = vllm_config

47 + self.model_config = vllm_config.model_config

48 + self.cache_config = vllm_config.cache_config

49 + self.lora_config = vllm_config.lora_config

50 + self.structured_outputs_config = vllm_config.structured_outputs_config

51 +

52 + self.generation_config_fields = self.model_config.try_get_generation_config()

53 +

54 + self.mm_registry = mm_registry

55 + self.mm_processor_cache = processor_cache_from_config(vllm_config, mm_registry)

56 +

57 + self.input_preprocessor = InputPreprocessor(

58 + self.model_config,

59 + tokenizer,

60 + mm_registry,

61 + mm_processor_cache=self.mm_processor_cache,

62 + )

63 +

64 + @property

65 + def tokenizer(self) -> AnyTokenizer | None:

66 + return self.input_preprocessor.tokenizer

67 +

68 + @tokenizer.setter

69 + def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:

70 + self.input_preprocessor.tokenizer = tokenizer

71 +

72 + def _validate_logprobs(

73 + self,

74 + params: SamplingParams,

75 + ) -> None:

76 + max_logprobs = self.model_config.max_logprobs

77 + if max_logprobs == -1:

78 + max_logprobs = self.model_config.get_vocab_size()

79 +

80 + # Validate sample logprobs.

81 + if params.logprobs:

82 + num_logprobs = params.logprobs

83 + if num_logprobs == -1:

84 + num_logprobs = self.model_config.get_vocab_size()

85 + if num_logprobs > max_logprobs:

86 + raise ValueError(

87 + f"Requested sample logprobs of {num_logprobs}, "

88 + f"which is greater than max allowed: {max_logprobs}"

89 + )

90 +

91 + # Validate prompt logprobs.

92 + if params.prompt_logprobs:

93 + num_prompt_logprobs = params.prompt_logprobs

94 + if num_prompt_logprobs == -1:

95 + num_prompt_logprobs = self.model_config.get_vocab_size()

96 + if num_prompt_logprobs > max_logprobs:

97 + raise ValueError(

98 + f"Requested prompt logprobs of {num_prompt_logprobs}, "

99 + f"which is greater than max allowed: {max_logprobs}"

100 + )

101 +

102 + def _validate_sampling_params(

103 + self,

104 + params: SamplingParams,

105 + ) -> None:

106 + self._validate_structured_output(params)

107 + self._validate_logit_bias(params)

108 +

109 + if params.allowed_token_ids is None:

110 + return

111 + if not params.allowed_token_ids:

112 + raise ValueError("allowed_token_ids is not None and empty!")

113 + if self.tokenizer is None:

114 + # When skip_tokenizer_init=True, we can't validate token IDs

115 + # Skip validation and let the model handle invalid tokens

116 + return

117 + vocab_size = len(self.tokenizer)

118 + if not all(0 <= tid < vocab_size for tid in params.allowed_token_ids):

119 + raise ValueError("allowed_token_ids contains out-of-vocab token id!")

120 +

121 + def _validate_logit_bias(

122 + self,

123 + params: SamplingParams,

124 + ) -> None:

125 + """Validate logit_bias token IDs are within vocabulary range."""

126 + if not params.logit_bias:

127 + return

128 +

129 + vocab_size = self.model_config.get_vocab_size()

130 + invalid_token_ids = []

131 +

132 + for token_id in params.logit_bias:

133 + if token_id < 0 or token_id >= vocab_size:

134 + invalid_token_ids.append(token_id)

135 +

136 + if invalid_token_ids:

137 + raise ValueError(

138 + f"token_id(s) {invalid_token_ids} in logit_bias contain "

139 + f"out-of-vocab token ids. Vocabulary size: {vocab_size}"

140 + )

141 +

142 + def _validate_supported_sampling_params(

143 + self,

144 + params: SamplingParams,

145 + ) -> None:

146 + # Logits processors not supported.

147 + if params.logits_processors:

148 + raise ValueError(

149 + "vLLM V1 does not support per request user provided logits processors."

150 + )

151 + # Async scheduling + spec decode currently incompatible with some

152 + # sampling parameters.

153 + if (

154 + self.vllm_config.speculative_config is not None

155 + and self.vllm_config.scheduler_config.async_scheduling

156 + and (

157 + params.frequency_penalty != 0.0

158 + or params.presence_penalty != 0.0

159 + or params.repetition_penalty != 1.0

160 + or params.bad_words_token_ids

161 + or params.structured_outputs

162 + )

163 + ):

164 + raise ValueError(

165 + "async scheduling with spec decoding doesn't yet support "

166 + "penalties, bad words or structured outputs in sampling parameters."

167 + )

168 +

169 + def _validate_params(

170 + self,

171 + params: SamplingParams | PoolingParams,

172 + ):

173 + """

174 + Validate supported SamplingParam.

175 + Should raise ValueError if unsupported for API Server.

176 + """

177 +

178 + if isinstance(params, PoolingParams):

179 + return

180 +

181 + self._validate_logprobs(params)

182 + self._validate_sampling_params(params)

183 + self._validate_supported_sampling_params(params)

184 +

185 + def _validate_multi_modal_uuids(self, prompt: PromptType) -> None:

186 + """

187 + Validate that user-provided multi_modal_uuids align with

188 + multi_modal_data in the incoming request prompt(s).

189 + Only checks lengths; `None` entries are allowed and will be

190 + auto-hashed downstream.

191 + """

192 +

193 + def _validate_single_prompt(single_prompt: dict | str) -> None:

194 + if not isinstance(single_prompt, dict):

195 + return

196 + mm_data = single_prompt.get("multi_modal_data")

197 + mm_uuids = single_prompt.get("multi_modal_uuids")

198 + if not mm_data or not mm_uuids:

199 + return

200 +

201 + for modality, items in mm_data.items():

202 + if modality in mm_uuids:

203 + data_len = len(items) if isinstance(items, list) else 1

204 + uuid_len = (

205 + len(mm_uuids[modality])

206 + if isinstance(mm_uuids[modality], list)

207 + else 1

208 + )

209 + if uuid_len != data_len:

210 + raise ValueError(

211 + f"multi_modal_uuids for modality '{modality}' "

212 + "must have same length as data: got "

213 + f"{uuid_len} uuids vs "

214 + f"{data_len} items."

215 + )

216 + else:

217 + raise ValueError(

218 + f"multi_modal_uuids for modality '{modality}' must "

219 + "be provided if multi_modal_data is provided."

220 + )

221 +

222 + # Handle explicit encoder/decoder prompts or singleton prompt

223 + if isinstance(prompt, dict) and "encoder_prompt" in prompt:

224 + enc = prompt.get("encoder_prompt")

225 + dec = prompt.get("decoder_prompt")

226 + if enc is not None:

227 + _validate_single_prompt(cast(dict | str, enc))

228 + if dec is not None:

229 + _validate_single_prompt(cast(dict | str, dec))

230 + else:

231 + _validate_single_prompt(prompt) # type: ignore[arg-type]

232 +

233 + def _validate_lora(self, lora_request: LoRARequest | None) -> None:

234 + if lora_request is None:

235 + return

236 +

237 + # LoRA request passed in while LoRA is not enabled

238 + if not self.lora_config:

239 + raise ValueError(

240 + f"Got lora_request {lora_request} but LoRA is not enabled!"

241 + )

242 +

243 + if self.tokenizer is not None:

244 + logger.warning_once(

245 + "vLLM has deprecated support for supporting different "

246 + "tokenizers for different LoRAs. By default, vLLM uses base "

247 + "model's tokenizer. If you are using a LoRA "

248 + "with its own tokenizer, consider specifying `--tokenizer "

249 + "[lora_path]` to use the LoRA tokenizer."

250 + )

251 +

252 + def _validate_structured_output(self, params: SamplingParams) -> None:

253 + if not params.structured_outputs or not self.structured_outputs_config:

254 + return

255 +

256 + if self.model_config.skip_tokenizer_init and params.structured_outputs:

257 + raise ValueError(

258 + "Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501

259 + )

260 +

261 + backend = self.structured_outputs_config.backend

262 + if _backend := params.structured_outputs._backend:

263 + # Request-level backend selection is not supported.

264 + # The values may differ if `params` is reused and was set

265 + # to a specific backend based on `auto` behavior in a previous

266 + # request. We remember that it was set as a result of `auto`

267 + # using the `_backend_was_auto` field set in the params.

268 + if backend != _backend and not (

269 + backend == "auto" and params.structured_outputs._backend_was_auto

270 + ):

271 + raise ValueError(

272 + "Request-level structured output backend selection is not "

273 + f"supported. The request specified '{_backend}', but vLLM "

274 + f"was initialised with '{backend}'. This error can be "

275 + "resolved by removing '_backend' from the request."

276 + )

277 + else:

278 + params.structured_outputs._backend = backend

279 +

280 + # Request content validation

281 + if (

282 + isinstance(params.structured_outputs.choice, list)

283 + and not params.structured_outputs.choice

284 + ):

285 + # It is invalid for choice to be an empty list

286 + raise ValueError(

287 + f"Choice '{params.structured_outputs.choice}' cannot be an empty list" # noqa: E501

288 + )

289 + # Reject empty string grammar early to avoid engine-side crashes

290 + if (

291 + isinstance(params.structured_outputs.grammar, str)

292 + and params.structured_outputs.grammar.strip() == ""

293 + ):

294 + raise ValueError("structured_outputs.grammar cannot be an empty string")

295 +

296 + if backend.startswith("xgrammar"):

297 + # xgrammar with no fallback

298 + validate_xgrammar_grammar(params)

299 + elif backend.startswith("guidance"):

300 + # TODO: ideally we would have the LLTokenizer here as Lark syntax

301 + # allows <|special_token|> and similar, see

302 + # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens

303 + # Without tokenizer these are disallowed in grammars.

304 + if isinstance(self.tokenizer, MistralTokenizer):

305 + raise ValueError(

306 + "Mistral tokenizer is not supported for the 'guidance' "

307 + "structured output backend. Please use ['xgrammar', 'outlines'] "

308 + "backends or tokenizer_mode='hf' instead."

309 + )

310 + validate_guidance_grammar(params, tokenizer=None)

311 + elif backend == "outlines":

312 + # outlines backend

313 + validate_structured_output_request_outlines(params)

314 + elif backend == "lm-format-enforcer":

315 + # lm format enforcer backend

316 + if isinstance(self.tokenizer, MistralTokenizer):

317 + raise ValueError(

318 + "Mistral tokenizer is not supported for the 'lm-format-enforcer' "

319 + "structured output backend. Please use ['xgrammar', 'outlines'] "

320 + "backends or tokenizer_mode='hf' instead."

321 + )

322 + validate_structured_output_request_lm_format_enforcer(params)

323 + else:

324 + # NOTE: backend must be "auto" here, because we have

325 + # checked supported_backends above.

326 + # In this mode, we set opinionated defaults based on what we think

327 + # will satisfy the most use cases without having to worry about

328 + # this setting. We include fallback behavior here, but not with any

329 + # other setting where a specific backend was specified.

330 + try:

331 + validate_xgrammar_grammar(params)

332 + params.structured_outputs._backend = "xgrammar"

333 + except ValueError:

334 + # The request either failed validation

335 + # or includes some jsonschema feature(s) that

336 + # are not supported in xgrammar.

337 + if isinstance(self.tokenizer, MistralTokenizer):

338 + # Fall back to outlines if the tokenizer is Mistral

339 + validate_structured_output_request_outlines(params)

340 + params.structured_outputs._backend = "outlines"

341 + else:

342 + # Fall back to guidance by default.

343 + validate_guidance_grammar(params, tokenizer=None)

344 + params.structured_outputs._backend = "guidance"

345 + # Remember that this backend was set automatically

346 + params.structured_outputs._backend_was_auto = True

347 +

348 + def _maybe_build_mm_uuids(

349 + self,

350 + request_id: str,

351 + prompt: PromptType,

352 + ) -> MultiModalUUIDDict | None:

353 + """Build per-item multimodal hash overrides when enabled. In this case,

354 + multimodal data items are identified by their request id, modality and

355 + index rather than their content.

356 +

357 + Returns a dictionary of modality -> list[str] of overrides, or None if

358 + disabled or no multimodal data is present.

359 + """

360 +

361 + def _extract_mm_data(p: PromptType):

362 + if isinstance(p, dict) and "encoder_prompt" in p:

363 + enc = p.get("encoder_prompt")

364 + if isinstance(enc, dict):

365 + return enc.get("multi_modal_data")

366 + return None

367 + if isinstance(p, dict):

368 + return p.get("multi_modal_data")

369 + return None

370 +

371 + mm_data = _extract_mm_data(prompt)

372 + if not mm_data:

373 + return None

374 +

375 + mm_uuids: dict[str, list[str | None] | str] = {}

376 + for modality, data in mm_data.items():

377 + # Hash each item for embedding inputs.

378 + n = (

379 + len(data)

380 + if isinstance(data, list) or MultiModalDataParser.is_embeddings(data)

381 + else 1

382 + )

383 + mm_uuids[modality] = [f"{request_id}-{modality}-{i}" for i in range(n)]

384 + return mm_uuids

385 +

386 + def process_inputs(

387 + self,

388 + request_id: str,

389 + prompt: PromptType,

390 + params: SamplingParams | PoolingParams,

391 + arrival_time: float | None = None,

392 + lora_request: LoRARequest | None = None,

393 + tokenization_kwargs: dict[str, Any] | None = None,

394 + trace_headers: Mapping[str, str] | None = None,

395 + priority: int = 0,

396 + data_parallel_rank: int | None = None,

397 + ) -> EngineCoreRequest:

398 + self._validate_lora(lora_request)

399 + self._validate_params(params)

400 +

401 + data_parallel_size = self.vllm_config.parallel_config.data_parallel_size

402 + if data_parallel_rank is not None and not (

403 + 0 <= data_parallel_rank < data_parallel_size

404 + ):

405 + raise ValueError(

406 + f"data_parallel_rank {data_parallel_rank} "

407 + f"is out of range [0, {data_parallel_size})."

408 + )

409 +

410 + if arrival_time is None:

411 + arrival_time = time.time()

412 +

413 + # Optionally generate multimodal hash overrides to avoid hashing

414 + # multimodal data items by their content as their identifiers.

415 +

416 + # NOTE: when users explicitly turn off BOTH prefix caching and input

417 + # processing caching, no multimodal features or embeddings will be

418 + # reused across requests, therefore identifying multimodal data items

419 + # by their content is no longer necessary, and we create uuids with

420 + # request id-modality-index as multimodal hash overrides.

421 + if (

422 + self.model_config.multimodal_config

423 + and self.model_config.multimodal_config.mm_processor_cache_gb == 0

424 + and not self.cache_config.enable_prefix_caching

425 + ):

426 + mm_uuids = self._maybe_build_mm_uuids(request_id, prompt)

427 + else:

428 + # Otherwise, use user-provided uuids as multimodal hash overrides

429 + # if provided.

430 + self._validate_multi_modal_uuids(prompt)

431 + if isinstance(prompt, dict):

432 + mm_uuids = cast(

433 + MultiModalUUIDDict | None, prompt.get("multi_modal_uuids")

434 + )

435 + else:

436 + mm_uuids = None

437 +

438 + # Process inputs, which includes:

439 + # 1. Tokenize text prompt, with LoRA request if one exists.

440 + # 2. For multimodal models with a merged preprocessor, preprocess

441 + # multimodal data and expand prompt token ids accordingly.

442 + processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(

443 + prompt,

444 + tokenization_kwargs=tokenization_kwargs,

445 + mm_uuids=mm_uuids,

446 + )

447 + from vllm.platforms import current_platform

448 +

449 + current_platform.validate_request(

450 + prompt=prompt,

451 + params=params,

452 + processed_inputs=processed_inputs,

453 + )

454 +

455 + eos_token_id = self.input_preprocessor.get_eos_token_id()

456 +

457 + encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)

458 + self._validate_model_inputs(encoder_inputs, decoder_inputs)

459 +

460 + # Mypy can be conservative for TypedDict unions; normalize access.

461 + if decoder_inputs["type"] == "embeds":

462 + prompt_token_ids = None

463 + prompt_embeds = decoder_inputs["prompt_embeds"]

464 + else:

465 + prompt_token_ids = decoder_inputs["prompt_token_ids"]

466 + prompt_embeds = None

467 +

468 + sampling_params = None

469 + pooling_params = None

470 + if isinstance(params, SamplingParams):

471 + # TODO: can we avoid cloning here in multiproc case?

472 + sampling_params = params.clone()

473 + # If unset max tokens, then generate up to the max_model_len.

474 + if sampling_params.max_tokens is None:

475 + seq_len = length_from_prompt_token_ids_or_embeds(

476 + prompt_token_ids, prompt_embeds

477 + )

478 + sampling_params.max_tokens = self.model_config.max_model_len - seq_len

479 + sampling_params.update_from_generation_config(

480 + self.generation_config_fields, eos_token_id

481 + )

482 + if self.tokenizer is not None:

483 + sampling_params.update_from_tokenizer(self.tokenizer)

484 + else:

485 + pooling_params = params.clone()

486 +

487 + # Multimodal related.

488 + mm_features: list[MultiModalFeatureSpec] | None = None

489 +

490 + if decoder_inputs["type"] == "multimodal":

491 + decoder_mm_inputs = decoder_inputs["mm_kwargs"]

492 + decoder_mm_positions = decoder_inputs["mm_placeholders"]

493 + decoder_mm_hashes = decoder_inputs["mm_hashes"]

494 +

495 + # Merge and flatten multimodal placeholders, hashes and inputs

496 + # from dictionaries to lists, and sort them by each item's position

497 + # in the input sequence.

498 + sorted_mm_idxs = argsort_mm_positions(decoder_mm_positions)

499 +

500 + mm_features = []

501 + for modality, idx in sorted_mm_idxs:

502 + mm_features.append(

503 + MultiModalFeatureSpec(

504 + data=decoder_mm_inputs[modality][idx],

505 + modality=modality,

506 + identifier=decoder_mm_hashes[modality][idx],

507 + mm_position=decoder_mm_positions[modality][idx],

508 + )

509 + )

510 +

511 + return EngineCoreRequest(

512 + request_id=request_id,

513 + prompt_token_ids=prompt_token_ids,

514 + prompt_embeds=prompt_embeds,

515 + mm_features=mm_features,

516 + sampling_params=sampling_params,

517 + pooling_params=pooling_params,

518 + eos_token_id=eos_token_id,

519 + arrival_time=arrival_time,

520 + lora_request=lora_request,

521 + cache_salt=decoder_inputs.get("cache_salt"),

522 + priority=priority,

523 + data_parallel_rank=data_parallel_rank,

524 + trace_headers=trace_headers,

525 + )

526 +

527 + def _validate_model_inputs(

528 + self, encoder_inputs: SingletonInputs | None, decoder_inputs: SingletonInputs

529 + ):

530 + if encoder_inputs is not None:

531 + self._validate_model_input(encoder_inputs, prompt_type="encoder")

532 +

533 + self._validate_model_input(decoder_inputs, prompt_type="decoder")

534 +

535 + def _validate_model_input(

536 + self,

537 + prompt_inputs: SingletonInputs,

538 + *,

539 + prompt_type: Literal["encoder", "decoder"],

540 + ):

541 + model_config = self.model_config

542 +

543 + prompt_ids = (

544 + None

545 + if prompt_inputs["type"] == "embeds"

546 + else prompt_inputs["prompt_token_ids"]

547 + )

548 + prompt_embeds = (

549 + prompt_inputs["prompt_embeds"]

550 + if prompt_inputs["type"] == "embeds"

551 + else None

552 + )

553 + prompt_len = length_from_prompt_token_ids_or_embeds(prompt_ids, prompt_embeds)

554 + if not prompt_ids:

555 + if prompt_type == "encoder" and model_config.is_multimodal_model:

556 + pass # Mllama may have empty encoder inputs for text-only data

557 + elif prompt_inputs["type"] == "embeds":

558 + pass # Prompt embeds should not have prompt_ids.

559 + else:

560 + raise ValueError(f"The {prompt_type} prompt cannot be empty")

561 +

562 + tokenizer = self.tokenizer

563 + if tokenizer is not None:

564 + max_input_id = max(prompt_ids or [], default=0)

565 +

566 + # NOTE: tokenizer.max_token_id is the tokenizer’s vocab size while

567 + # self.model_config.get_vocab_size() is the model’s vocab size.

568 + # For Qwen3 models, the language model has extra tokens that do

569 + # not exist in the tokenizer, and vice versa for multimodal

570 + # placeholder tokens in some multimodal models.

571 + # See https://github.com/QwenLM/Qwen3/issues/29#issuecomment-1933720399 # noqa: E501

572 + # and https://github.com/vllm-project/vllm/pull/22471#discussion_r2312251421 # noqa: E501

573 +

574 + # Here we take the max of the two to determine if a token id is

575 + # truly out-of-vocabulary.

576 + if max_input_id > max(

577 + tokenizer.max_token_id, self.model_config.get_vocab_size() - 1

578 + ):

579 + raise ValueError(f"Token id {max_input_id} is out of vocabulary")

580 +

581 + max_prompt_len = self.model_config.max_model_len

582 + if prompt_len > max_prompt_len:

583 + if prompt_type == "encoder" and model_config.is_multimodal_model:

584 + mm_registry = self.input_preprocessor.mm_registry

585 + mm_processor = mm_registry.create_processor(

586 + model_config,

587 + tokenizer=tokenizer,

588 + )

589 + assert isinstance(mm_processor, EncDecMultiModalProcessor)

590 +

591 + if mm_processor.pad_dummy_encoder_prompt:

592 + return # Skip encoder length check for Whisper

593 +

594 + if model_config.is_multimodal_model:

595 + suggestion = (

596 + "Make sure that `max_model_len` is no smaller than the "

597 + "number of text tokens plus multimodal tokens. For image "

598 + "inputs, the number of image tokens depends on the number "

599 + "of images, and possibly their aspect ratios as well."

600 + )

601 + else:

602 + suggestion = (

603 + "Make sure that `max_model_len` is no smaller than the "

604 + "number of text tokens."

605 + )

606 +

607 + raise ValueError(

608 + f"The {prompt_type} prompt (length {prompt_len}) is "

609 + f"longer than the maximum model length of {max_prompt_len}. "

610 + f"{suggestion}"

611 + )

612 +

613 + # TODO: Find out how many placeholder tokens are there so we can

614 + # check that chunked prefill does not truncate them

615 + # max_batch_len = self.scheduler_config.max_num_batched_tokens

616 +

617 + if (

618 + prompt_len == max_prompt_len

619 + and prompt_type == "decoder"

620 + and not model_config.is_multimodal_model

621 + and self.model_config.runner_type != "pooling"

622 + ):

623 + suggestion = (

624 + "Make sure that `max_model_len` is no smaller than the "

625 + "number of text tokens (prompt + requested output tokens)."

626 + )

627 + raise ValueError(

628 + f"The {prompt_type} prompt (length {prompt_len}) plus the number of "

629 + f"requested output tokens (at least 1) is longer than the maximum "

630 + f"model length of {max_prompt_len}. {suggestion}"

631 + )

632 +

633 + def stat_mm_cache(self) -> MultiModalCacheStats | None:

634 + return self.input_preprocessor.stat_mm_cache()

635 +

636 + def clear_mm_cache(self) -> None:

637 + self.input_preprocessor.clear_mm_cache()

vllm/v1/engine/llm_engine.py

+16 -12

@@ -7,7 +7,7 @@ from copy import copy

7 from typing import Any, cast

9 import torch.nn as nn

10 -from typing_extensions import TypeVar

10 +from typing_extensions import TypeVar, deprecated

12 import vllm.envs as envs

13 from vllm.config import ParallelConfig, VllmConfig

@@ -28,9 +28,9 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_

28 from vllm.usage.usage_lib import UsageContext

29 from vllm.v1.engine import EngineCoreRequest

30 from vllm.v1.engine.core_client import EngineCoreClient

31 +from vllm.v1.engine.input_processor import InputProcessor

32 from vllm.v1.engine.output_processor import OutputProcessor

33 from vllm.v1.engine.parallel_sampling import ParentRequest

33 -from vllm.v1.engine.processor import Processor

34 from vllm.v1.executor import Executor

35 from vllm.v1.metrics.loggers import StatLoggerFactory, StatLoggerManager

36 from vllm.v1.metrics.reader import Metric, get_metrics_snapshot

@@ -88,7 +88,7 @@ class LLMEngine:

88 else:

89 tokenizer = init_tokenizer_from_configs(self.model_config)

91 - self.processor = Processor(self.vllm_config, tokenizer)

91 + self.input_processor = InputProcessor(self.vllm_config, tokenizer)

92 self.io_processor = get_io_processor(

93 self.vllm_config,

94 self.model_config.io_processor_plugin,

@@ -135,6 +135,14 @@ class LLMEngine:

135 # Don't keep the dummy data in memory

136 self.reset_mm_cache()

137

138 + @property

139 + @deprecated(

140 + "`LLMEngine.processor` has been renamed to `LLMEngine.input_processor`. "

141 + "The old name will be removed in v0.13."

142 + )

143 + def processor(self):

144 + return self.input_processor

145 +

146 @classmethod

147 def from_vllm_config(

148 cls,

@@ -231,11 +239,7 @@ class LLMEngine:

239 request = prompt

240 else:

241 assert prompt_text is None

234 - logger.warning_once(

235 - "Processor has been moved under LLM and will "

236 - "be removed from LLMEngine in v0.13."

237 - )

238 - request = self.processor.process_inputs(

242 + request = self.input_processor.process_inputs(

243 request_id,

244 prompt,

245 params,

@@ -307,7 +311,7 @@ class LLMEngine:

311 self.logger_manager.record(

312 scheduler_stats=outputs.scheduler_stats,

313 iteration_stats=iteration_stats,

310 - mm_cache_stats=self.processor.stat_mm_cache(),

314 + mm_cache_stats=self.input_processor.stat_mm_cache(),

315 )

316 self.do_log_stats_with_interval()

317

@@ -320,7 +324,7 @@ class LLMEngine:

324 self.engine_core.profile(False)

325

326 def reset_mm_cache(self):

323 - self.processor.clear_mm_cache()

327 + self.input_processor.clear_mm_cache()

328 self.engine_core.reset_mm_cache()

329

330 def reset_prefix_cache(self):

@@ -347,11 +351,11 @@ class LLMEngine:

351

352 @property

353 def tokenizer(self) -> AnyTokenizer | None:

350 - return self.processor.tokenizer

354 + return self.input_processor.tokenizer

355

356 @tokenizer.setter

357 def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:

354 - self.processor.tokenizer = tokenizer

358 + self.input_processor.tokenizer = tokenizer

359

360 def get_tokenizer(self) -> AnyTokenizer:

361 if self.tokenizer is None:

vllm/v1/engine/processor.py

+12 -629

@@ -1,637 +1,20 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import warnings

4 -import time

5 -from collections.abc import Mapping

6 -from typing import Any, Literal, cast

8 -from vllm.config import VllmConfig

9 -from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs

10 -from vllm.inputs.parse import split_enc_dec_inputs

11 -from vllm.inputs.preprocess import InputPreprocessor

12 -from vllm.logger import init_logger

13 -from vllm.lora.request import LoRARequest

14 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

15 -from vllm.multimodal.cache import processor_cache_from_config

16 -from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict

17 -from vllm.multimodal.parse import MultiModalDataParser

18 -from vllm.multimodal.processing import EncDecMultiModalProcessor

19 -from vllm.multimodal.utils import argsort_mm_positions

20 -from vllm.pooling_params import PoolingParams

21 -from vllm.sampling_params import SamplingParams

22 -from vllm.transformers_utils.tokenizer import AnyTokenizer

23 -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

24 -from vllm.utils import length_from_prompt_token_ids_or_embeds

25 -from vllm.v1.engine import EngineCoreRequest

26 -from vllm.v1.metrics.stats import MultiModalCacheStats

27 -from vllm.v1.structured_output.backend_guidance import validate_guidance_grammar

28 -from vllm.v1.structured_output.backend_lm_format_enforcer import (

29 - validate_structured_output_request_lm_format_enforcer,

30 -)

31 -from vllm.v1.structured_output.backend_outlines import (

32 - validate_structured_output_request_outlines,

33 -)

34 -from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar

6 +def __getattr__(name: str):

7 + if name == "Processor":

8 + from .input_processor import InputProcessor

36 -logger = init_logger(__name__)

37 -

38 -

39 -class Processor:

40 - def __init__(

41 - self,

42 - vllm_config: VllmConfig,

43 - tokenizer: AnyTokenizer | None,

44 - mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

45 - ) -> None:

46 - self.vllm_config = vllm_config

47 - self.model_config = vllm_config.model_config

48 - self.cache_config = vllm_config.cache_config

49 - self.lora_config = vllm_config.lora_config

50 - self.structured_outputs_config = vllm_config.structured_outputs_config

51 -

52 - self.generation_config_fields = self.model_config.try_get_generation_config()

53 -

54 - self.mm_registry = mm_registry

55 - self.mm_processor_cache = processor_cache_from_config(vllm_config, mm_registry)

56 -

57 - self.input_preprocessor = InputPreprocessor(

58 - self.model_config,

59 - tokenizer,

60 - mm_registry,

61 - mm_processor_cache=self.mm_processor_cache,

10 + warnings.warn(

11 + "`vllm.v1.engine.processor.Processor` has been moved to "

12 + "`vllm.v1.engine.input_processor.InputProcessor`. "

13 + "The old name will be removed in v0.13.",

14 + DeprecationWarning,

15 + stacklevel=2,

16 )

64 - @property

65 - def tokenizer(self) -> AnyTokenizer | None:

66 - return self.input_preprocessor.tokenizer

67 -

68 - @tokenizer.setter

69 - def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:

70 - self.input_preprocessor.tokenizer = tokenizer

71 -

72 - def _validate_logprobs(

73 - self,

74 - params: SamplingParams,

75 - ) -> None:

76 - max_logprobs = self.model_config.max_logprobs

77 - if max_logprobs == -1:

78 - max_logprobs = self.model_config.get_vocab_size()

79 -

80 - # Validate sample logprobs.

81 - if params.logprobs:

82 - num_logprobs = params.logprobs

83 - if num_logprobs == -1:

84 - num_logprobs = self.model_config.get_vocab_size()

85 - if num_logprobs > max_logprobs:

86 - raise ValueError(

87 - f"Requested sample logprobs of {num_logprobs}, "

88 - f"which is greater than max allowed: {max_logprobs}"

89 - )

90 -

91 - # Validate prompt logprobs.

92 - if params.prompt_logprobs:

93 - num_prompt_logprobs = params.prompt_logprobs

94 - if num_prompt_logprobs == -1:

95 - num_prompt_logprobs = self.model_config.get_vocab_size()

96 - if num_prompt_logprobs > max_logprobs:

97 - raise ValueError(

98 - f"Requested prompt logprobs of {num_prompt_logprobs}, "

99 - f"which is greater than max allowed: {max_logprobs}"

100 - )

101 -

102 - def _validate_sampling_params(

103 - self,

104 - params: SamplingParams,

105 - ) -> None:

106 - self._validate_structured_output(params)

107 - self._validate_logit_bias(params)

108 -

109 - if params.allowed_token_ids is None:

110 - return

111 - if not params.allowed_token_ids:

112 - raise ValueError("allowed_token_ids is not None and empty!")

113 - if self.tokenizer is None:

114 - # When skip_tokenizer_init=True, we can't validate token IDs

115 - # Skip validation and let the model handle invalid tokens

116 - return

117 - vocab_size = len(self.tokenizer)

118 - if not all(0 <= tid < vocab_size for tid in params.allowed_token_ids):

119 - raise ValueError("allowed_token_ids contains out-of-vocab token id!")

120 -

121 - def _validate_logit_bias(

122 - self,

123 - params: SamplingParams,

124 - ) -> None:

125 - """Validate logit_bias token IDs are within vocabulary range."""

126 - if not params.logit_bias:

127 - return

128 -

129 - vocab_size = self.model_config.get_vocab_size()

130 - invalid_token_ids = []

131 -

132 - for token_id in params.logit_bias:

133 - if token_id < 0 or token_id >= vocab_size:

134 - invalid_token_ids.append(token_id)

135 -

136 - if invalid_token_ids:

137 - raise ValueError(

138 - f"token_id(s) {invalid_token_ids} in logit_bias contain "

139 - f"out-of-vocab token ids. Vocabulary size: {vocab_size}"

140 - )

141 -

142 - def _validate_supported_sampling_params(

143 - self,

144 - params: SamplingParams,

145 - ) -> None:

146 - # Logits processors not supported.

147 - if params.logits_processors:

148 - raise ValueError(

149 - "vLLM V1 does not support per request user provided logits processors."

150 - )

151 - # Async scheduling + spec decode currently incompatible with some

152 - # sampling parameters.

153 - if (

154 - self.vllm_config.speculative_config is not None

155 - and self.vllm_config.scheduler_config.async_scheduling

156 - and (

157 - params.frequency_penalty != 0.0

158 - or params.presence_penalty != 0.0

159 - or params.repetition_penalty != 1.0

160 - or params.bad_words_token_ids

161 - or params.structured_outputs

162 - )

163 - ):

164 - raise ValueError(

165 - "async scheduling with spec decoding doesn't yet support "

166 - "penalties, bad words or structured outputs in sampling parameters."

167 - )

168 -

169 - def _validate_params(

170 - self,

171 - params: SamplingParams | PoolingParams,

172 - ):

173 - """

174 - Validate supported SamplingParam.

175 - Should raise ValueError if unsupported for API Server.

176 - """

177 -

178 - if isinstance(params, PoolingParams):

179 - return

180 -

181 - self._validate_logprobs(params)

182 - self._validate_sampling_params(params)

183 - self._validate_supported_sampling_params(params)

184 -

185 - def _validate_multi_modal_uuids(self, prompt: PromptType) -> None:

186 - """

187 - Validate that user-provided multi_modal_uuids align with

188 - multi_modal_data in the incoming request prompt(s).

189 - Only checks lengths; `None` entries are allowed and will be

190 - auto-hashed downstream.

191 - """

192 -

193 - def _validate_single_prompt(single_prompt: dict | str) -> None:

194 - if not isinstance(single_prompt, dict):

195 - return

196 - mm_data = single_prompt.get("multi_modal_data")

197 - mm_uuids = single_prompt.get("multi_modal_uuids")

198 - if not mm_data or not mm_uuids:

199 - return

200 -

201 - for modality, items in mm_data.items():

202 - if modality in mm_uuids:

203 - data_len = len(items) if isinstance(items, list) else 1

204 - uuid_len = (

205 - len(mm_uuids[modality])

206 - if isinstance(mm_uuids[modality], list)

207 - else 1

208 - )

209 - if uuid_len != data_len:

210 - raise ValueError(

211 - f"multi_modal_uuids for modality '{modality}' "

212 - "must have same length as data: got "

213 - f"{uuid_len} uuids vs "

214 - f"{data_len} items."

215 - )

216 - else:

217 - raise ValueError(

218 - f"multi_modal_uuids for modality '{modality}' must "

219 - "be provided if multi_modal_data is provided."

220 - )

221 -

222 - # Handle explicit encoder/decoder prompts or singleton prompt

223 - if isinstance(prompt, dict) and "encoder_prompt" in prompt:

224 - enc = prompt.get("encoder_prompt")

225 - dec = prompt.get("decoder_prompt")

226 - if enc is not None:

227 - _validate_single_prompt(cast(dict | str, enc))

228 - if dec is not None:

229 - _validate_single_prompt(cast(dict | str, dec))

230 - else:

231 - _validate_single_prompt(prompt) # type: ignore[arg-type]

232 -

233 - def _validate_lora(self, lora_request: LoRARequest | None) -> None:

234 - if lora_request is None:

235 - return

236 -

237 - # LoRA request passed in while LoRA is not enabled

238 - if not self.lora_config:

239 - raise ValueError(

240 - f"Got lora_request {lora_request} but LoRA is not enabled!"

241 - )

242 -

243 - if self.tokenizer is not None:

244 - logger.warning_once(

245 - "vLLM has deprecated support for supporting different "

246 - "tokenizers for different LoRAs. By default, vLLM uses base "

247 - "model's tokenizer. If you are using a LoRA "

248 - "with its own tokenizer, consider specifying `--tokenizer "

249 - "[lora_path]` to use the LoRA tokenizer."

250 - )

251 -

252 - def _validate_structured_output(self, params: SamplingParams) -> None:

253 - if not params.structured_outputs or not self.structured_outputs_config:

254 - return

255 -

256 - if self.model_config.skip_tokenizer_init and params.structured_outputs:

257 - raise ValueError(

258 - "Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501

259 - )

260 -

261 - backend = self.structured_outputs_config.backend

262 - if _backend := params.structured_outputs._backend:

263 - # Request-level backend selection is not supported.

264 - # The values may differ if `params` is reused and was set

265 - # to a specific backend based on `auto` behavior in a previous

266 - # request. We remember that it was set as a result of `auto`

267 - # using the `_backend_was_auto` field set in the params.

268 - if backend != _backend and not (

269 - backend == "auto" and params.structured_outputs._backend_was_auto

270 - ):

271 - raise ValueError(

272 - "Request-level structured output backend selection is not "

273 - f"supported. The request specified '{_backend}', but vLLM "

274 - f"was initialised with '{backend}'. This error can be "

275 - "resolved by removing '_backend' from the request."

276 - )

277 - else:

278 - params.structured_outputs._backend = backend

279 -

280 - # Request content validation

281 - if (

282 - isinstance(params.structured_outputs.choice, list)

283 - and not params.structured_outputs.choice

284 - ):

285 - # It is invalid for choice to be an empty list

286 - raise ValueError(

287 - f"Choice '{params.structured_outputs.choice}' cannot be an empty list" # noqa: E501

288 - )

289 - # Reject empty string grammar early to avoid engine-side crashes

290 - if (

291 - isinstance(params.structured_outputs.grammar, str)

292 - and params.structured_outputs.grammar.strip() == ""

293 - ):

294 - raise ValueError("structured_outputs.grammar cannot be an empty string")

295 -

296 - if backend.startswith("xgrammar"):

297 - # xgrammar with no fallback

298 - validate_xgrammar_grammar(params)

299 - elif backend.startswith("guidance"):

300 - # TODO: ideally we would have the LLTokenizer here as Lark syntax

301 - # allows <|special_token|> and similar, see

302 - # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens

303 - # Without tokenizer these are disallowed in grammars.

304 - if isinstance(self.tokenizer, MistralTokenizer):

305 - raise ValueError(

306 - "Mistral tokenizer is not supported for the 'guidance' "

307 - "structured output backend. Please use ['xgrammar', 'outlines'] "

308 - "backends or tokenizer_mode='hf' instead."

309 - )

310 - validate_guidance_grammar(params, tokenizer=None)

311 - elif backend == "outlines":

312 - # outlines backend

313 - validate_structured_output_request_outlines(params)

314 - elif backend == "lm-format-enforcer":

315 - # lm format enforcer backend

316 - if isinstance(self.tokenizer, MistralTokenizer):

317 - raise ValueError(

318 - "Mistral tokenizer is not supported for the 'lm-format-enforcer' "

319 - "structured output backend. Please use ['xgrammar', 'outlines'] "

320 - "backends or tokenizer_mode='hf' instead."

321 - )

322 - validate_structured_output_request_lm_format_enforcer(params)

323 - else:

324 - # NOTE: backend must be "auto" here, because we have

325 - # checked supported_backends above.

326 - # In this mode, we set opinionated defaults based on what we think

327 - # will satisfy the most use cases without having to worry about

328 - # this setting. We include fallback behavior here, but not with any

329 - # other setting where a specific backend was specified.

330 - try:

331 - validate_xgrammar_grammar(params)

332 - params.structured_outputs._backend = "xgrammar"

333 - except ValueError:

334 - # The request either failed validation

335 - # or includes some jsonschema feature(s) that

336 - # are not supported in xgrammar.

337 - if isinstance(self.tokenizer, MistralTokenizer):

338 - # Fall back to outlines if the tokenizer is Mistral

339 - validate_structured_output_request_outlines(params)

340 - params.structured_outputs._backend = "outlines"

341 - else:

342 - # Fall back to guidance by default.

343 - validate_guidance_grammar(params, tokenizer=None)

344 - params.structured_outputs._backend = "guidance"

345 - # Remember that this backend was set automatically

346 - params.structured_outputs._backend_was_auto = True

347 -

348 - def _maybe_build_mm_uuids(

349 - self,

350 - request_id: str,

351 - prompt: PromptType,

352 - ) -> MultiModalUUIDDict | None:

353 - """Build per-item multimodal hash overrides when enabled. In this case,

354 - multimodal data items are identified by their request id, modality and

355 - index rather than their content.

356 -

357 - Returns a dictionary of modality -> list[str] of overrides, or None if

358 - disabled or no multimodal data is present.

359 - """

360 -

361 - def _extract_mm_data(p: PromptType):

362 - if isinstance(p, dict) and "encoder_prompt" in p:

363 - enc = p.get("encoder_prompt")

364 - if isinstance(enc, dict):

365 - return enc.get("multi_modal_data")

366 - return None

367 - if isinstance(p, dict):

368 - return p.get("multi_modal_data")

369 - return None

370 -

371 - mm_data = _extract_mm_data(prompt)

372 - if not mm_data:

373 - return None

374 -

375 - mm_uuids: dict[str, list[str | None] | str] = {}

376 - for modality, data in mm_data.items():

377 - # Hash each item for embedding inputs.

378 - n = (

379 - len(data)

380 - if isinstance(data, list) or MultiModalDataParser.is_embeddings(data)

381 - else 1

382 - )

383 - mm_uuids[modality] = [f"{request_id}-{modality}-{i}" for i in range(n)]

384 - return mm_uuids

385 -

386 - def process_inputs(

387 - self,

388 - request_id: str,

389 - prompt: PromptType,

390 - params: SamplingParams | PoolingParams,

391 - arrival_time: float | None = None,

392 - lora_request: LoRARequest | None = None,

393 - tokenization_kwargs: dict[str, Any] | None = None,

394 - trace_headers: Mapping[str, str] | None = None,

395 - priority: int = 0,

396 - data_parallel_rank: int | None = None,

397 - ) -> EngineCoreRequest:

398 - self._validate_lora(lora_request)

399 - self._validate_params(params)

400 -

401 - data_parallel_size = self.vllm_config.parallel_config.data_parallel_size

402 - if data_parallel_rank is not None and not (

403 - 0 <= data_parallel_rank < data_parallel_size

404 - ):

405 - raise ValueError(

406 - f"data_parallel_rank {data_parallel_rank} "

407 - f"is out of range [0, {data_parallel_size})."

408 - )

409 -

410 - if arrival_time is None:

411 - arrival_time = time.time()

412 -

413 - # Optionally generate multimodal hash overrides to avoid hashing

414 - # multimodal data items by their content as their identifiers.

415 -

416 - # NOTE: when users explicitly turn off BOTH prefix caching and input

417 - # processing caching, no multimodal features or embeddings will be

418 - # reused across requests, therefore identifying multimodal data items

419 - # by their content is no longer necessary, and we create uuids with

420 - # request id-modality-index as multimodal hash overrides.

421 - if (

422 - self.model_config.multimodal_config

423 - and self.model_config.multimodal_config.mm_processor_cache_gb == 0

424 - and not self.cache_config.enable_prefix_caching

425 - ):

426 - mm_uuids = self._maybe_build_mm_uuids(request_id, prompt)

427 - else:

428 - # Otherwise, use user-provided uuids as multimodal hash overrides

429 - # if provided.

430 - self._validate_multi_modal_uuids(prompt)

431 - if isinstance(prompt, dict):

432 - mm_uuids = cast(

433 - MultiModalUUIDDict | None, prompt.get("multi_modal_uuids")

434 - )

435 - else:

436 - mm_uuids = None

437 -

438 - # Process inputs, which includes:

439 - # 1. Tokenize text prompt, with LoRA request if one exists.

440 - # 2. For multimodal models with a merged preprocessor, preprocess

441 - # multimodal data and expand prompt token ids accordingly.

442 - processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(

443 - prompt,

444 - tokenization_kwargs=tokenization_kwargs,

445 - mm_uuids=mm_uuids,

446 - )

447 - from vllm.platforms import current_platform

448 -

449 - current_platform.validate_request(

450 - prompt=prompt,

451 - params=params,

452 - processed_inputs=processed_inputs,

453 - )

454 -

455 - eos_token_id = self.input_preprocessor.get_eos_token_id()

456 -

457 - encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)

458 - self._validate_model_inputs(encoder_inputs, decoder_inputs)

459 -

460 - # Mypy can be conservative for TypedDict unions; normalize access.

461 - if decoder_inputs["type"] == "embeds":

462 - prompt_token_ids = None

463 - prompt_embeds = decoder_inputs["prompt_embeds"]

464 - else:

465 - prompt_token_ids = decoder_inputs["prompt_token_ids"]

466 - prompt_embeds = None

467 -

468 - sampling_params = None

469 - pooling_params = None

470 - if isinstance(params, SamplingParams):

471 - # TODO: can we avoid cloning here in multiproc case?

472 - sampling_params = params.clone()

473 - # If unset max tokens, then generate up to the max_model_len.

474 - if sampling_params.max_tokens is None:

475 - seq_len = length_from_prompt_token_ids_or_embeds(

476 - prompt_token_ids, prompt_embeds

477 - )

478 - sampling_params.max_tokens = self.model_config.max_model_len - seq_len

479 - sampling_params.update_from_generation_config(

480 - self.generation_config_fields, eos_token_id

481 - )

482 - if self.tokenizer is not None:

483 - sampling_params.update_from_tokenizer(self.tokenizer)

484 - else:

485 - pooling_params = params.clone()

486 -

487 - # Multimodal related.

488 - mm_features: list[MultiModalFeatureSpec] | None = None

489 -

490 - if decoder_inputs["type"] == "multimodal":

491 - decoder_mm_inputs = decoder_inputs["mm_kwargs"]

492 - decoder_mm_positions = decoder_inputs["mm_placeholders"]

493 - decoder_mm_hashes = decoder_inputs["mm_hashes"]

494 -

495 - # Merge and flatten multimodal placeholders, hashes and inputs

496 - # from dictionaries to lists, and sort them by each item's position

497 - # in the input sequence.

498 - sorted_mm_idxs = argsort_mm_positions(decoder_mm_positions)

499 -

500 - mm_features = []

501 - for modality, idx in sorted_mm_idxs:

502 - mm_features.append(

503 - MultiModalFeatureSpec(

504 - data=decoder_mm_inputs[modality][idx],

505 - modality=modality,

506 - identifier=decoder_mm_hashes[modality][idx],

507 - mm_position=decoder_mm_positions[modality][idx],

508 - )

509 - )

510 -

511 - return EngineCoreRequest(

512 - request_id=request_id,

513 - prompt_token_ids=prompt_token_ids,

514 - prompt_embeds=prompt_embeds,

515 - mm_features=mm_features,

516 - sampling_params=sampling_params,

517 - pooling_params=pooling_params,

518 - eos_token_id=eos_token_id,

519 - arrival_time=arrival_time,

520 - lora_request=lora_request,

521 - cache_salt=decoder_inputs.get("cache_salt"),

522 - priority=priority,

523 - data_parallel_rank=data_parallel_rank,

524 - trace_headers=trace_headers,

525 - )

526 -

527 - def _validate_model_inputs(

528 - self, encoder_inputs: SingletonInputs | None, decoder_inputs: SingletonInputs

529 - ):

530 - if encoder_inputs is not None:

531 - self._validate_model_input(encoder_inputs, prompt_type="encoder")

532 -

533 - self._validate_model_input(decoder_inputs, prompt_type="decoder")

534 -

535 - def _validate_model_input(

536 - self,

537 - prompt_inputs: SingletonInputs,

538 - *,

539 - prompt_type: Literal["encoder", "decoder"],

540 - ):

541 - model_config = self.model_config

542 -

543 - prompt_ids = (

544 - None

545 - if prompt_inputs["type"] == "embeds"

546 - else prompt_inputs["prompt_token_ids"]

547 - )

548 - prompt_embeds = (

549 - prompt_inputs["prompt_embeds"]

550 - if prompt_inputs["type"] == "embeds"

551 - else None

552 - )

553 - prompt_len = length_from_prompt_token_ids_or_embeds(prompt_ids, prompt_embeds)

554 - if not prompt_ids:

555 - if prompt_type == "encoder" and model_config.is_multimodal_model:

556 - pass # Mllama may have empty encoder inputs for text-only data

557 - elif prompt_inputs["type"] == "embeds":

558 - pass # Prompt embeds should not have prompt_ids.

559 - else:

560 - raise ValueError(f"The {prompt_type} prompt cannot be empty")

561 -

562 - tokenizer = self.tokenizer

563 - if tokenizer is not None:

564 - max_input_id = max(prompt_ids or [], default=0)

565 -

566 - # NOTE: tokenizer.max_token_id is the tokenizer’s vocab size while

567 - # self.model_config.get_vocab_size() is the model’s vocab size.

568 - # For Qwen3 models, the language model has extra tokens that do

569 - # not exist in the tokenizer, and vice versa for multimodal

570 - # placeholder tokens in some multimodal models.

571 - # See https://github.com/QwenLM/Qwen3/issues/29#issuecomment-1933720399 # noqa: E501

572 - # and https://github.com/vllm-project/vllm/pull/22471#discussion_r2312251421 # noqa: E501

573 -

574 - # Here we take the max of the two to determine if a token id is

575 - # truly out-of-vocabulary.

576 - if max_input_id > max(

577 - tokenizer.max_token_id, self.model_config.get_vocab_size() - 1

578 - ):

579 - raise ValueError(f"Token id {max_input_id} is out of vocabulary")

580 -

581 - max_prompt_len = self.model_config.max_model_len

582 - if prompt_len > max_prompt_len:

583 - if prompt_type == "encoder" and model_config.is_multimodal_model:

584 - mm_registry = self.input_preprocessor.mm_registry

585 - mm_processor = mm_registry.create_processor(

586 - model_config,

587 - tokenizer=tokenizer,

588 - )

589 - assert isinstance(mm_processor, EncDecMultiModalProcessor)

590 -

591 - if mm_processor.pad_dummy_encoder_prompt:

592 - return # Skip encoder length check for Whisper

593 -

594 - if model_config.is_multimodal_model:

595 - suggestion = (

596 - "Make sure that `max_model_len` is no smaller than the "

597 - "number of text tokens plus multimodal tokens. For image "

598 - "inputs, the number of image tokens depends on the number "

599 - "of images, and possibly their aspect ratios as well."

600 - )

601 - else:

602 - suggestion = (

603 - "Make sure that `max_model_len` is no smaller than the "

604 - "number of text tokens."

605 - )

606 -

607 - raise ValueError(

608 - f"The {prompt_type} prompt (length {prompt_len}) is "

609 - f"longer than the maximum model length of {max_prompt_len}. "

610 - f"{suggestion}"

611 - )

612 -

613 - # TODO: Find out how many placeholder tokens are there so we can

614 - # check that chunked prefill does not truncate them

615 - # max_batch_len = self.scheduler_config.max_num_batched_tokens

616 -

617 - if (

618 - prompt_len == max_prompt_len

619 - and prompt_type == "decoder"

620 - and not model_config.is_multimodal_model

621 - and self.model_config.runner_type != "pooling"

622 - ):

623 - suggestion = (

624 - "Make sure that `max_model_len` is no smaller than the "

625 - "number of text tokens (prompt + requested output tokens)."

626 - )

627 - raise ValueError(

628 - f"The {prompt_type} prompt (length {prompt_len}) plus the number of "

629 - f"requested output tokens (at least 1) is longer than the maximum "

630 - f"model length of {max_prompt_len}. {suggestion}"

631 - )

632 -

633 - def stat_mm_cache(self) -> MultiModalCacheStats | None:

634 - return self.input_preprocessor.stat_mm_cache()

18 + return InputProcessor

636 - def clear_mm_cache(self) -> None:

637 - self.input_preprocessor.clear_mm_cache()

20 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")