@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 73391a1baa

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 06:15 UTC

[Renderer] Move InputPreprocessor into Renderer (1/2) (#34510)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Cyrus Leung committed Feb 15, 2026 at 02:14 UTC 73391a1baa459e78be1ade466517c7206ab7dd7c

39 files changed +456 -458

tests/entrypoints/openai/test_chat_error.py

+2 -1

@@ -54,6 +54,7 @@ class MockModelConfig:

54 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

55 skip_tokenizer_init = False

56 is_encoder_decoder: bool = False

57 + is_multimodal_model: bool = False

59 def get_diff_sampling_param(self):

60 return self.diff_sampling_param or {}

@@ -67,7 +68,7 @@ class MockVllmConfig:

68 def _build_renderer(model_config: MockModelConfig):

69 _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)

70 - return HfRenderer(

71 + return HfRenderer.from_config(

72 MockVllmConfig(model_config),

73 tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

74 )

tests/entrypoints/openai/test_completion_error.py

+2 -1

@@ -53,6 +53,7 @@ class MockModelConfig:

53 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

54 skip_tokenizer_init = False

55 is_encoder_decoder: bool = False

56 + is_multimodal_model: bool = False

58 def get_diff_sampling_param(self):

59 return self.diff_sampling_param or {}

@@ -78,7 +79,7 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:

79 def _build_renderer(model_config: MockModelConfig):

80 _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)

81 - return HfRenderer(

82 + return HfRenderer.from_config(

83 MockVllmConfig(model_config),

84 tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

85 )

tests/entrypoints/openai/test_lora_resolvers.py

+2 -1

@@ -52,6 +52,7 @@ class MockModelConfig:

52 generation_config: str = "auto"

53 skip_tokenizer_init: bool = False

54 is_encoder_decoder: bool = False

55 + is_multimodal_model: bool = False

57 def get_diff_sampling_param(self):

58 return self.diff_sampling_param or {}

@@ -95,7 +96,7 @@ def register_mock_resolver():

96 def _build_renderer(model_config: MockModelConfig):

97 _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)

98 - return HfRenderer(

99 + return HfRenderer.from_config(

100 MockVllmConfig(model_config),

101 tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

102 )

tests/entrypoints/openai/test_serving_chat.py

+4 -5

@@ -529,6 +529,7 @@ class MockModelConfig:

529 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

530 skip_tokenizer_init: bool = False

531 is_encoder_decoder: bool = False

532 + is_multimodal_model: bool = False

533

534 def get_diff_sampling_param(self):

535 return self.diff_sampling_param or {}

@@ -542,7 +543,7 @@ class MockVllmConfig:

543 def _build_renderer(model_config: MockModelConfig):

544 _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)

545

545 - return HfRenderer(

546 + return HfRenderer.from_config(

547 MockVllmConfig(model_config),

548 tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

549 )

@@ -756,9 +757,8 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():

757 mock_tokenizer = MagicMock(spec=MistralTokenizer)

758 mock_renderer = MistralRenderer(

759 MockVllmConfig(mock_engine.model_config),

759 - tokenizer_kwargs={},

760 + tokenizer=mock_tokenizer,

761 )

761 - mock_renderer._tokenizer = mock_tokenizer

762 # Force the Mistral chat template renderer to return token IDs.

763 # Choose a prompt length that is < max_model_len, but large enough that

764 # adding max_tokens should exceed the model context window.

@@ -798,9 +798,8 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():

798 mock_tokenizer = MagicMock(spec=MistralTokenizer)

799 mock_renderer = MistralRenderer(

800 MockVllmConfig(mock_engine.model_config),

801 - tokenizer_kwargs={},

801 + tokenizer=mock_tokenizer,

802 )

803 - mock_renderer._tokenizer = mock_tokenizer

803 # prompt_token_ids length == max_model_len should be rejected for

804 # completion-like requests (ChatCompletionRequest).

805 mock_renderer.render_messages_async = AsyncMock(

tests/renderers/test_completions.py

+12 -10

@@ -38,6 +38,7 @@ class MockModelConfig:

38 enable_prompt_embeds: bool = True

39 skip_tokenizer_init: bool = False

40 is_encoder_decoder: bool = False

41 + is_multimodal_model: bool = False

44 @dataclass

@@ -78,15 +79,16 @@ def _build_renderer(

80 renderer = HfRenderer(

81 MockVllmConfig(model_config),

81 - tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

82 + tokenizer=(

83 + None

84 + if model_config.skip_tokenizer_init

85 + else DummyTokenizer(

86 + truncation_side=truncation_side,

87 + max_chars_per_token=max_chars_per_token,

88 + )

89 + ),

90 )

84 - if not model_config.skip_tokenizer_init:

85 - renderer._tokenizer = DummyTokenizer(

86 - truncation_side=truncation_side,

87 - max_chars_per_token=max_chars_per_token,

88 - )

89 -

92 return renderer

@@ -277,7 +279,7 @@ class TestRenderPrompt:

279 )

280

281 # Should not even attempt tokenization

280 - assert renderer._tokenizer._captured_encode_kwargs == {}

282 + assert renderer.tokenizer._captured_encode_kwargs == {}

283

284 def test_text_max_length_exceeded_nonobvious(self):

285 renderer = _build_renderer(MockModelConfig(), max_chars_per_token=2)

@@ -298,8 +300,8 @@ class TestRenderPrompt:

300 )

301

302 # Should only tokenize the first max_total_tokens + 1 tokens

301 - assert renderer._tokenizer._captured_encode_kwargs["truncation"] is True

302 - assert renderer._tokenizer._captured_encode_kwargs["max_length"] == 101

303 + assert renderer.tokenizer._captured_encode_kwargs["truncation"] is True

304 + assert renderer.tokenizer._captured_encode_kwargs["max_length"] == 101

305

306 def test_token_max_length_exceeded(self):

307 renderer = _build_renderer(MockModelConfig())

tests/renderers/test_mistral.py

+2 -2

@@ -36,6 +36,7 @@ class MockModelConfig:

36 enable_prompt_embeds: bool = True

37 skip_tokenizer_init: bool = False

38 is_encoder_decoder: bool = False

39 + is_multimodal_model: bool = False

42 @dataclass

@@ -57,9 +58,8 @@ async def test_async_mistral_tokenizer_does_not_block_event_loop():

58 mock_tokenizer.apply_chat_template = mocked_apply_chat_template

59 mock_renderer = MistralRenderer(

60 MockVllmConfig(mock_model_config),

60 - tokenizer_kwargs={},

61 + tokenizer=mock_tokenizer,

62 )

62 - mock_renderer._tokenizer = mock_tokenizer

64 task = mock_renderer.render_messages_async([], ChatParams())

tests/v1/e2e/test_streaming_input.py

+1 -1

@@ -19,7 +19,7 @@ import pytest

19 import pytest_asyncio

21 from vllm import SamplingParams

22 -from vllm.inputs import StreamingInput

22 +from vllm.engine.protocol import StreamingInput

23 from vllm.outputs import RequestOutput

24 from vllm.platforms import current_platform

25 from vllm.sampling_params import RequestOutputKind

tests/v1/streaming_input/test_async_llm_streaming.py

+1 -1

@@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, MagicMock

8 import pytest

10 -from vllm.inputs import StreamingInput

10 +from vllm.engine.protocol import StreamingInput

11 from vllm.outputs import RequestOutput

12 from vllm.sampling_params import RequestOutputKind, SamplingParams

13 from vllm.v1.engine.async_llm import AsyncLLM

vllm/benchmarks/mm_processor.py

+90 -6

@@ -18,7 +18,7 @@ import dataclasses

18 import json

19 import time

20 from datetime import datetime

21 -from typing import Any

21 +from typing import TYPE_CHECKING, Any

23 import numpy as np

@@ -28,9 +28,6 @@ from vllm.benchmarks.datasets import (

28 )

29 from vllm.benchmarks.throughput import get_requests

30 from vllm.engine.arg_utils import EngineArgs

31 -from vllm.multimodal.processing.context import (

32 - get_timing_stats_from_engine_client,

33 -)

31 from vllm.utils.gc_utils import freeze_gc_heap

32 from vllm.utils.import_utils import PlaceholderModule

@@ -39,16 +36,103 @@ try:

36 except ImportError:

37 pd = PlaceholderModule("pandas")

39 +if TYPE_CHECKING: # Avoid having to mock during docs build

40 + from vllm.v1.engine.llm_engine import LLMEngine

41 +else:

42 + LLMEngine = object

43 +

44 +

45 +def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]:

46 + """

47 + Get all multimodal timing stats from the LLM engine.

48 +

49 + Collects both preprocessing stats (HF processor, hashing, cache lookup,

50 + prompt update) and encoder forward pass timing, merged by request_id.

51 +

52 + Args:

53 + llm_engine: The LLM engine (has input_processor and workers).

54 +

55 + Returns:

56 + Dictionary mapping request_id to merged stats dict containing

57 + both preprocessing and encoder timing metrics.

58 +

59 + Example:

60 + {

61 + 'request-123': {

62 + 'hf_processor_time': 0.45,

63 + 'hashing_time': 0.02,

64 + 'cache_lookup_time': 0.01,

65 + 'prompt_update_time': 0.03,

66 + 'preprocessor_total_time': 0.51,

67 + 'encoder_forward_time': 0.23,

68 + 'num_encoder_calls': 1

69 + }

70 + }

71 + """

72 + observability_config = llm_engine.vllm_config.observability_config

73 + if not observability_config or not observability_config.enable_mm_processor_stats:

74 + return {}

75 +

76 + renderer = llm_engine.renderer

77 + mm_processor = renderer.get_mm_processor()

78 + preprocessing_stats = mm_processor.info.ctx.get_all_timing_stats()

79 +

80 + encoder_stats = dict[str, dict[str, float]]()

81 + for worker_stats in llm_engine.collective_rpc("get_encoder_timing_stats"):

82 + if not worker_stats:

83 + continue

84 +

85 + for request_id, stats_dict in worker_stats.items():

86 + if request_id not in encoder_stats:

87 + encoder_stats[request_id] = dict(stats_dict)

88 + else:

89 + # Aggregate timing metrics across workers

90 + current_time = encoder_stats[request_id].get(

91 + "encoder_forward_time", 0.0

92 + )

93 + new_time = stats_dict.get("encoder_forward_time", 0.0)

94 + encoder_stats[request_id]["encoder_forward_time"] = max(

95 + current_time, new_time

96 + )

97 +

98 + current_calls = encoder_stats[request_id].get("num_encoder_calls", 0)

99 + new_calls = stats_dict.get("num_encoder_calls", 0)

100 + encoder_stats[request_id]["num_encoder_calls"] = max(

101 + current_calls, new_calls

102 + )

103 +

104 + merged_stats = dict[str, dict[str, float]]()

105 +

106 + for request_id, prep_dict in preprocessing_stats.items():

107 + merged_stats[request_id] = dict(prep_dict)

108 +

109 + for request_id, enc_dict in encoder_stats.items():

110 + if request_id in merged_stats:

111 + merged_stats[request_id].update(enc_dict)

112 + continue

113 +

114 + # In V1 engine, the request_id in encoder_stats has a suffix

115 + # appended to the original request_id (which is used in

116 + # preprocessing_stats).

117 + # We try to strip the suffix to find the matching request.

118 + possible_original_id = request_id.rpartition("-")[0]

119 + if possible_original_id and possible_original_id in merged_stats:

120 + merged_stats[possible_original_id].update(enc_dict)

121 + else:

122 + merged_stats[request_id] = dict(enc_dict)

123 +

124 + return merged_stats

125 +

126

127 def collect_mm_processor_stats(

44 - llm_engine: Any,

128 + llm_engine: LLMEngine,

129 num_warmup_reqs: int = 0,

130 ) -> dict[str, list[float]]:

131 """

132 Collect multimodal processor timing stats.

133 Returns a dictionary mapping stage names to lists of timing values (in seconds).

134 """

51 - all_stats = get_timing_stats_from_engine_client(llm_engine)

135 + all_stats = get_timing_stats_from_engine(llm_engine)

136

137 stat_keys = [

138 "hf_processor_time",

vllm/engine/protocol.py

+14 -1

@@ -3,6 +3,7 @@

4 from abc import ABC, abstractmethod

5 from collections.abc import AsyncGenerator, Iterable, Mapping

6 +from dataclasses import dataclass

7 from typing import TYPE_CHECKING, Any

9 from vllm.config import ModelConfig, VllmConfig

@@ -10,7 +11,7 @@ from vllm.distributed.weight_transfer.base import (

11 WeightTransferInitRequest,

12 WeightTransferUpdateRequest,

13 )

13 -from vllm.inputs.data import PromptType, StreamingInput

14 +from vllm.inputs.data import PromptType

15 from vllm.lora.request import LoRARequest

16 from vllm.outputs import PoolingRequestOutput, RequestOutput

17 from vllm.plugins.io_processors import IOProcessor

@@ -26,6 +27,18 @@ if TYPE_CHECKING:

27 from vllm.v1.engine import PauseMode

30 +@dataclass

31 +class StreamingInput:

32 + """Input data for a streaming generation request.

33 +

34 + This is used with generate() to support multi-turn streaming sessions

35 + where inputs are provided via an async generator.

36 + """

37 +

38 + prompt: PromptType

39 + sampling_params: SamplingParams | None = None

40 +

41 +

42 class EngineClient(ABC):

43 """Protocol class for Clients to Engine"""

vllm/entrypoints/llm.py

+16 -29

@@ -72,7 +72,7 @@ from vllm.outputs import (

72 )

73 from vllm.platforms import current_platform

74 from vllm.pooling_params import PoolingParams

75 -from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs

75 +from vllm.renderers import ChatParams, merge_kwargs

76 from vllm.renderers.inputs import DictPrompt, TokPrompt

77 from vllm.renderers.inputs.preprocess import (

78 conversation_to_seq,

@@ -384,7 +384,7 @@ class LLM:

384 return parallel_config.world_size

385

386 def reset_mm_cache(self) -> None:

387 - self.input_processor.clear_mm_cache()

387 + self.renderer.clear_mm_cache()

388 self.llm_engine.reset_mm_cache()

389

390 def get_default_sampling_params(self) -> SamplingParams:

@@ -876,19 +876,6 @@ class LLM:

876

877 return outputs

878

879 - def _get_cmpl_tok_params(self, tokenization_kwargs: dict[str, Any] | None):

880 - model_config = self.model_config

881 - encoder_config = model_config.encoder_config or {}

882 -

883 - return TokenizeParams(

884 - max_total_tokens=model_config.max_model_len,

885 - do_lower_case=encoder_config.get("do_lower_case", False),

886 - # For Whisper, special tokens should be provided by the user based

887 - # on the task and language of their request. Also needed to avoid

888 - # appending an EOS token to the prompt which disrupts generation.

889 - add_special_tokens=not model_config.is_encoder_decoder,

890 - ).with_kwargs(tokenization_kwargs)

891 -

879 def _preprocess_cmpl(

880 self,

881 prompts: Sequence[PromptType],

@@ -910,20 +897,12 @@ class LLM:

897 parsed_prompts = [

898 parse_model_prompt(model_config, prompt) for prompt in prompts

899 ]

913 - tok_params = self._get_cmpl_tok_params(tokenization_kwargs)

900 + tok_params = renderer.default_cmpl_tok_params.with_kwargs(

901 + **(tokenization_kwargs or {})

902 + )

903

904 return renderer.render_cmpl(parsed_prompts, tok_params)

905

917 - def _get_chat_tok_params(self, tokenization_kwargs: dict[str, Any] | None):

918 - model_config = self.model_config

919 - encoder_config = model_config.encoder_config or {}

920 -

921 - return TokenizeParams(

922 - max_total_tokens=model_config.max_model_len,

923 - do_lower_case=encoder_config.get("do_lower_case", False),

924 - add_special_tokens=False,

925 - ).with_kwargs(tokenization_kwargs)

926 -

906 def _preprocess_chat(

907 self,

908 conversations: Sequence[list[ChatCompletionMessageParam]],

@@ -961,7 +940,9 @@ class LLM:

940 ),

941 ),

942 )

964 - tok_params = self._get_chat_tok_params(tokenization_kwargs)

943 + tok_params = renderer.default_chat_tok_params.with_kwargs(

944 + **(tokenization_kwargs or {})

945 + )

946

947 _, engine_prompts = renderer.render_chat(

948 conversations,

@@ -1653,7 +1634,10 @@ class LLM:

1634 architecture=architecture,

1635 )

1636

1656 - tok_params = self._get_cmpl_tok_params(tokenization_kwargs)

1637 + renderer = self.renderer

1638 + tok_params = renderer.default_cmpl_tok_params.with_kwargs(

1639 + **(tokenization_kwargs or {})

1640 + )

1641 encode_kwargs = tok_params.get_encode_kwargs()

1642

1643 if model_config.is_cross_encoder:

@@ -1970,7 +1954,10 @@ class LLM:

1954 dict(truncate_prompt_tokens=params.truncate_prompt_tokens),

1955 )

1956

1973 - tok_params = self._get_cmpl_tok_params(tokenization_kwargs)

1957 + renderer = self.renderer

1958 + tok_params = renderer.default_cmpl_tok_params.with_kwargs(

1959 + **(tokenization_kwargs or {})

1960 + )

1961

1962 tokenization_kwargs = tok_params.get_encode_kwargs()

1963 engine_request = self.input_processor.process_inputs(

vllm/entrypoints/openai/realtime/serving.py

+2 -2

@@ -8,11 +8,11 @@ from typing import Literal, cast

9 import numpy as np

11 -from vllm.engine.protocol import EngineClient

11 +from vllm.engine.protocol import EngineClient, StreamingInput

12 from vllm.entrypoints.logger import RequestLogger

13 from vllm.entrypoints.openai.engine.serving import OpenAIServing

14 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

15 -from vllm.inputs.data import PromptType, StreamingInput

15 +from vllm.inputs.data import PromptType

16 from vllm.logger import init_logger

17 from vllm.model_executor.models.interfaces import SupportsRealtime

vllm/inputs/__init__.py

-2

@@ -12,7 +12,6 @@ from .data import (

12 PromptType,

13 SingletonInputs,

14 SingletonPrompt,

15 - StreamingInput,

15 TextPrompt,

16 TokenInputs,

17 TokensPrompt,

@@ -36,5 +35,4 @@ __all__ = [

35 "EncoderDecoderInputs",

36 "ProcessorInputs",

37 "SingletonInputs",

39 - "StreamingInput",

38 ]

vllm/inputs/data.py

-15

@@ -1,13 +1,10 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from dataclasses import dataclass

3 from typing import TYPE_CHECKING, Any, Literal, TypeAlias

5 import torch

6 from typing_extensions import NotRequired, TypedDict

9 -from vllm.sampling_params import SamplingParams

10 -

8 if TYPE_CHECKING:

9 from vllm.multimodal.inputs import (

10 MultiModalDataDict,

@@ -299,15 +296,3 @@ which can be passed to

296

297 SingletonInputs: TypeAlias = DecoderOnlyInputs | MultiModalEncDecInputs

298 """The inputs for a single encoder/decoder prompt."""

302 -

303 -

304 -@dataclass

305 -class StreamingInput:

306 - """Input data for a streaming generation request.

307 -

308 - This is used with generate() to support multi-turn streaming sessions

309 - where inputs are provided via an async generator.

310 - """

311 -

312 - prompt: PromptType

313 - sampling_params: SamplingParams | None = None

vllm/inputs/preprocess.py

+23 -91

@@ -9,13 +9,11 @@ from typing_extensions import assert_never

9 from vllm.config import VllmConfig

10 from vllm.logger import init_logger

11 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

12 -from vllm.multimodal.cache import BaseMultiModalProcessorCache

12 from vllm.multimodal.inputs import (

13 MultiModalDataDict,

14 MultiModalInputs,

15 MultiModalUUIDDict,

16 )

18 -from vllm.multimodal.processing import BaseMultiModalProcessor

17 from vllm.renderers import BaseRenderer, renderer_from_config

18 from vllm.renderers.inputs import (

19 DecoderDictPrompt,

@@ -28,8 +26,6 @@ from vllm.renderers.inputs import (

26 )

27 from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt

28 from vllm.tokenizers import TokenizerLike

31 -from vllm.utils.jsontree import json_iter_leaves

32 -from vllm.v1.metrics.stats import MultiModalCacheStats

30 from .data import (

31 DecoderInputs,

@@ -57,17 +53,12 @@ class InputPreprocessor:

53 vllm_config: VllmConfig,

54 renderer: BaseRenderer | None = None,

55 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

60 - mm_processor_cache: BaseMultiModalProcessorCache | None = None,

56 ) -> None:

57 super().__init__()

59 self.model_config = vllm_config.model_config

65 - self.observability_config = vllm_config.observability_config

60 self.renderer = renderer or renderer_from_config(vllm_config)

61 self.mm_registry = mm_registry

68 - self.mm_processor_cache = mm_processor_cache

69 -

70 - self.mm_cache_stats = MultiModalCacheStats() if mm_processor_cache else None

63 @property

64 def tokenizer(self) -> TokenizerLike | None:

@@ -124,23 +115,6 @@ class InputPreprocessor:

115

116 return decoder_input_ids

117

127 - def _get_tokenization_kw(

128 - self,

129 - overrides: dict[str, Any] | None = None,

130 - ) -> dict[str, Any]:

131 - kwargs = dict[str, Any]()

132 -

133 - if self.model_config.is_encoder_decoder:

134 - # For Whisper, special tokens should be provided by the user based

135 - # on the task and language of their request. Also needed to avoid

136 - # appending an EOS token to the prompt which disrupts generation.

137 - kwargs["add_special_tokens"] = False

138 -

139 - if overrides:

140 - kwargs.update(overrides)

141 -

142 - return kwargs

143 -

118 def _tokenize_prompt(

119 self,

120 prompt: str,

@@ -150,26 +124,18 @@ class InputPreprocessor:

124 Apply the model's tokenizer to a text prompt, returning the

125 corresponding token IDs.

126 """

153 - tokenizer = self.get_tokenizer()

154 - tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)

155 -

156 - encoder_config = self.model_config.encoder_config

127 + renderer = self.renderer

128

158 - if encoder_config and encoder_config.get("do_lower_case", False):

159 - prompt = prompt.lower()

160 -

161 - return tokenizer.encode(prompt, **tokenization_kwargs)

129 + tok_params = renderer.default_cmpl_tok_params.with_kwargs(

130 + **(tokenization_kwargs or {})

131 + )

132

163 - def _get_mm_processor(self) -> BaseMultiModalProcessor:

164 - if not hasattr(self, "_mm_processor"):

165 - self._mm_processor = self.mm_registry.create_processor(

166 - self.model_config,

167 - self.observability_config,

168 - tokenizer=self.tokenizer,

169 - cache=self.mm_processor_cache,

170 - )

133 + tok_prompt = renderer.tokenize_prompt(

134 + TextPrompt(prompt=prompt),

135 + tok_params,

136 + )

137

172 - return self._mm_processor

138 + return tok_prompt["prompt_token_ids"]

139

140 def _process_multimodal(

141 self,

@@ -184,33 +150,20 @@ class InputPreprocessor:

150 Apply the model's multi-modal processor to a multi-modal prompt,

151 returning the corresponding token IDs and metadata.

152 """

187 - mm_processor = self._get_mm_processor()

153 + mm_processor = self.renderer.get_mm_processor()

154

155 if mm_processor_kwargs is None:

156 mm_processor_kwargs = {}

157

158 mm_items = mm_processor.info.parse_mm_data(mm_data)

193 - mm_input = mm_processor.apply(

159 +

160 + return mm_processor.apply(

161 prompt,

162 mm_items,

163 hf_processor_mm_kwargs=mm_processor_kwargs,

164 tokenization_kwargs=tokenization_kwargs,

165 mm_uuids=mm_uuids,

166 )

200 - mm_hashes = mm_input["mm_hashes"]

201 -

202 - # Validate that all mm items have a string as their hash

203 - contains_only_strings = all(

204 - isinstance(leaf, str) for leaf in json_iter_leaves(mm_hashes)

205 - )

206 - if not contains_only_strings:

207 - raise ValueError(

208 - f"mm_hashes must contain only strings, got: {mm_hashes}. "

209 - "This is likely due to an incorrect custom implementation of "

210 - "MultiModalProcessor.apply method."

211 - )

212 -

213 - return mm_input

167

168 def _process_embeds(

169 self,

@@ -245,19 +198,18 @@ class InputPreprocessor:

198 def _truncate_inputs(

199 self, inputs: list[int], tokenization_kwargs: dict[str, Any] | None = None

200 ) -> list[int]:

248 - if (

249 - not tokenization_kwargs

250 - or "truncation" not in tokenization_kwargs

251 - or self.tokenizer is None

252 - ):

253 - return inputs

201 + renderer = self.renderer

202

255 - max_length = tokenization_kwargs["max_length"]

203 + tok_params = renderer.default_cmpl_tok_params.with_kwargs(

204 + **(tokenization_kwargs or {})

205 + )

206

257 - if self.tokenizer.truncation_side == "left":

258 - return inputs[-max_length:]

259 - else:

260 - return inputs[:max_length]

207 + tok_prompt = renderer.tokenize_prompt(

208 + TokensPrompt(prompt_token_ids=inputs),

209 + tok_params,

210 + )

211 +

212 + return tok_prompt["prompt_token_ids"]

213

214 def _process_tokens(

215 self,

@@ -539,26 +491,6 @@ class InputPreprocessor:

491 """Preprocess the input prompt."""

492 res = self._preprocess(prompt, tokenization_kwargs, mm_uuids=mm_uuids)

493

542 - if self.mm_processor_cache and self.mm_cache_stats is not None:

543 - delta = self.mm_processor_cache.make_stats(delta=True)

544 - self.mm_cache_stats.requests += 1

545 - self.mm_cache_stats.queries += delta.total

546 - self.mm_cache_stats.hits += delta.hits

494 + self.renderer.update_mm_cache_stats()

495

496 return res

549 -

550 - def stat_mm_cache(self) -> MultiModalCacheStats | None:

551 - mm_cache_stats = self.mm_cache_stats

552 - if mm_cache_stats is None:

553 - return None

554 -

555 - self.mm_cache_stats = MultiModalCacheStats()

556 -

557 - return mm_cache_stats

558 -

559 - def clear_mm_cache(self) -> None:

560 - if self.mm_processor_cache is not None:

561 - self.mm_processor_cache.clear_cache()

562 -

563 - if self.mm_cache_stats is not None:

564 - self.mm_cache_stats.reset = True

vllm/model_executor/models/clip.py

+16 -7

@@ -208,14 +208,23 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):

208 *,

209 mm_uuids: MultiModalUUIDDict | None = None,

210 ) -> MultiModalInputs:

211 - if prompt and mm_items:

212 - raise ValueError(

213 - "CLIP accepts text-only or image-only inputs, not both! "

214 - "Image-only inputs means passing an image with an empty text "

215 - "prompt."

216 - )

217 -

211 if mm_items:

212 + if isinstance(prompt, str):

213 + if len(prompt) > 0:

214 + raise ValueError(

215 + "CLIP accepts text-only or image-only inputs, not both! "

216 + "You must pass an image with an empty text prompt."

217 + )

218 + else:

219 + special_tokens = self.info.get_tokenizer().all_special_ids

220 + if all(tok in special_tokens for tok in prompt):

221 + prompt = []

222 + else:

223 + raise ValueError(

224 + "CLIP accepts text-only or image-only inputs, not both! "

225 + "You must pass an image with an empty token prompt."

226 + )

227 +

228 # For multi-modal data, the prompt after processing should

229 # only contain the dummy image tokens

230 tokenization_kwargs = {

vllm/model_executor/models/lfm2_vl.py

@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (

42 PromptReplacement,

43 PromptUpdateDetails,

44 )

45 +from vllm.renderers import TokenizeParams

46 from vllm.sequence import IntermediateTensors

47 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -90,6 +91,9 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):

91 def get_image_processor(self, **kwargs: object) -> Lfm2VlImageProcessorFast:

92 return self.get_hf_processor(**kwargs).image_processor

94 + def get_default_tok_params(self) -> TokenizeParams:

95 + return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

96 +

97 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

98 return {"image": None}

vllm/model_executor/models/mllama4.py

@@ -66,6 +66,7 @@ from vllm.multimodal.processing import (

66 PromptUpdate,

67 PromptUpdateDetails,

68 )

69 +from vllm.renderers import TokenizeParams

70 from vllm.sequence import IntermediateTensors

71 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -554,6 +555,9 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):

555 Llama4Processor, use_fast=kwargs.pop("use_fast", True), **kwargs

556 )

557

558 + def get_default_tok_params(self) -> TokenizeParams:

559 + return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

560 +

561 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

562 # Although vLLM can support more images from an infra capability

563 # perspective, we do not recommend using >10 images in practice.

vllm/model_executor/models/nano_nemotron_vl.py

@@ -76,6 +76,7 @@ from vllm.multimodal.processing.processor import (

76 PromptUpdateDetails,

77 _seq2tokens,

78 )

79 +from vllm.renderers import TokenizeParams

80 from vllm.sequence import IntermediateTensors

81 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config

82 from vllm.transformers_utils.configs.radio import RadioConfig

@@ -1093,6 +1094,9 @@ class BaseNanoNemotronVLProcessingInfo(BaseProcessingInfo):

1094 ) -> BaseNanoNemotronVLProcessor:

1095 raise NotImplementedError

1096

1097 + def get_default_tok_params(self) -> TokenizeParams:

1098 + return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

1099 +

1100 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

1101 return {"image": None}

1102

vllm/model_executor/models/nemotron_parse.py

@@ -58,6 +58,7 @@ from vllm.multimodal.processing import (

58 PromptReplacement,

59 PromptUpdate,

60 )

61 +from vllm.renderers import TokenizeParams

62 from vllm.tokenizers import TokenizerLike

63 from vllm.transformers_utils.configs.radio import RadioConfig

64 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -608,6 +609,9 @@ class NemotronParseProcessingInfo(BaseProcessingInfo):

609 **kwargs,

610 )

611

612 + def get_default_tok_params(self) -> TokenizeParams:

613 + return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

614 +

615 @property

616 def skip_prompt_length_check(self) -> bool:

617 return True # Because the encoder prompt is padded

vllm/model_executor/models/ovis.py

@@ -53,6 +53,7 @@ from vllm.multimodal.processing import (

53 BaseProcessingInfo,

54 PromptReplacement,

55 )

56 +from vllm.renderers import TokenizeParams

57 from vllm.sequence import IntermediateTensors

58 from vllm.transformers_utils.processors.ovis import OvisProcessor

59 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -264,6 +265,9 @@ class OvisProcessingInfo(BaseProcessingInfo):

265 **kwargs,

266 )

267

268 + def get_default_tok_params(self) -> TokenizeParams:

269 + return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

270 +

271 def get_image_segment_len(self) -> int:

272 visual_tokenizer_config = self.get_hf_config().visual_tokenizer_config

273 image_size = visual_tokenizer_config.backbone_config.image_size

vllm/model_executor/models/ovis2_5.py

@@ -35,6 +35,7 @@ from vllm.multimodal.processing import (

35 BaseProcessingInfo,

36 PromptReplacement,

37 )

38 +from vllm.renderers import TokenizeParams

39 from vllm.sequence import IntermediateTensors

40 from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor

41 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -183,6 +184,9 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):

184 temporal_patch_size=vit_config.temporal_patch_size,

185 )

186

187 + def get_default_tok_params(self) -> TokenizeParams:

188 + return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

189 +

190 def get_image_processor(self) -> BaseImageProcessor:

191 return self.get_hf_processor().image_processor # type: ignore

192

vllm/model_executor/models/paligemma.py

@@ -32,6 +32,7 @@ from vllm.multimodal.processing import (

32 PromptUpdate,

33 PromptUpdateDetails,

34 )

35 +from vllm.renderers import TokenizeParams

36 from vllm.sequence import IntermediateTensors

37 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -102,6 +103,9 @@ class PaliGemmaProcessingInfo(BaseProcessingInfo):

103 def get_vision_encoder_info(self):

104 return get_vision_encoder_info(self.get_hf_config())

105

106 + def get_default_tok_params(self) -> TokenizeParams:

107 + return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

108 +

109 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

110 return {"image": 1}

111

vllm/model_executor/models/siglip.py

+16 -7

@@ -194,14 +194,23 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):

194 *,

195 mm_uuids: MultiModalUUIDDict | None = None,

196 ) -> MultiModalInputs:

197 - if prompt and mm_items:

198 - raise ValueError(

199 - "Siglip accepts text-only or image-only inputs, not both! "

200 - "Image-only inputs means passing an image with an empty text "

201 - "prompt."

202 - )

203 -

197 if mm_items:

198 + if isinstance(prompt, str):

199 + if len(prompt) > 0:

200 + raise ValueError(

201 + "SigLIP accepts text-only or image-only inputs, not both! "

202 + "You must pass an image with an empty text prompt."

203 + )

204 + else:

205 + special_tokens = self.info.get_tokenizer().all_special_ids

206 + if all(tok in special_tokens for tok in prompt):

207 + prompt = []

208 + else:

209 + raise ValueError(

210 + "SigLIP accepts text-only or image-only inputs, not both! "

211 + "You must pass an image with an empty token prompt."

212 + )

213 +

214 # For multi-modal data, the prompt after processing should

215 # only contain the image token

216 tokenization_kwargs = {

vllm/model_executor/models/ultravox.py

@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (

42 PromptReplacement,

43 PromptUpdate,

44 )

45 +from vllm.renderers import TokenizeParams

46 from vllm.sequence import IntermediateTensors

47 from vllm.transformers_utils.configs.ultravox import UltravoxConfig

48 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -133,6 +134,9 @@ class UltravoxProcessingInfo(BaseProcessingInfo):

134 assert isinstance(feature_extractor, WhisperFeatureExtractor)

135 return feature_extractor

136

137 + def get_default_tok_params(self) -> TokenizeParams:

138 + return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

139 +

140 def get_data_parser(self):

141 feature_extractor = self.get_feature_extractor()

142

vllm/model_executor/models/voxtral_realtime.py

+2 -1

@@ -17,8 +17,9 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig

18 from vllm.compilation.decorators import support_torch_compile

19 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

20 +from vllm.engine.protocol import StreamingInput

21 from vllm.envs import VLLM_ENGINE_ITERATION_TIMEOUT_S

21 -from vllm.inputs.data import PromptType, StreamingInput, TokensPrompt

22 +from vllm.inputs.data import PromptType, TokensPrompt

23 from vllm.logger import init_logger

24 from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime

25 from vllm.model_executor.models.voxtral import (

vllm/model_executor/models/whisper.py

@@ -55,6 +55,7 @@ from vllm.multimodal.processing import (

55 PromptReplacement,

56 PromptUpdate,

57 )

58 +from vllm.renderers import TokenizeParams

59 from vllm.transformers_utils.processor import cached_processor_from_config

60 from vllm.utils.jsontree import json_map_leaves

61 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -644,6 +645,12 @@ class WhisperProcessingInfo(BaseProcessingInfo):

645 def get_hf_config(self) -> WhisperConfig:

646 return self.ctx.get_hf_config(WhisperConfig)

647

648 + def get_default_tok_params(self) -> TokenizeParams:

649 + # Special tokens should be provided by the user based on the

650 + # task and language of their request. Also needed to avoid

651 + # appending an EOS token to the prompt which disrupts generation.

652 + return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

653 +

654 def get_data_parser(self):

655 feature_extractor = self.get_feature_extractor()

656

vllm/multimodal/processing/context.py

+16 -104

@@ -21,6 +21,7 @@ from vllm.multimodal.parse import (

21 MultiModalDataItems,

22 MultiModalDataParser,

23 )

24 +from vllm.renderers import TokenizeParams

25 from vllm.tokenizers import TokenizerLike

26 from vllm.transformers_utils.processor import cached_processor_from_config

27 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides

@@ -93,110 +94,6 @@ class MultiModalProcessorTimingStats:

94 }

96 -def get_timing_stats_from_engine_client(

97 - engine_client: Any,

98 -) -> dict[str, dict[str, float]]:

99 - """

100 - Get all multimodal timing stats from the engine client.

101 -

102 - Collects both preprocessing stats (HF processor, hashing, cache lookup,

103 - prompt update) and encoder forward pass timing, merged by request_id.

104 -

105 - Args:

106 - engine_client: The engine client (has input_processor and workers).

107 -

108 - Returns:

109 - Dictionary mapping request_id to merged stats dict containing

110 - both preprocessing and encoder timing metrics.

111 -

112 - Example:

113 - {

114 - 'request-123': {

115 - 'hf_processor_time': 0.45,

116 - 'hashing_time': 0.02,

117 - 'cache_lookup_time': 0.01,

118 - 'prompt_update_time': 0.03,

119 - 'preprocessor_total_time': 0.51,

120 - 'encoder_forward_time': 0.23,

121 - 'num_encoder_calls': 1

122 - }

123 - }

124 - """

125 - try:

126 - if not engine_client.vllm_config.observability_config.enable_mm_processor_stats:

127 - return {}

128 - except (AttributeError, RuntimeError):

129 - return {}

130 -

131 - preprocessing_stats = {}

132 - try:

133 - input_processor = engine_client.input_processor

134 - input_preprocessor = input_processor.input_preprocessor

135 -

136 - if hasattr(input_preprocessor, "_get_mm_processor"):

137 - mm_processor = input_preprocessor._get_mm_processor()

138 - if mm_processor is not None and hasattr(mm_processor, "info"):

139 - ctx = mm_processor.info.ctx

140 - preprocessing_stats = ctx.get_all_timing_stats()

141 - except (AttributeError, RuntimeError):

142 - pass

143 -

144 - encoder_stats = {}

145 - try:

146 - if hasattr(engine_client, "collective_rpc"):

147 - encoder_stats_results = engine_client.collective_rpc(

148 - "get_encoder_timing_stats"

149 - )

150 - if encoder_stats_results and len(encoder_stats_results) > 0:

151 - for worker_stats in encoder_stats_results:

152 - if not worker_stats:

153 - continue

154 - for request_id, stats_dict in worker_stats.items():

155 - if request_id not in encoder_stats:

156 - encoder_stats[request_id] = dict(stats_dict)

157 - else:

158 - # Aggregate timing metrics across workers

159 - current_time = encoder_stats[request_id].get(

160 - "encoder_forward_time", 0.0

161 - )

162 - new_time = stats_dict.get("encoder_forward_time", 0.0)

163 - encoder_stats[request_id]["encoder_forward_time"] = max(

164 - current_time, new_time

165 - )

166 -

167 - current_calls = encoder_stats[request_id].get(

168 - "num_encoder_calls", 0

169 - )

170 - new_calls = stats_dict.get("num_encoder_calls", 0)

171 - encoder_stats[request_id]["num_encoder_calls"] = max(

172 - current_calls, new_calls

173 - )

174 - except (AttributeError, RuntimeError):

175 - pass

176 -

177 - merged_stats = {}

178 -

179 - for request_id, prep_dict in preprocessing_stats.items():

180 - merged_stats[request_id] = dict(prep_dict)

181 -

182 - for request_id, enc_dict in encoder_stats.items():

183 - if request_id in merged_stats:

184 - merged_stats[request_id].update(enc_dict)

185 - continue

186 -

187 - # In V1 engine, the request_id in encoder_stats has a suffix

188 - # appended to the original request_id (which is used in

189 - # preprocessing_stats).

190 - # We try to strip the suffix to find the matching request.

191 - possible_original_id = request_id.rpartition("-")[0]

192 - if possible_original_id and possible_original_id in merged_stats:

193 - merged_stats[possible_original_id].update(enc_dict)

194 - else:

195 - merged_stats[request_id] = dict(enc_dict)

196 -

197 - return merged_stats

198 -

199 -

97 @contextmanager

98 def timed_preprocessor_operation(ctx: "InputProcessingContext", stage_name: str):

99 """

@@ -576,6 +473,21 @@ class BaseProcessingInfo:

473 """

474 return self.ctx.get_hf_processor(**kwargs)

475

476 + def get_default_tok_params(self) -> TokenizeParams:

477 + """Construct the default parameters for tokenization."""

478 + model_config = self.ctx.model_config

479 + encoder_config = model_config.encoder_config or {}

480 +

481 + return TokenizeParams(

482 + max_total_tokens=model_config.max_model_len,

483 + do_lower_case=encoder_config.get("do_lower_case", False),

484 + add_special_tokens=True,

485 + )

486 +

487 + @cached_property

488 + def default_tok_params(self) -> TokenizeParams:

489 + return self.get_default_tok_params()

490 +

491 def _get_expected_hidden_size(self) -> int | None:

492 """

493 Get expected hidden size for embedding validation if `mm_embeds` are enabled.

vllm/renderers/base.py

+123 -22

@@ -3,12 +3,17 @@

3 import asyncio

4 from abc import ABC, abstractmethod

5 from collections.abc import Sequence

6 -from typing import TYPE_CHECKING, Any, overload

6 +from functools import cached_property

7 +from typing import TYPE_CHECKING, Any, Generic, overload

8 +

9 +from typing_extensions import TypeVar

11 from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

12 from vllm.logger import init_logger

13 from vllm.tokenizers import TokenizerLike

14 from vllm.utils.async_utils import AsyncMicrobatchTokenizer

15 +from vllm.utils.torch_utils import set_default_torch_num_threads

16 +from vllm.v1.metrics.stats import MultiModalCacheStats

18 from .embed_utils import safe_load_prompt_embeds

19 from .inputs import (

@@ -26,11 +31,16 @@ if TYPE_CHECKING:

31 ChatCompletionMessageParam,

32 ConversationMessage,

33 )

34 + from vllm.multimodal.cache import BaseMultiModalProcessorCache

35 + from vllm.multimodal.processing import BaseMultiModalProcessor

37 logger = init_logger(__name__)

33 -class BaseRenderer(ABC):

40 +_T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike)

41 +

42 +

43 +class BaseRenderer(ABC, Generic[_T]):

44 @classmethod

45 @abstractmethod

46 def from_config(

@@ -40,20 +50,36 @@ class BaseRenderer(ABC):

50 ) -> "BaseRenderer":

51 raise NotImplementedError

43 - def __init__(self, config: "VllmConfig") -> None:

53 + def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None:

54 super().__init__()

56 + self.config = config

57 self.model_config = config.model_config

59 + self.tokenizer = tokenizer

60 +

61 # Lazy initialization since offline LLM doesn't use async

62 self._async_tokenizer: AsyncMicrobatchTokenizer | None = None

51 - @property

52 - @abstractmethod

53 - def tokenizer(self) -> TokenizerLike | None:

54 - raise NotImplementedError

64 + self.mm_processor: BaseMultiModalProcessor | None = None

65 + self._mm_cache_stats: MultiModalCacheStats | None = None

66 + if config.model_config.is_multimodal_model:

67 + from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry

68 +

69 + mm_processor_cache = mm_registry.processor_cache_from_config(config)

70 +

71 + with set_default_torch_num_threads():

72 + self.mm_processor = mm_registry.create_processor(

73 + config.model_config,

74 + config.observability_config,

75 + tokenizer=tokenizer,

76 + cache=mm_processor_cache,

77 + )

78 +

79 + if mm_processor_cache:

80 + self._mm_cache_stats = MultiModalCacheStats()

56 - def get_tokenizer(self) -> TokenizerLike:

82 + def get_tokenizer(self) -> _T:

83 tokenizer = self.tokenizer

84 if tokenizer is None:

85 raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")

@@ -66,6 +92,49 @@ class BaseRenderer(ABC):

93 return self._async_tokenizer

95 + def get_mm_processor(self) -> "BaseMultiModalProcessor":

96 + if self.mm_processor is None:

97 + raise ValueError("Multi-modal processor not available for text-only models")

98 +

99 + return self.mm_processor

100 +

101 + @property

102 + def mm_processor_cache(self) -> "BaseMultiModalProcessorCache | None":

103 + if self.mm_processor is None:

104 + return None

105 +

106 + return self.mm_processor.cache

107 +

108 + def stat_mm_cache(self) -> MultiModalCacheStats | None:

109 + mm_cache_stats = self._mm_cache_stats

110 + if mm_cache_stats is None:

111 + return None

112 +

113 + self._mm_cache_stats = MultiModalCacheStats()

114 +

115 + return mm_cache_stats

116 +

117 + def update_mm_cache_stats(self) -> None:

118 + mm_processor_cache = self.mm_processor_cache

119 + mm_cache_stats = self._mm_cache_stats

120 +

121 + if mm_processor_cache and mm_cache_stats:

122 + delta = mm_processor_cache.make_stats(delta=True)

123 + mm_cache_stats.record(delta.total, delta.hits)

124 +

125 + def clear_mm_cache(self) -> None:

126 + mm_processor_cache = self.mm_processor_cache

127 + if mm_processor_cache is not None:

128 + mm_processor_cache.clear_cache()

129 +

130 + if self._mm_cache_stats is not None:

131 + self._mm_cache_stats.reset = True

132 +

133 + def shutdown(self) -> None:

134 + mm_processor_cache = self.mm_processor_cache

135 + if mm_processor_cache is not None:

136 + mm_processor_cache.close()

137 +

138 def get_bos_token_id(self) -> int | None:

139 if self.tokenizer is None:

140 logger.warning_once(

@@ -84,6 +153,36 @@ class BaseRenderer(ABC):

153

154 return self.tokenizer.eos_token_id

155

156 + @cached_property

157 + def default_cmpl_tok_params(self) -> TokenizeParams:

158 + mm_processor = self.mm_processor

159 + if mm_processor is not None:

160 + return mm_processor.info.default_tok_params

161 +

162 + model_config = self.model_config

163 + encoder_config = model_config.encoder_config or {}

164 +

165 + return TokenizeParams(

166 + max_total_tokens=model_config.max_model_len,

167 + do_lower_case=encoder_config.get("do_lower_case", False),

168 + add_special_tokens=True,

169 + )

170 +

171 + @cached_property

172 + def default_chat_tok_params(self) -> TokenizeParams:

173 + mm_processor = self.mm_processor

174 + if mm_processor is not None:

175 + return mm_processor.info.default_tok_params

176 +

177 + model_config = self.model_config

178 + encoder_config = model_config.encoder_config or {}

179 +

180 + return TokenizeParams(

181 + max_total_tokens=model_config.max_model_len,

182 + do_lower_case=encoder_config.get("do_lower_case", False),

183 + add_special_tokens=False,

184 + )

185 +

186 # Step 1: Convert raw inputs to prompts

187 def render_prompt(

188 self,

@@ -317,18 +416,14 @@ class BaseRenderer(ABC):

416 def render_cmpl(

417 self,

418 prompts: Sequence[DictPrompt | bytes],

320 - tok_params: TokenizeParams,

419 + tok_params: TokenizeParams | None = None,

420 *,

421 prompt_extras: dict[str, Any] | None = None,

422 ):

324 - dict_prompts = self.render_prompts(prompts)

325 -

326 - # NOTE: Some MM models have non-default `add_special_tokens`

327 - # so we handle tokenization in multi-modal processor

328 - if self.model_config.is_multimodal_model:

329 - self._apply_prompt_extras(dict_prompts, prompt_extras)

330 - return dict_prompts

423 + if tok_params is None:

424 + tok_params = self.default_cmpl_tok_params

425

426 + dict_prompts = self.render_prompts(prompts)

427 tok_prompts = self.tokenize_prompts(dict_prompts, tok_params)

428

429 self._apply_prompt_extras(tok_prompts, prompt_extras)

@@ -339,14 +434,14 @@ class BaseRenderer(ABC):

434 async def render_cmpl_async(

435 self,

436 prompts: Sequence[DictPrompt | bytes],

342 - tok_params: TokenizeParams,

437 + tok_params: TokenizeParams | None = None,

438 *,

439 prompt_extras: dict[str, Any] | None = None,

440 ):

346 - dict_prompts = await self.render_prompts_async(prompts)

441 + if tok_params is None:

442 + tok_params = self.default_cmpl_tok_params

443

348 - # NOTE: MM data cannot be passed to online Completions API

349 - # so we don't have the special case that is in the offline version

444 + dict_prompts = await self.render_prompts_async(prompts)

445 tok_prompts = await self.tokenize_prompts_async(dict_prompts, tok_params)

446

447 self._apply_prompt_extras(tok_prompts, prompt_extras)

@@ -358,10 +453,13 @@ class BaseRenderer(ABC):

453 self,

454 conversations: Sequence[list["ChatCompletionMessageParam"]],

455 chat_params: ChatParams,

361 - tok_params: TokenizeParams,

456 + tok_params: TokenizeParams | None = None,

457 *,

458 prompt_extras: dict[str, Any] | None = None,

459 ):

460 + if tok_params is None:

461 + tok_params = self.default_chat_tok_params

462 +

463 rendered = [

464 self.render_messages(conversation, chat_params)

465 for conversation in conversations

@@ -384,10 +482,13 @@ class BaseRenderer(ABC):

482 self,

483 conversations: Sequence[list["ChatCompletionMessageParam"]],

484 chat_params: ChatParams,

387 - tok_params: TokenizeParams,

485 + tok_params: TokenizeParams | None = None,

486 *,

487 prompt_extras: dict[str, Any] | None = None,

488 ):

489 + if tok_params is None:

490 + tok_params = self.default_chat_tok_params

491 +

492 rendered = [

493 self.render_messages_async(conversation, chat_params)

494 for conversation in conversations

vllm/renderers/deepseek_v32.py

+5 -26

@@ -13,7 +13,6 @@ from vllm.logger import init_logger

13 from vllm.tokenizers import cached_get_tokenizer

14 from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer

16 -from ..tokenizers.hf import HfTokenizer

16 from .base import BaseRenderer

17 from .inputs import DictPrompt

18 from .inputs.preprocess import parse_dec_only_prompt

@@ -22,23 +21,14 @@ from .params import ChatParams

21 logger = init_logger(__name__)

25 -class DeepseekV32Renderer(BaseRenderer):

24 +class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):

25 @classmethod

27 - def from_config(

26 + def from_config( # type: ignore[override]

27 cls,

28 config: VllmConfig,

29 tokenizer_kwargs: dict[str, Any],

31 - ) -> "BaseRenderer":

32 - return cls(config, tokenizer_kwargs)

33 -

34 - def __init__(

35 - self,

36 - config: VllmConfig,

37 - tokenizer_kwargs: dict[str, Any],

38 - ) -> None:

39 - super().__init__(config)

40 -

41 - model_config = self.model_config

30 + ) -> "DeepseekV32Renderer":

31 + model_config = config.model_config

32 if model_config.skip_tokenizer_init:

33 tokenizer = None

34 else:

@@ -47,18 +37,7 @@ class DeepseekV32Renderer(BaseRenderer):

37 **tokenizer_kwargs,

38 )

50 - self._tokenizer = tokenizer

51 -

52 - @property

53 - def tokenizer(self) -> HfTokenizer | None:

54 - return self._tokenizer

55 -

56 - def get_tokenizer(self) -> HfTokenizer:

57 - tokenizer = self.tokenizer

58 - if tokenizer is None:

59 - raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")

60 -

61 - return tokenizer

40 + return cls(config, tokenizer)

42 def render_messages(

43 self,

vllm/renderers/grok2.py

+5 -25

@@ -21,23 +21,14 @@ from .params import ChatParams

21 logger = init_logger(__name__)

24 -class Grok2Renderer(BaseRenderer):

24 +class Grok2Renderer(BaseRenderer[Grok2Tokenizer]):

25 @classmethod

26 - def from_config(

26 + def from_config( # type: ignore[override]

27 cls,

28 config: VllmConfig,

29 tokenizer_kwargs: dict[str, Any],

30 - ) -> "BaseRenderer":

31 - return cls(config, tokenizer_kwargs)

32 -

33 - def __init__(

34 - self,

35 - config: VllmConfig,

36 - tokenizer_kwargs: dict[str, Any],

37 - ) -> None:

38 - super().__init__(config)

39 -

40 - model_config = self.model_config

30 + ) -> "Grok2Renderer":

31 + model_config = config.model_config

32 if model_config.skip_tokenizer_init:

33 tokenizer = None

34 else:

@@ -46,18 +37,7 @@ class Grok2Renderer(BaseRenderer):

37 **tokenizer_kwargs,

38 )

49 - self._tokenizer = tokenizer

50 -

51 - @property

52 - def tokenizer(self) -> Grok2Tokenizer | None:

53 - return self._tokenizer

54 -

55 - def get_tokenizer(self) -> Grok2Tokenizer:

56 - tokenizer = self.tokenizer

57 - if tokenizer is None:

58 - raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")

59 -

60 - return tokenizer

40 + return cls(config, tokenizer)

42 def render_messages(

43 self,

vllm/renderers/hf.py

+14 -27

@@ -585,27 +585,14 @@ def replace_vision_chunk_video_placeholder(

585 return prompt_raw

586

587

588 -class HfRenderer(BaseRenderer):

588 +class HfRenderer(BaseRenderer[HfTokenizer]):

589 @classmethod

590 - def from_config(

590 + def from_config( # type: ignore[override]

591 cls,

592 config: VllmConfig,

593 tokenizer_kwargs: dict[str, Any],

594 - ) -> "BaseRenderer":

595 - return cls(config, tokenizer_kwargs)

596 -

597 - def __init__(

598 - self,

599 - config: VllmConfig,

600 - tokenizer_kwargs: dict[str, Any],

601 - ) -> None:

602 - super().__init__(config)

603 -

604 - model_config = self.model_config

605 - self.use_unified_vision_chunk = getattr(

606 - model_config.hf_config, "use_unified_vision_chunk", False

607 - )

608 -

594 + ) -> "HfRenderer":

595 + model_config = config.model_config

596 if model_config.skip_tokenizer_init:

597 tokenizer = None

598 else:

@@ -617,18 +604,18 @@ class HfRenderer(BaseRenderer):

604 ),

605 )

606

620 - self._tokenizer = tokenizer

621 -

622 - @property

623 - def tokenizer(self) -> HfTokenizer | None:

624 - return self._tokenizer

607 + return cls(config, tokenizer)

608

626 - def get_tokenizer(self) -> HfTokenizer:

627 - tokenizer = self.tokenizer

628 - if tokenizer is None:

629 - raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")

609 + def __init__(

610 + self,

611 + config: VllmConfig,

612 + tokenizer: HfTokenizer | None,

613 + ) -> None:

614 + super().__init__(config, tokenizer)

615

631 - return tokenizer

616 + self.use_unified_vision_chunk = getattr(

617 + config.model_config.hf_config, "use_unified_vision_chunk", False

618 + )

619

620 def render_messages(

621 self,

vllm/renderers/mistral.py

+12 -25

@@ -50,23 +50,14 @@ def safe_apply_chat_template(

50 raise ValueError(str(e)) from e

53 -class MistralRenderer(BaseRenderer):

53 +class MistralRenderer(BaseRenderer[MistralTokenizer]):

54 @classmethod

55 - def from_config(

55 + def from_config( # type: ignore[override]

56 cls,

57 config: VllmConfig,

58 tokenizer_kwargs: dict[str, Any],

59 - ) -> "BaseRenderer":

60 - return cls(config, tokenizer_kwargs)

61 -

62 - def __init__(

63 - self,

64 - config: VllmConfig,

65 - tokenizer_kwargs: dict[str, Any],

66 - ) -> None:

67 - super().__init__(config)

68 -

69 - model_config = self.model_config

59 + ) -> "MistralRenderer":

60 + model_config = config.model_config

61 if model_config.skip_tokenizer_init:

62 tokenizer = None

63 else:

@@ -75,24 +66,20 @@ class MistralRenderer(BaseRenderer):

66 **tokenizer_kwargs,

67 )

78 - self._tokenizer = tokenizer

69 + return cls(config, tokenizer)

70 +

71 + def __init__(

72 + self,

73 + config: VllmConfig,

74 + tokenizer: MistralTokenizer | None,

75 + ) -> None:

76 + super().__init__(config, tokenizer)

78 self._apply_chat_template_executor = ThreadPoolExecutor(max_workers=1)

79 self._apply_chat_template_async = make_async(

80 safe_apply_chat_template, executor=self._apply_chat_template_executor

81 )

85 - @property

86 - def tokenizer(self) -> MistralTokenizer | None:

87 - return self._tokenizer

88 -

89 - def get_tokenizer(self) -> MistralTokenizer:

90 - tokenizer = self.tokenizer

91 - if tokenizer is None:

92 - raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")

93 -

94 - return tokenizer

95 -

83 def render_messages(

84 self,

85 messages: list[ChatCompletionMessageParam],

vllm/renderers/params.py

+6 -6

@@ -3,7 +3,6 @@

3 from dataclasses import dataclass, field

4 from typing import TYPE_CHECKING, Any, TypeVar

6 -from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption

6 from vllm.exceptions import VLLMValidationError

7 from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

8 from vllm.logger import init_logger

@@ -12,9 +11,13 @@ from vllm.utils.import_utils import LazyLoader

12 if TYPE_CHECKING:

13 import torch

14 +

15 + from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption

16 else:

17 torch = LazyLoader("torch", globals(), "torch")

19 + ChatTemplateContentFormatOption = object

20 +

21 logger = init_logger(__name__)

@@ -43,7 +46,7 @@ class ChatParams:

46 chat_template: str | None = None

47 """The chat template to apply."""

46 - chat_template_content_format: ChatTemplateContentFormatOption = "auto"

49 + chat_template_content_format: "ChatTemplateContentFormatOption" = "auto"

50 """The format of the chat template."""

52 chat_template_kwargs: dict[str, Any] = field(default_factory=dict)

@@ -163,10 +166,7 @@ class TokenizeParams:

166 value=truncate_prompt_tokens,

167 )

168

166 - def with_kwargs(self, tokenization_kwargs: dict[str, Any] | None):

167 - if tokenization_kwargs is None:

168 - tokenization_kwargs = {}

169 -

169 + def with_kwargs(self, **tokenization_kwargs: Any):

170 max_length = tokenization_kwargs.pop("max_length", self.max_input_tokens)

171 pad_prompt_tokens = tokenization_kwargs.pop(

172 "pad_prompt_tokens", self.pad_prompt_tokens

vllm/renderers/terratorch.py

+4 -15

@@ -10,7 +10,6 @@ from vllm.entrypoints.chat_utils import (

10 parse_chat_messages_async,

11 )

12 from vllm.logger import init_logger

13 -from vllm.tokenizers import TokenizerLike

14 from .base import BaseRenderer

15 from .inputs import DictPrompt

@@ -24,24 +23,14 @@ class TerratorchRenderer(BaseRenderer):

23 @classmethod

24 def from_config(

25 cls,

27 - config: VllmConfig,

26 + config: VllmConfig, # type: ignore[override]

27 tokenizer_kwargs: dict[str, Any],

29 - ) -> "BaseRenderer":

30 - return cls(config)

31 -

32 - def __init__(self, config: VllmConfig) -> None:

33 - super().__init__(config)

34 -

35 - model_config = self.model_config

28 + ) -> "TerratorchRenderer":

29 + model_config = config.model_config

30 if not model_config.skip_tokenizer_init:

31 raise ValueError("Terratorch renderer requires `skip_tokenizer_init=True`")

39 - @property

40 - def tokenizer(self) -> TokenizerLike | None:

41 - return None

42 -

43 - def get_tokenizer(self) -> TokenizerLike:

44 - raise ValueError("Tokenizer not available for Terratorch renderer")

33 + return cls(config, None)

35 def render_messages(

36 self,

vllm/v1/engine/async_llm.py

+8 -8

@@ -19,8 +19,8 @@ from vllm.distributed.weight_transfer.base import (

19 WeightTransferUpdateRequest,

20 )

21 from vllm.engine.arg_utils import AsyncEngineArgs

22 -from vllm.engine.protocol import EngineClient

23 -from vllm.inputs import PromptType, StreamingInput

22 +from vllm.engine.protocol import EngineClient, StreamingInput

23 +from vllm.inputs import PromptType

24 from vllm.logger import init_logger

25 from vllm.lora.request import LoRARequest

26 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

@@ -268,12 +268,12 @@ class AsyncLLM(EngineClient):

268

269 shutdown_prometheus()

270

271 + if renderer := getattr(self, "renderer", None):

272 + renderer.shutdown()

273 +

274 if engine_core := getattr(self, "engine_core", None):

275 engine_core.shutdown()

276

274 - if input_processor := getattr(self, "input_processor", None):

275 - input_processor.close()

276 -

277 handler = getattr(self, "output_handler", None)

278 if handler is not None:

279 cancel_task_threadsafe(handler)

@@ -654,7 +654,7 @@ class AsyncLLM(EngineClient):

654 output_processor = self.output_processor

655 log_stats = self.log_stats

656 logger_manager = self.logger_manager

657 - input_processor = self.input_processor

657 + renderer = self.renderer

658 chunk_size = envs.VLLM_V1_OUTPUT_PROC_CHUNK_SIZE

659

660 async def output_handler():

@@ -702,7 +702,7 @@ class AsyncLLM(EngineClient):

702 engine_idx=outputs.engine_index,

703 scheduler_stats=outputs.scheduler_stats,

704 iteration_stats=iteration_stats,

705 - mm_cache_stats=input_processor.stat_mm_cache(),

705 + mm_cache_stats=renderer.stat_mm_cache(),

706 )

707 except Exception as e:

708 logger.exception("AsyncLLM output_handler failed.")

@@ -881,7 +881,7 @@ class AsyncLLM(EngineClient):

881 await asyncio.gather(*coros)

882

883 async def reset_mm_cache(self) -> None:

884 - self.input_processor.clear_mm_cache()

884 + self.renderer.clear_mm_cache()

885 await self.engine_core.reset_mm_cache_async()

886

887 async def reset_prefix_cache(

vllm/v1/engine/input_processor.py

+11 -15

@@ -33,9 +33,9 @@ from vllm.sampling_params import SamplingParams

33 from vllm.tasks import POOLING_TASKS, SupportedTask

34 from vllm.tokenizers import TokenizerLike

35 from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid

36 +from vllm.utils.jsontree import json_iter_leaves

37 from vllm.utils.torch_utils import set_default_torch_num_threads

38 from vllm.v1.engine import EngineCoreRequest

38 -from vllm.v1.metrics.stats import MultiModalCacheStats

40 logger = init_logger(__name__)

@@ -60,8 +60,6 @@ class InputProcessor:

60 self.generation_config_fields = model_config.try_get_generation_config()

62 self.renderer = renderer or renderer_from_config(vllm_config)

63 - self.mm_registry = mm_registry

64 - self.mm_processor_cache = mm_registry.processor_cache_from_config(vllm_config)

64 self.supports_mm_inputs = mm_registry.supports_multimodal_inputs(model_config)

65 self.mm_encoder_cache_size = 0

@@ -78,7 +76,6 @@ class InputProcessor:

76 vllm_config,

77 renderer=renderer,

78 mm_registry=mm_registry,

81 - mm_processor_cache=self.mm_processor_cache,

79 )

81 @property

@@ -136,7 +133,7 @@ class InputProcessor:

133 )

134

135 def _parse_mm_items(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:

139 - mm_processor = self.input_preprocessor._get_mm_processor()

136 + mm_processor = self.renderer.get_mm_processor()

137 return mm_processor.info.parse_mm_data(mm_data)

138

139 def _validate_singleton_mm_uuids(self, prompt: SingletonPrompt) -> None:

@@ -415,6 +412,15 @@ class InputProcessor:

412 decoder_mm_positions = decoder_inputs["mm_placeholders"]

413 decoder_mm_hashes = decoder_inputs["mm_hashes"]

414

415 + if not all(

416 + isinstance(leaf, str) for leaf in json_iter_leaves(decoder_mm_hashes)

417 + ):

418 + raise ValueError(

419 + f"mm_hashes must contain only strings, got: {decoder_mm_hashes}. "

420 + "This is likely due to an incorrect custom implementation of "

421 + "MultiModalProcessor.apply method."

422 + )

423 +

424 # Merge and flatten multimodal placeholders, hashes and inputs

425 # from dictionaries to lists, and sort them by each item's position

426 # in the input sequence.

@@ -562,13 +568,3 @@ class InputProcessor:

568 self._validate_model_input(encoder_inputs, prompt_type="encoder")

569

570 self._validate_model_input(decoder_inputs, prompt_type="decoder")

565 -

566 - def stat_mm_cache(self) -> MultiModalCacheStats | None:

567 - return self.input_preprocessor.stat_mm_cache()

568 -

569 - def clear_mm_cache(self) -> None:

570 - self.input_preprocessor.clear_mm_cache()

571 -

572 - def close(self) -> None:

573 - if self.mm_processor_cache is not None:

574 - self.mm_processor_cache.close()

vllm/v1/engine/llm_engine.py

+2 -2

@@ -320,7 +320,7 @@ class LLMEngine:

320 self.logger_manager.record(

321 scheduler_stats=outputs.scheduler_stats,

322 iteration_stats=iteration_stats,

323 - mm_cache_stats=self.input_processor.stat_mm_cache(),

323 + mm_cache_stats=self.renderer.stat_mm_cache(),

324 )

325 self.do_log_stats_with_interval()

326

@@ -333,7 +333,7 @@ class LLMEngine:

333 self.engine_core.profile(False)

334

335 def reset_mm_cache(self):

336 - self.input_processor.clear_mm_cache()

336 + self.renderer.clear_mm_cache()

337 self.engine_core.reset_mm_cache()

338

339 def reset_prefix_cache(

vllm/v1/metrics/stats.py

@@ -151,6 +151,12 @@ class MultiModalCacheStats(BaseCacheStats):

151 that were queried.

152 """

153

154 + def record(self, num_queries: int, num_hits: int) -> None:

155 + """Aggregate request information into the stats."""

156 + self.requests += 1

157 + self.queries += num_queries

158 + self.hits += num_hits

159 +

160

161 @dataclass

162 class KVCacheEvictionEvent: