@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 27f4c2fd46

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 21:15 UTC

[Renderer] Separate out `RendererConfig` from `ModelConfig` (#30145)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 7, 2025 at 15:15 UTC 27f4c2fd46b99778d7ea19dfe7751fbaab615177

105 files changed +971 -799

docs/contributing/model/transcription.md

+6 -6

@@ -22,7 +22,7 @@ Declare supported languages and capabilities:

22 import torch

23 from torch import nn

25 - from vllm.config import ModelConfig, SpeechToTextConfig

25 + from vllm.config import RendererConfig, SpeechToTextConfig

26 from vllm.inputs.data import PromptType

27 from vllm.model_executor.models.interfaces import SupportsTranscription

@@ -52,7 +52,7 @@ This is for controlling general behavior of the API when serving your model:

52 @classmethod

53 def get_speech_to_text_config(

54 cls,

55 - model_config: ModelConfig,

55 + renderer_config: RendererConfig,

56 task_type: Literal["transcribe", "translate"],

57 ) -> SpeechToTextConfig:

58 return SpeechToTextConfig(

@@ -83,7 +83,7 @@ Return a dict containing `multi_modal_data` with the audio, and either a `prompt

83 cls,

84 audio: np.ndarray,

85 stt_config: SpeechToTextConfig,

86 - model_config: ModelConfig,

86 + renderer_config: RendererConfig,

87 language: str | None,

88 task_type: Literal["transcribe", "translate"],

89 request_prompt: str,

@@ -120,7 +120,7 @@ Return a dict with separate `encoder_prompt` and `decoder_prompt` entries:

120 cls,

121 audio: np.ndarray,

122 stt_config: SpeechToTextConfig,

123 - model_config: ModelConfig,

123 + renderer_config: RendererConfig,

124 language: str | None,

125 task_type: Literal["transcribe", "translate"],

126 request_prompt: str,

@@ -183,7 +183,7 @@ Provide a fast duration→token estimate to improve streaming usage statistics:

183 cls,

184 audio_duration_s: float,

185 stt_config: SpeechToTextConfig,

186 - model_config: ModelConfig,

186 + renderer_config: RendererConfig,

187 ) -> int | None:

188 # Return None if unknown; otherwise return an estimate.

189 return int(audio_duration_s * stt_config.sample_rate // 320) # example

@@ -216,7 +216,7 @@ Relevant server logic:

216 prompt = self.model_cls.get_generation_prompt(

217 audio=chunk,

218 stt_config=self.asr_config,

219 - model_config=self.model_config,

219 + renderer_config=self.renderer_config,

220 language=language,

221 task_type=self.task_type,

222 request_prompt=request.prompt,

tests/compile/distributed/test_sequence_parallelism.py

@@ -17,6 +17,7 @@ from vllm.config import (

17 DeviceConfig,

18 ModelConfig,

19 PassConfig,

20 + RendererConfig,

21 VllmConfig,

22 get_current_vllm_config,

23 set_current_vllm_config,

@@ -276,6 +277,7 @@ def sequence_parallelism_pass_on_test_model(

277

278 vllm_config = VllmConfig(

279 model_config=model_config,

280 + renderer_config=RendererConfig(model_config=model_config),

281 device_config=device_config,

282 compilation_config=compilation_config,

283 )

tests/compile/test_functionalization.py

+5 -1

@@ -15,6 +15,7 @@ from vllm.config import (

15 CompilationConfig,

16 ModelConfig,

17 PassConfig,

18 + RendererConfig,

19 VllmConfig,

20 set_current_vllm_config,

21 )

@@ -219,8 +220,11 @@ def test_fix_functionalization(

220 torch.set_default_device("cuda")

221 torch.set_default_dtype(dtype)

222

223 + model_config = ModelConfig(dtype=dtype)

224 +

225 vllm_config = VllmConfig(

223 - model_config=ModelConfig(dtype=dtype),

226 + model_config=model_config,

227 + renderer_config=RendererConfig(model_config=model_config),

228 compilation_config=CompilationConfig(

229 custom_ops=["all"],

230 pass_config=PassConfig(

tests/compile/test_fusion.py

+5 -1

@@ -15,6 +15,7 @@ from vllm.config import (

15 CompilationMode,

16 ModelConfig,

17 PassConfig,

18 + RendererConfig,

19 VllmConfig,

20 )

21 from vllm.model_executor.layers.layernorm import RMSNorm

@@ -154,8 +155,11 @@ def test_fusion_rmsnorm_quant(

155 custom_ops.append("+rms_norm")

156 if enable_quant_fp8_custom_op:

157 custom_ops.append("+quant_fp8")

158 +

159 + model_config = ModelConfig(dtype=dtype)

160 vllm_config = VllmConfig(

158 - model_config=ModelConfig(dtype=dtype),

161 + model_config=model_config,

162 + renderer_config=RendererConfig(model_config=model_config),

163 compilation_config=CompilationConfig(

164 mode=CompilationMode.VLLM_COMPILE,

165 custom_ops=custom_ops,

tests/compile/test_fusion_attn.py

@@ -24,6 +24,7 @@ from vllm.config import (

24 CompilationMode,

25 ModelConfig,

26 PassConfig,

27 + RendererConfig,

28 SchedulerConfig,

29 VllmConfig,

30 set_current_vllm_config,

@@ -325,6 +326,7 @@ def test_attention_quant_pattern(

326 )

327 vllm_config = VllmConfig(

328 model_config=model_config,

329 + renderer_config=RendererConfig(model_config=model_config),

330 scheduler_config=SchedulerConfig(

331 max_num_seqs=1024,

332 max_model_len=model_config.max_model_len,

tests/compile/test_pass_manager.py

+6 -2

@@ -7,7 +7,7 @@ import torch

8 from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass

9 from vllm.compilation.pass_manager import PostGradPassManager

10 -from vllm.config import ModelConfig, VllmConfig

10 +from vllm.config import ModelConfig, RendererConfig, VllmConfig

13 # dummy custom pass that doesn't inherit

@@ -43,7 +43,11 @@ class ProperPass(InductorPass):

43 )

44 def test_pass_manager_uuid(callable):

45 # Some passes need dtype to be set

46 - config = VllmConfig(model_config=ModelConfig(dtype=torch.bfloat16))

46 + model_config = ModelConfig(dtype=torch.bfloat16)

47 + config = VllmConfig(

48 + model_config=model_config,

49 + renderer_config=RendererConfig(model_config=model_config),

50 + )

52 pass_manager = PostGradPassManager()

53 pass_manager.configure(config)

tests/compile/test_qk_norm_rope_fusion.py

+4 -1

@@ -19,6 +19,7 @@ from vllm.config import (

19 CompilationMode,

20 ModelConfig,

21 PassConfig,

22 + RendererConfig,

23 VllmConfig,

24 set_current_vllm_config,

25 )

@@ -133,8 +134,10 @@ def test_qk_norm_rope_fusion(

134 if enable_rope_custom_op:

135 custom_ops.append("+rotary_embedding")

136

137 + model_config = ModelConfig(dtype=dtype)

138 vllm_config = VllmConfig(

137 - model_config=ModelConfig(dtype=dtype),

139 + model_config=model_config,

140 + renderer_config=RendererConfig(model_config=model_config),

141 compilation_config=CompilationConfig(

142 mode=CompilationMode.VLLM_COMPILE,

143 custom_ops=custom_ops,

tests/distributed/test_kvlayout.py

@@ -5,6 +5,7 @@ from vllm.config import (

5 DeviceConfig,

6 KVTransferConfig,

7 ModelConfig,

8 + RendererConfig,

9 VllmConfig,

10 set_current_vllm_config,

11 )

@@ -47,6 +48,7 @@ def test_get_kv_connector_cache_layout_with_nixl_connector():

48 vllm_config = VllmConfig(

49 device_config=DeviceConfig("cpu"),

50 model_config=model_config,

51 + renderer_config=RendererConfig(model_config=model_config),

52 kv_transfer_config=kv_transfer_config,

53 )

54 with set_current_vllm_config(vllm_config):

@@ -70,6 +72,7 @@ def test_get_kv_connector_cache_layout_with_multi_connector():

72 vllm_config = VllmConfig(

73 device_config=DeviceConfig("cpu"),

74 model_config=model_config,

75 + renderer_config=RendererConfig(model_config=model_config),

76 kv_transfer_config=kv_transfer_config,

77 )

78 with set_current_vllm_config(vllm_config):

tests/entrypoints/openai/test_chat_template.py

+4 -18

@@ -3,7 +3,6 @@

4 import pytest

6 -from vllm.config import ModelConfig

6 from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template

7 from vllm.entrypoints.openai.protocol import ChatCompletionRequest

8 from vllm.tokenizers import get_tokenizer

@@ -107,24 +106,11 @@ def test_get_gen_prompt(

106 model_info = HF_EXAMPLE_MODELS.find_hf_info(model)

107 model_info.check_available_online(on_fail="skip")

108

110 - model_config = ModelConfig(

111 - model,

112 - tokenizer=model_info.tokenizer or model,

113 - tokenizer_mode=model_info.tokenizer_mode,

114 - trust_remote_code=model_info.trust_remote_code,

115 - revision=model_info.revision,

116 - hf_overrides=model_info.hf_overrides,

117 - skip_tokenizer_init=model_info.require_embed_inputs,

118 - enable_prompt_embeds=model_info.require_embed_inputs,

119 - enable_mm_embeds=model_info.require_embed_inputs,

120 - enforce_eager=model_info.enforce_eager,

121 - dtype=model_info.dtype,

122 - )

109 + renderer_config = model_info.build_renderer_config(model)

110

124 - # Initialize the tokenizer

111 tokenizer = get_tokenizer(

126 - tokenizer_name=model_config.tokenizer,

127 - trust_remote_code=model_config.trust_remote_code,

112 + renderer_config.tokenizer,

113 + trust_remote_code=renderer_config.trust_remote_code,

114 )

115 template_content = load_chat_template(chat_template=template)

116

@@ -143,7 +129,7 @@ def test_get_gen_prompt(

129 tokenizer=tokenizer,

130 conversation=mock_request.messages,

131 chat_template=mock_request.chat_template or template_content,

146 - model_config=model_config,

132 + renderer_config=renderer_config,

133 tools=None,

134 add_generation_prompt=mock_request.add_generation_prompt,

135 continue_final_message=mock_request.continue_final_message,

tests/entrypoints/openai/test_lora_resolvers.py

+15 -6

@@ -33,26 +33,34 @@ class MockModelConfig:

33 """Minimal mock ModelConfig for testing."""

35 model: str = MODEL_NAME

36 - tokenizer: str = MODEL_NAME

36 trust_remote_code: bool = False

38 - tokenizer_mode: str = "auto"

37 max_model_len: int = 100

40 - tokenizer_revision: str | None = None

38 multimodal_config: MultiModalConfig = field(default_factory=MultiModalConfig)

39 hf_config: MockHFConfig = field(default_factory=MockHFConfig)

40 logits_processors: list[str] | None = None

41 logits_processor_pattern: str | None = None

42 diff_sampling_param: dict | None = None

46 - allowed_local_media_path: str = ""

47 - allowed_media_domains: list[str] | None = None

43 encoder_config = None

44 generation_config: str = "auto"

50 - skip_tokenizer_init: bool = False

46 def get_diff_sampling_param(self):

47 return self.diff_sampling_param or {}

50 +@dataclass

51 +class MockRendererConfig:

52 + """Minimal mock RendererConfig for testing."""

53 +

54 + model_config: MockModelConfig

55 +

56 + tokenizer: str = MODEL_NAME

57 + tokenizer_mode: str = "auto"

58 + tokenizer_revision: str | None = None

59 + skip_tokenizer_init: bool = False

60 + allowed_local_media_path: str = ""

61 + allowed_media_domains: list[str] | None = None

62 +

63 +

64 class MockLoRAResolver(LoRAResolver):

65 async def resolve_lora(

66 self, base_model_name: str, lora_name: str

@@ -114,6 +122,7 @@ def mock_serving_setup():

122 mock_engine.add_lora.reset_mock()

123

124 mock_engine.model_config = MockModelConfig()

125 + mock_engine.renderer_config = MockRendererConfig(mock_engine.model_config)

126 mock_engine.input_processor = MagicMock()

127 mock_engine.io_processor = MagicMock()

128

tests/entrypoints/openai/test_serving_chat.py

+21 -7

@@ -346,27 +346,33 @@ class MockHFConfig:

346 class MockModelConfig:

347 task = "generate"

348 runner_type = "generate"

349 - tokenizer = MODEL_NAME

349 trust_remote_code = False

351 - tokenizer_mode = "auto"

350 max_model_len = 100

353 - tokenizer_revision = None

351 multimodal_config = MultiModalConfig()

352 hf_config = MockHFConfig()

353 logits_processors: list[str] | None = None

354 logits_processor_pattern = None

355 diff_sampling_param: dict | None = None

359 - allowed_local_media_path: str = ""

360 - allowed_media_domains: list[str] | None = None

356 encoder_config = None

357 generation_config: str = "auto"

363 - media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

364 - skip_tokenizer_init = False

358

359 def get_diff_sampling_param(self):

360 return self.diff_sampling_param or {}

361

362

363 +@dataclass

364 +class MockRendererConfig:

365 + model_config: MockModelConfig = field(default_factory=MockModelConfig)

366 +

367 + tokenizer = MODEL_NAME

368 + tokenizer_mode = "auto"

369 + tokenizer_revision = None

370 + skip_tokenizer_init = False

371 + media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

372 + allowed_local_media_path: str = ""

373 + allowed_media_domains: list[str] | None = None

374 +

375 +

376 def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:

377 models = OpenAIServingModels(

378 engine_client=engine,

@@ -399,6 +405,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:

405 @dataclass

406 class MockEngine:

407 model_config: MockModelConfig = field(default_factory=MockModelConfig)

408 + renderer_config: MockRendererConfig = field(default_factory=MockRendererConfig)

409 input_processor: MagicMock = field(default_factory=MagicMock)

410 io_processor: MagicMock = field(default_factory=MagicMock)

411

@@ -429,6 +436,7 @@ async def test_serving_chat_returns_correct_model_name():

436 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

437 mock_engine.errored = False

438 mock_engine.model_config = MockModelConfig()

439 + mock_engine.renderer_config = MockRendererConfig(mock_engine.model_config)

440 mock_engine.input_processor = MagicMock()

441 mock_engine.io_processor = MagicMock()

442

@@ -459,6 +467,7 @@ async def test_serving_chat_should_set_correct_max_tokens():

467 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

468 mock_engine.errored = False

469 mock_engine.model_config = MockModelConfig()

470 + mock_engine.renderer_config = MockRendererConfig(mock_engine.model_config)

471 mock_engine.input_processor = MagicMock()

472 mock_engine.io_processor = MagicMock()

473

@@ -492,6 +501,7 @@ async def test_serving_chat_should_set_correct_max_tokens():

501 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

502 mock_engine.errored = False

503 mock_engine.model_config = mock_model_config

504 + mock_engine.renderer_config = MockRendererConfig(mock_model_config)

505 mock_engine.input_processor = MagicMock()

506 mock_engine.io_processor = MagicMock()

507

@@ -537,6 +547,7 @@ async def test_serving_chat_should_set_correct_max_tokens():

547 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

548 mock_engine.errored = False

549 mock_engine.model_config = mock_model_config

550 + mock_engine.renderer_config = MockRendererConfig(mock_model_config)

551 mock_engine.input_processor = MagicMock()

552 mock_engine.io_processor = MagicMock()

553

@@ -583,6 +594,7 @@ async def test_serving_chat_could_load_correct_generation_config():

594 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

595 mock_engine.errored = False

596 mock_engine.model_config = mock_model_config

597 + mock_engine.renderer_config = MockRendererConfig(mock_model_config)

598 mock_engine.input_processor = MagicMock()

599 mock_engine.io_processor = MagicMock()

600

@@ -629,6 +641,7 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):

641 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

642 mock_engine.errored = False

643 mock_engine.model_config = mock_model_config

644 + mock_engine.renderer_config = MockRendererConfig(mock_model_config)

645 mock_engine.input_processor = MagicMock()

646 mock_engine.io_processor = MagicMock()

647

@@ -662,6 +675,7 @@ async def test_serving_chat_data_parallel_rank_extraction():

675 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

676 mock_engine.errored = False

677 mock_engine.model_config = MockModelConfig()

678 + mock_engine.renderer_config = MockRendererConfig(mock_engine.model_config)

679 mock_engine.input_processor = MagicMock()

680 mock_engine.io_processor = MagicMock()

681

tests/entrypoints/openai/test_serving_engine.py

+7 -1

@@ -7,7 +7,7 @@ from unittest.mock import Mock

8 import pytest

10 -from vllm.config import ModelConfig

10 +from vllm.config import ModelConfig, RendererConfig

11 from vllm.entrypoints.openai.serving_engine import OpenAIServing

12 from vllm.entrypoints.openai.serving_models import OpenAIServingModels

13 from vllm.tokenizers import MistralTokenizer

@@ -19,10 +19,16 @@ def serving() -> OpenAIServing:

20 # Create minimal mocks

21 engine_client = Mock()

22 +

23 model_config = Mock(spec=ModelConfig)

24 model_config.max_model_len = 32768

25 +

26 + renderer_config = Mock(spec=RendererConfig)

27 + renderer_config.model_config = model_config

28 +

29 models = Mock(spec=OpenAIServingModels)

30 models.model_config = model_config

31 + models.renderer_config = renderer_config

32 models.input_processor = Mock()

33 models.io_processor = Mock()

tests/entrypoints/openai/test_serving_models.py

+7 -1

@@ -6,7 +6,7 @@ from unittest.mock import MagicMock

7 import pytest

9 -from vllm.config import ModelConfig

9 +from vllm.config import ModelConfig, RendererConfig

10 from vllm.engine.protocol import EngineClient

11 from vllm.entrypoints.openai.protocol import (

12 ErrorResponse,

@@ -27,9 +27,15 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (

27 async def _async_serving_models_init() -> OpenAIServingModels:

28 mock_engine_client = MagicMock(spec=EngineClient)

29 # Set the max_model_len attribute to avoid missing attribute

30 +

31 mock_model_config = MagicMock(spec=ModelConfig)

32 mock_model_config.max_model_len = 2048

33 +

34 + mock_renderer_config = MagicMock(spec=RendererConfig)

35 + mock_renderer_config.model_config = mock_model_config

36 +

37 mock_engine_client.model_config = mock_model_config

38 + mock_engine_client.renderer_config = mock_renderer_config

39 mock_engine_client.input_processor = MagicMock()

40 mock_engine_client.io_processor = MagicMock()

tests/entrypoints/test_chat_utils.py

+71 -123

@@ -12,7 +12,7 @@ from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy

12 from vllm.assets.audio import AudioAsset

13 from vllm.assets.image import ImageAsset

14 from vllm.assets.video import VideoAsset

15 -from vllm.config import ModelConfig

15 +from vllm.config import ModelConfig, RendererConfig

16 from vllm.entrypoints.chat_utils import (

17 _try_extract_ast,

18 apply_mistral_chat_template,

@@ -233,7 +233,7 @@ def test_parse_chat_messages_single_image(

233 ],

234 }

235 ],

236 - phi3v_model_config,

236 + RendererConfig(model_config=phi3v_model_config),

237 content_format="string",

238 )

239

@@ -265,7 +265,7 @@ def test_parse_chat_messages_single_image_with_uuid(

265 ],

266 }

267 ],

268 - phi3v_model_config,

268 + RendererConfig(model_config=phi3v_model_config),

269 content_format="string",

270 )

271

@@ -295,7 +295,7 @@ def test_parse_chat_messages_single_empty_image_with_uuid(

295 ],

296 }

297 ],

298 - phi3v_model_config,

298 + RendererConfig(model_config=phi3v_model_config),

299 content_format="string",

300 )

301

@@ -328,7 +328,7 @@ def test_parse_chat_messages_single_image_with_bad_uuid_format(

328 ],

329 }

330 ],

331 - phi3v_model_config,

331 + RendererConfig(model_config=phi3v_model_config),

332 content_format="string",

333 )

334

@@ -369,7 +369,7 @@ def test_parse_chat_messages_multiple_images_with_uuids(

369 ],

370 }

371 ],

372 - phi3v_model_config,

372 + RendererConfig(model_config=phi3v_model_config),

373 content_format="string",

374 )

375

@@ -409,7 +409,7 @@ def test_parse_chat_messages_multiple_empty_images_with_uuids(

409 ],

410 }

411 ],

412 - phi3v_model_config,

412 + RendererConfig(model_config=phi3v_model_config),

413 content_format="string",

414 )

415

@@ -451,7 +451,7 @@ def test_parse_chat_messages_mixed_empty_images_with_uuids(

451 ],

452 }

453 ],

454 - phi3v_model_config,

454 + RendererConfig(model_config=phi3v_model_config),

455 content_format="string",

456 )

457

@@ -485,7 +485,7 @@ async def test_parse_chat_messages_single_image_with_uuid_async(

485 ],

486 }

487 ],

488 - phi3v_model_config,

488 + RendererConfig(model_config=phi3v_model_config),

489 content_format="string",

490 )

491

@@ -516,7 +516,7 @@ async def test_parse_chat_messages_empty_image_with_uuid_async(

516 ],

517 }

518 ],

519 - phi3v_model_config,

519 + RendererConfig(model_config=phi3v_model_config),

520 content_format="string",

521 )

522

@@ -554,7 +554,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async(

554 ],

555 }

556 ],

557 - phi3v_model_config,

557 + RendererConfig(model_config=phi3v_model_config),

558 content_format="string",

559 )

560

@@ -595,7 +595,7 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(

595 ],

596 }

597 ],

598 - phi3v_model_config,

598 + RendererConfig(model_config=phi3v_model_config),

599 content_format="string",

600 )

601

@@ -634,7 +634,7 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(

634 ],

635 }

636 ],

637 - phi3v_model_config,

637 + RendererConfig(model_config=phi3v_model_config),

638 content_format="string",

639 )

640

@@ -660,7 +660,7 @@ def test_parse_chat_messages_empty_system(

660 "content": [{"type": "text", "text": "Who are you?"}],

661 },

662 ],

663 - mistral_model_config,

663 + RendererConfig(model_config=mistral_model_config),

664 content_format="string",

665 )

666 assert conversation == [

@@ -677,7 +677,7 @@ def test_parse_chat_messages_empty_system(

677 "content": [{"type": "text", "text": "Who are you?"}],

678 },

679 ],

680 - mistral_model_config,

680 + RendererConfig(model_config=mistral_model_config),

681 content_format="openai",

682 )

683 assert conversation == [

@@ -701,7 +701,7 @@ async def test_parse_chat_messages_single_image_async(

701 ],

702 }

703 ],

704 - phi3v_model_config,

704 + RendererConfig(model_config=phi3v_model_config),

705 content_format="string",

706 )

707

@@ -730,7 +730,7 @@ def test_parse_chat_messages_multiple_images(

730 ],

731 }

732 ],

733 - phi3v_model_config,

733 + RendererConfig(model_config=phi3v_model_config),

734 content_format="string",

735 )

736

@@ -758,7 +758,7 @@ def test_parse_chat_messages_empty_pil_image_with_uuid(

758 ],

759 }

760 ],

761 - phi3v_model_config,

761 + RendererConfig(model_config=phi3v_model_config),

762 content_format="string",

763 )

764

@@ -786,7 +786,7 @@ def test_parse_chat_messages_empty_image_embeds_with_uuid(

786 ],

787 }

788 ],

789 - phi3v_model_config_image_embeds,

789 + RendererConfig(model_config=phi3v_model_config_image_embeds),

790 content_format="string",

791 )

792

@@ -818,7 +818,7 @@ def test_parse_chat_messages_empty_audio_embeds_with_uuid(

818 ],

819 }

820 ],

821 - audio_embeds_model_config,

821 + RendererConfig(model_config=audio_embeds_model_config),

822 content_format="string",

823 )

824

@@ -858,7 +858,7 @@ def test_parse_chat_messages_audio_embeds_with_string(

858 ],

859 }

860 ],

861 - audio_embeds_model_config,

861 + RendererConfig(model_config=audio_embeds_model_config),

862 content_format="string",

863 )

864

@@ -900,7 +900,7 @@ async def test_parse_chat_messages_audio_embeds_async(

900 ],

901 }

902 ],

903 - audio_embeds_model_config,

903 + RendererConfig(model_config=audio_embeds_model_config),

904 content_format="string",

905 )

906

@@ -1108,7 +1108,7 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(

1108 ],

1109 }

1110 ],

1111 - phi3v_model_config_image_embeds,

1111 + RendererConfig(model_config=phi3v_model_config_image_embeds),

1112 content_format="string",

1113 )

1114

@@ -1144,7 +1144,7 @@ async def test_parse_chat_messages_multiple_images_async(

1144 ],

1145 }

1146 ],

1147 - phi3v_model_config,

1147 + RendererConfig(model_config=phi3v_model_config),

1148 content_format="string",

1149 )

1150

@@ -1176,7 +1176,7 @@ def test_parse_chat_messages_placeholder_already_in_prompt(

1176 ],

1177 }

1178 ],

1179 - phi3v_model_config,

1179 + RendererConfig(model_config=phi3v_model_config),

1180 content_format="string",

1181 )

1182 assert conversation == [

@@ -1208,7 +1208,7 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(

1208 ],

1209 }

1210 ],

1211 - phi3v_model_config,

1211 + RendererConfig(model_config=phi3v_model_config),

1212 content_format="string",

1213 )

1214

@@ -1245,7 +1245,7 @@ def test_parse_chat_messages_multiple_images_across_messages(

1245 ],

1246 },

1247 ],

1248 - phi3v_model_config,

1248 + RendererConfig(model_config=phi3v_model_config),

1249 content_format="string",

1250 )

1251

@@ -1289,7 +1289,7 @@ def test_parse_chat_messages_multiple_images_with_uuids_across_messages(

1289 ],

1290 },

1291 ],

1292 - phi3v_model_config,

1292 + RendererConfig(model_config=phi3v_model_config),

1293 content_format="string",

1294 )

1295

@@ -1314,7 +1314,7 @@ def test_parse_chat_messages_context_text_format(

1314 {"role": "assistant", "content": "Some stuff."},

1315 {"role": "user", "content": "What about this one?"},

1316 ],

1317 - phi3v_model_config,

1317 + RendererConfig(model_config=phi3v_model_config),

1318 content_format="openai",

1319 )

1320

@@ -1367,7 +1367,7 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(

1367 ],

1368 }

1369 ],

1370 - phi3v_model_config,

1370 + RendererConfig(model_config=phi3v_model_config),

1371 content_format="string",

1372 )

1373

@@ -1410,7 +1410,7 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(

1410 ],

1411 },

1412 ],

1413 - phi3v_model_config,

1413 + RendererConfig(model_config=phi3v_model_config),

1414 content_format="string",

1415 )

1416

@@ -1430,7 +1430,7 @@ def test_parse_chat_messages_multiple_images_uncommon_input(

1430 ],

1431 }

1432 ],

1433 - phi3v_model_config,

1433 + RendererConfig(model_config=phi3v_model_config),

1434 content_format="string",

1435 )

1436

@@ -1464,7 +1464,7 @@ def test_parse_chat_messages_multiple_images_interleave(

1464 ],

1465 }

1466 ],

1467 - phi3v_model_config_mm_interleaved,

1467 + RendererConfig(model_config=phi3v_model_config_mm_interleaved),

1468 content_format="string",

1469 )

1470

@@ -1500,7 +1500,7 @@ async def test_parse_chat_messages_multiple_images_interleave_async(

1500 ],

1501 }

1502 ],

1503 - phi3v_model_config_mm_interleaved,

1503 + RendererConfig(model_config=phi3v_model_config_mm_interleaved),

1504 content_format="string",

1505 )

1506

@@ -1545,7 +1545,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(

1545 ],

1546 }

1547 ],

1548 - phi3v_model_config_mm_interleaved,

1548 + RendererConfig(model_config=phi3v_model_config_mm_interleaved),

1549 content_format="string",

1550 )

1551

@@ -1583,7 +1583,7 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave(

1583 ],

1584 },

1585 ],

1586 - phi3v_model_config_mm_interleaved,

1586 + RendererConfig(model_config=phi3v_model_config_mm_interleaved),

1587 content_format="string",

1588 )

1589

@@ -1631,7 +1631,7 @@ def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interl

1631 ],

1632 },

1633 ],

1634 - phi3v_model_config_mm_interleaved,

1634 + RendererConfig(model_config=phi3v_model_config_mm_interleaved),

1635 content_format="string",

1636 )

1637

@@ -1675,7 +1675,7 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(

1675 ],

1676 },

1677 ],

1678 - qwen25omni_model_config_mm_interleaved,

1678 + RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),

1679 content_format="string",

1680 )

1681

@@ -1743,7 +1743,7 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interl

1743 ],

1744 },

1745 ],

1746 - qwen25omni_model_config_mm_interleaved,

1746 + RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),

1747 content_format="string",

1748 )

1749

@@ -1813,7 +1813,7 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_mes

1813 ],

1814 },

1815 ],

1816 - qwen25omni_model_config_mm_interleaved,

1816 + RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),

1817 content_format="string",

1818 )

1819

@@ -1879,7 +1879,7 @@ def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_message

1879 ],

1880 },

1881 ],

1882 - qwen25omni_model_config_mm_interleaved,

1882 + RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),

1883 content_format="string",

1884 )

1885

@@ -1927,7 +1927,7 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders(

1927 ],

1928 }

1929 ],

1930 - phi3v_model_config_mm_interleaved,

1930 + RendererConfig(model_config=phi3v_model_config_mm_interleaved),

1931 content_format="string",

1932 )

1933

@@ -1945,24 +1945,11 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):

1945 model_info = HF_EXAMPLE_MODELS.find_hf_info(model)

1946 model_info.check_available_online(on_fail="skip")

1947

1948 - model_config = ModelConfig(

1949 - model,

1950 - tokenizer=model_info.tokenizer or model,

1951 - tokenizer_mode=model_info.tokenizer_mode,

1952 - revision=model_info.revision,

1953 - trust_remote_code=model_info.trust_remote_code,

1954 - hf_overrides=model_info.hf_overrides,

1955 - skip_tokenizer_init=model_info.require_embed_inputs,

1956 - enable_prompt_embeds=model_info.require_embed_inputs,

1957 - enable_mm_embeds=model_info.require_embed_inputs,

1958 - enforce_eager=model_info.enforce_eager,

1959 - dtype=model_info.dtype,

1960 - )

1961 -

1962 - # Build the tokenizer

1948 + renderer_config = model_info.build_renderer_config(model)

1949 +

1950 tokenizer = get_tokenizer(

1964 - model,

1965 - trust_remote_code=model_config.trust_remote_code,

1951 + renderer_config.tokenizer,

1952 + trust_remote_code=renderer_config.trust_remote_code,

1953 )

1954

1955 tools = (

@@ -1985,7 +1972,7 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):

1972 tokenizer,

1973 chat_template=None,

1974 tools=tools,

1988 - model_config=model_config,

1975 + model_config=renderer_config.model_config,

1976 )

1977 assert isinstance(chat_template, str)

1978

@@ -2047,24 +2034,11 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa

2034 "enable_thinking": True,

2035 }

2036

2050 - model_config = ModelConfig(

2051 - model,

2052 - tokenizer=model_info.tokenizer or model,

2053 - tokenizer_mode=model_info.tokenizer_mode,

2054 - revision=model_info.revision,

2055 - trust_remote_code=model_info.trust_remote_code,

2056 - hf_overrides=model_info.hf_overrides,

2057 - skip_tokenizer_init=model_info.require_embed_inputs,

2058 - enable_prompt_embeds=model_info.require_embed_inputs,

2059 - enable_mm_embeds=model_info.require_embed_inputs,

2060 - enforce_eager=model_info.enforce_eager,

2061 - dtype=model_info.dtype,

2062 - )

2063 -

2064 - # Build the tokenizer

2037 + renderer_config = model_info.build_renderer_config(model)

2038 +

2039 tokenizer = get_tokenizer(

2066 - model,

2067 - trust_remote_code=model_config.trust_remote_code,

2040 + renderer_config.tokenizer,

2041 + trust_remote_code=renderer_config.trust_remote_code,

2042 )

2043

2044 # Test detecting the tokenizer's chat_template

@@ -2072,7 +2046,7 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa

2046 tokenizer,

2047 chat_template=None,

2048 tools=tools,

2075 - model_config=model_config,

2049 + model_config=renderer_config.model_config,

2050 )

2051 with pytest.raises(

2052 ValueError, match="Found unexpected chat template kwargs from request"

@@ -2143,23 +2117,11 @@ def test_resolve_content_format_hf_defined(model, expected_format):

2117 model_info = HF_EXAMPLE_MODELS.find_hf_info(model)

2118 model_info.check_available_online(on_fail="skip")

2119

2146 - model_config = ModelConfig(

2147 - model,

2148 - tokenizer=model_info.tokenizer or model,

2149 - tokenizer_mode=model_info.tokenizer_mode,

2150 - revision=model_info.revision,

2151 - trust_remote_code=model_info.trust_remote_code,

2152 - hf_overrides=model_info.hf_overrides,

2153 - skip_tokenizer_init=model_info.require_embed_inputs,

2154 - enable_prompt_embeds=model_info.require_embed_inputs,

2155 - enable_mm_embeds=model_info.require_embed_inputs,

2156 - enforce_eager=model_info.enforce_eager,

2157 - dtype=model_info.dtype,

2158 - )

2120 + renderer_config = model_info.build_renderer_config(model)

2121

2122 tokenizer = get_tokenizer(

2161 - model,

2162 - trust_remote_code=model_config.trust_remote_code,

2123 + renderer_config.tokenizer,

2124 + trust_remote_code=renderer_config.trust_remote_code,

2125 )

2126

2127 # Test detecting the tokenizer's chat_template

@@ -2167,7 +2129,7 @@ def test_resolve_content_format_hf_defined(model, expected_format):

2129 tokenizer,

2130 chat_template=None,

2131 tools=None,

2170 - model_config=model_config,

2132 + model_config=renderer_config.model_config,

2133 )

2134 assert isinstance(chat_template, str)

2135

@@ -2181,7 +2143,7 @@ def test_resolve_content_format_hf_defined(model, expected_format):

2143 None,

2144 "auto",

2145 tokenizer,

2184 - model_config=model_config,

2146 + renderer_config=renderer_config,

2147 )

2148

2149 assert resolved_format == expected_format

@@ -2203,23 +2165,11 @@ def test_resolve_content_format_fallbacks(model, expected_format):

2165 model_info = HF_EXAMPLE_MODELS.find_hf_info(model)

2166 model_info.check_available_online(on_fail="skip")

2167

2206 - model_config = ModelConfig(

2207 - model,

2208 - tokenizer=model_info.tokenizer or model,

2209 - tokenizer_mode=model_info.tokenizer_mode,

2210 - revision=model_info.revision,

2211 - trust_remote_code=model_info.trust_remote_code,

2212 - hf_overrides=model_info.hf_overrides,

2213 - skip_tokenizer_init=model_info.require_embed_inputs,

2214 - enable_prompt_embeds=model_info.require_embed_inputs,

2215 - enable_mm_embeds=model_info.require_embed_inputs,

2216 - enforce_eager=model_info.enforce_eager,

2217 - dtype=model_info.dtype,

2218 - )

2168 + renderer_config = model_info.build_renderer_config(model)

2169

2170 tokenizer = get_tokenizer(

2221 - model_config.tokenizer,

2222 - trust_remote_code=model_config.trust_remote_code,

2171 + renderer_config.tokenizer,

2172 + trust_remote_code=renderer_config.trust_remote_code,

2173 )

2174

2175 # Test detecting the tokenizer's chat_template

@@ -2227,7 +2177,7 @@ def test_resolve_content_format_fallbacks(model, expected_format):

2177 tokenizer,

2178 chat_template=None,

2179 tools=None,

2230 - model_config=model_config,

2180 + model_config=renderer_config.model_config,

2181 )

2182 assert isinstance(chat_template, str)

2183

@@ -2241,7 +2191,7 @@ def test_resolve_content_format_fallbacks(model, expected_format):

2191 None,

2192 "auto",

2193 tokenizer,

2244 - model_config=model_config,

2194 + renderer_config=renderer_config,

2195 )

2196

2197 assert resolved_format == expected_format

@@ -2272,15 +2222,13 @@ def test_resolve_content_format_fallbacks(model, expected_format):

2222 ],

2223 )

2224 def test_resolve_content_format_examples(template_path, expected_format):

2275 - model_config = ModelConfig(

2276 - PHI3V_MODEL_ID, # Dummy

2277 - tokenizer=PHI3V_MODEL_ID, # Dummy

2278 - trust_remote_code=True,

2279 - )

2225 + model = PHI3V_MODEL_ID # Dummy

2226 + model_config = ModelConfig(model, trust_remote_code=True)

2227 + renderer_config = RendererConfig(model_config=model_config, tokenizer=model)

2228

2229 dummy_tokenizer = get_tokenizer(

2282 - PHI3V_MODEL_ID, # Dummy

2283 - trust_remote_code=model_config.trust_remote_code,

2230 + renderer_config.tokenizer,

2231 + trust_remote_code=renderer_config.trust_remote_code,

2232 )

2233 dummy_tokenizer.chat_template = None

2234

@@ -2297,7 +2245,7 @@ def test_resolve_content_format_examples(template_path, expected_format):

2245 None,

2246 "auto",

2247 dummy_tokenizer,

2300 - model_config=model_config,

2248 + renderer_config=renderer_config,

2249 )

2250

2251 assert resolved_format == expected_format

@@ -2332,7 +2280,7 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):

2280

2281 conversation_with_thinking, _, _ = parse_chat_messages(

2282 messages,

2335 - mistral_model_config,

2283 + RendererConfig(model_config=mistral_model_config),

2284 content_format="openai",

2285 )

2286

@@ -2432,7 +2380,7 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(

2380 ],

2381 }

2382 ],

2435 - qwen2_audio_model_config,

2383 + RendererConfig(model_config=qwen2_audio_model_config),

2384 content_format="string",

2385 )

2386

@@ -2466,7 +2414,7 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(

2414 ],

2415 }

2416 ],

2469 - qwen2_audio_model_config,

2417 + RendererConfig(model_config=qwen2_audio_model_config),

2418 content_format="string",

2419 )

2420

tests/lora/test_lora_manager.py

+11 -3

@@ -8,7 +8,7 @@ import torch

8 from safetensors.torch import load_file

9 from torch import nn

11 -from vllm.config import ModelConfig, VllmConfig

11 +from vllm.config import ModelConfig, RendererConfig, VllmConfig

12 from vllm.config.lora import LoRAConfig

13 from vllm.lora.layers import (

14 ColumnParallelLinearWithLoRA,

@@ -422,7 +422,11 @@ def test_lru_cache_worker_adapter_manager(dist_init, dummy_model, device, tmp_pa

422 )

423

424 model_config = ModelConfig(max_model_len=16)

425 - vllm_config = VllmConfig(model_config=model_config, lora_config=lora_config)

425 + vllm_config = VllmConfig(

426 + model_config=model_config,

427 + renderer_config=RendererConfig(model_config=model_config),

428 + lora_config=lora_config,

429 + )

430

431 vllm_config.scheduler_config.max_num_seqs = 4

432 vllm_config.scheduler_config.max_num_batched_tokens = 2

@@ -525,7 +529,11 @@ def test_worker_adapter_manager(dist_init, dummy_model_gate_up, device, tmp_path

529 )

530

531 model_config = ModelConfig(max_model_len=16)

528 - vllm_config = VllmConfig(model_config=model_config, lora_config=lora_config)

532 + vllm_config = VllmConfig(

533 + model_config=model_config,

534 + renderer_config=RendererConfig(model_config=model_config),

535 + lora_config=lora_config,

536 + )

537

538 vllm_config.scheduler_config.max_num_seqs = 4

539 vllm_config.scheduler_config.max_num_batched_tokens = 2

tests/lora/test_worker.py

@@ -11,6 +11,7 @@ from vllm.config import (

11 DeviceConfig,

12 ModelConfig,

13 ParallelConfig,

14 + RendererConfig,

15 SchedulerConfig,

16 VllmConfig,

17 )

@@ -43,6 +44,7 @@ def test_worker_apply_lora(qwen3_lora_files):

45 vllm_config = VllmConfig(

46 model_config=model_config,

47 + renderer_config=RendererConfig(model_config=model_config),

48 load_config=LoadConfig(

49 download_dir=None,

50 load_format="dummy",

tests/model_executor/test_model_load_with_params.py

+13 -9

@@ -42,8 +42,10 @@ def test_model_loading_with_params(vllm_runner, monkeypatch):

42 "Write a short story about a robot that dreams for the first time.\n"

43 )

45 - model_config = vllm_model.llm.llm_engine.model_config

46 - model_tokenizer = vllm_model.llm.llm_engine.tokenizer

45 + llm_engine = vllm_model.llm.llm_engine

46 + model_config = llm_engine.model_config

47 + renderer_config = llm_engine.renderer_config

48 + tokenizer = llm_engine.tokenizer

50 # asserts on the bert model config file

51 assert model_config.encoder_config["max_seq_length"] == 512

@@ -54,8 +56,8 @@ def test_model_loading_with_params(vllm_runner, monkeypatch):

56 assert model_config.pooler_config.normalize

58 # asserts on the tokenizer loaded

57 - assert model_config.tokenizer == "BAAI/bge-base-en-v1.5"

58 - assert model_tokenizer.model_max_length == 512

59 + assert renderer_config.tokenizer == "BAAI/bge-base-en-v1.5"

60 + assert tokenizer.model_max_length == 512

62 def check_model(model):

63 assert isinstance(model, BertEmbeddingModel)

@@ -86,8 +88,10 @@ def test_roberta_model_loading_with_params(vllm_runner, monkeypatch):

88 "Write a short story about a robot that dreams for the first time.\n"

89 )

89 - model_config = vllm_model.llm.llm_engine.model_config

90 - model_tokenizer = vllm_model.llm.llm_engine.tokenizer

91 + llm_engine = vllm_model.llm.llm_engine

92 + model_config = llm_engine.model_config

93 + renderer_config = llm_engine.renderer_config

94 + tokenizer = llm_engine.tokenizer

96 # asserts on the bert model config file

97 assert model_config.encoder_config["max_seq_length"] == 512

@@ -98,8 +102,8 @@ def test_roberta_model_loading_with_params(vllm_runner, monkeypatch):

102 assert model_config.pooler_config.normalize

103

104 # asserts on the tokenizer loaded

101 - assert model_config.tokenizer == "intfloat/multilingual-e5-base"

102 - assert model_tokenizer.model_max_length == 512

105 + assert renderer_config.tokenizer == "intfloat/multilingual-e5-base"

106 + assert tokenizer.model_max_length == 512

107

108 def check_model(model):

109 assert isinstance(model, RobertaEmbeddingModel)

@@ -128,7 +132,7 @@ def test_facebook_roberta_model_loading_with_params(vllm_runner, monkeypatch):

132 "Write a short story about a robot that dreams for the first time.\n"

133 )

134

131 - assert vllm_model.llm.llm_engine.model_config.tokenizer == model_name

135 + assert vllm_model.llm.llm_engine.renderer_config.tokenizer == model_name

136

137 def check_model(model):

138 assert isinstance(model, RobertaEmbeddingModel)

tests/models/language/pooling/test_gritlm.py

+3 -2

@@ -6,7 +6,7 @@ import pytest

6 from scipy.spatial.distance import cosine

8 from vllm import LLM, SamplingParams

9 -from vllm.config import ModelConfig

9 +from vllm.config import ModelConfig, RendererConfig

11 from ....utils import RemoteOpenAIServer

@@ -31,7 +31,8 @@ def test_find_array():

31 dtype="bfloat16",

32 seed=0,

33 )

34 - pooling = GritLMMeanPool(model_config=model_config)

34 + renderer_config = RendererConfig(model_config=model_config)

35 + pooling = GritLMMeanPool(renderer_config=renderer_config)

37 arr = _arr([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])

tests/models/multimodal/processing/test_common.py

+5 -17

@@ -25,7 +25,6 @@ from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingC

25 from vllm.tokenizers import (

26 MistralTokenizer,

27 TokenizerLike,

28 - cached_tokenizer_from_config,

28 )

30 from ....multimodal.utils import random_audio, random_image, random_video

@@ -212,31 +211,20 @@ def _test_processing_correctness(

211 else:

212 model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch)

213 model_id = model_id_or_arch

214 +

215 model_info.check_available_online(on_fail="skip")

216 model_info.check_transformers_version(on_fail="skip")

217

218 - model_config = ModelConfig(

219 - model_id,

220 - tokenizer=model_info.tokenizer or model_id,

221 - tokenizer_mode=model_info.tokenizer_mode,

222 - revision=model_info.revision,

223 - trust_remote_code=model_info.trust_remote_code,

224 - hf_overrides=model_info.hf_overrides,

218 + renderer_config = model_info.build_renderer_config(

219 + model=model_id,

220 # Ensure that the cache can fit all of the data

221 mm_processor_cache_gb=2048,

227 - skip_tokenizer_init=model_info.require_embed_inputs,

228 - enable_prompt_embeds=model_info.require_embed_inputs,

229 - enable_mm_embeds=model_info.require_embed_inputs,

230 - enforce_eager=model_info.enforce_eager,

231 - dtype=model_info.dtype,

222 )

223 + model_config = renderer_config.model_config

224

225 model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)

226 factories = model_cls._processor_factory

236 - ctx = InputProcessingContext(

237 - model_config,

238 - tokenizer=cached_tokenizer_from_config(model_config),

239 - )

227 + ctx = InputProcessingContext.from_config(renderer_config)

228 cache = MultiModalProcessorOnlyCache(model_config)

229

230 processing_info = factories.info(ctx)

tests/models/multimodal/processing/test_glm4_1v.py

+2 -2

@@ -40,7 +40,7 @@ def test_processor_override(

40 mm_processor_kwargs=None,

41 limit_mm_per_prompt={"video": 1},

42 )

43 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

43 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

44 tokenizer = processor.info.get_tokenizer()

45 hf_processor_mm_kwargs = {"fps": fps}

@@ -79,7 +79,7 @@ def test_video_loader_consistency(

79 mm_processor_kwargs=None,

80 limit_mm_per_prompt={"video": 1},

81 )

82 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

82 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

83 hf_processor_mm_kwargs = {"fps": fps}

85 # Build the image str / prompt based on the number of images we pass

tests/models/multimodal/processing/test_h2ovl.py

+1 -1

@@ -162,7 +162,7 @@ def test_processor_override(

162 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

163 limit_mm_per_prompt={"image": len(size_factors)},

164 )

165 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

165 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

166 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

167

168 min_num = min_dynamic_patch if dynamic_image_size else 1

tests/models/multimodal/processing/test_idefics3.py

+1 -1

@@ -38,7 +38,7 @@ def test_processor_override(

38 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

39 limit_mm_per_prompt={"image": num_imgs},

40 )

41 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

41 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

42 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

44 # Build the image str / prompt based on the number of images we pass

tests/models/multimodal/processing/test_internvl.py

+1 -1

@@ -116,7 +116,7 @@ def test_processor_override(

116 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

117 limit_mm_per_prompt={"image": len(size_factors)},

118 )

119 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

119 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

120 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

121

122 min_num = min_dynamic_patch if dynamic_image_size else 1

tests/models/multimodal/processing/test_llama4.py

+1 -1

@@ -30,7 +30,7 @@ def test_processor_override(

30 limit_mm_per_prompt={"image": num_imgs},

31 mm_processor_cache_gb=mm_processor_cache_gb,

32 )

33 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

33 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

34 config = processor.info.get_hf_config()

35 tokenizer = processor.info.get_tokenizer()

36 hf_processor = processor.info.get_hf_processor()

tests/models/multimodal/processing/test_llava_next.py

+3 -3

@@ -42,7 +42,7 @@ def test_processor_max_tokens(model_id):

42 mm_processor_kwargs=None,

43 limit_mm_per_prompt={"image": 1},

44 )

45 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

45 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

46 info = processor.info

48 seen_aspect_ratios = set[float]()

@@ -140,7 +140,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):

140 mm_processor_kwargs=None,

141 limit_mm_per_prompt={"image": num_imgs},

142 )

143 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

143 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

144

145 image_ratios = [

146 (171, 152),

@@ -173,7 +173,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):

173 mm_processor_kwargs=None,

174 limit_mm_per_prompt={"image": num_imgs},

175 )

176 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

176 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

177

178 seen_aspect_ratios = set[float]()

179 image_sizes = list[ImageSize]()

tests/models/multimodal/processing/test_llava_onevision.py

+3 -3

@@ -42,7 +42,7 @@ def test_processor_max_tokens(model_id):

42 mm_processor_kwargs=None,

43 limit_mm_per_prompt={"image": 1},

44 )

45 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

45 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

46 info = processor.info

48 seen_aspect_ratios = set[float]()

@@ -138,7 +138,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):

138 mm_processor_kwargs=None,

139 limit_mm_per_prompt={"image": num_imgs},

140 )

141 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

141 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

142

143 image_ratios = [

144 (171, 152),

@@ -171,7 +171,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):

171 mm_processor_kwargs=None,

172 limit_mm_per_prompt={"image": num_imgs},

173 )

174 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

174 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

175

176 seen_aspect_ratios = set[float]()

177 image_sizes = list[ImageSize]()

tests/models/multimodal/processing/test_minimax_vl_01.py

+2 -2

@@ -24,7 +24,7 @@ def test_processor_override(

24 mm_processor_kwargs=None,

25 limit_mm_per_prompt={"image": num_imgs},

26 )

27 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

27 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

28 prompt = "<image>" * num_imgs

29 image = Image.new("RGB", size=(364, 364))

30 mm_data = {"image": [image] * num_imgs}

@@ -83,7 +83,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):

83 mm_processor_kwargs=None,

84 limit_mm_per_prompt={"image": num_imgs},

85 )

86 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

86 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

88 image_ratios = [

89 (171, 152),

tests/models/multimodal/processing/test_mllama4.py

+1 -1

@@ -25,7 +25,7 @@ def test_profiling(model_id: str, max_model_len: int):

25 limit_mm_per_prompt=mm_counts,

26 )

28 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

28 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

29 profiler = MultiModalProfiler(processor)

31 decoder_dummy_data = profiler.get_decoder_dummy_data(

tests/models/multimodal/processing/test_nemotron_vl.py

+1 -1

@@ -118,7 +118,7 @@ def test_processor_override(

118 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

119 limit_mm_per_prompt={"image": len(size_factors)},

120 )

121 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

121 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

122 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

123

124 min_num = min_dynamic_patch if dynamic_image_size else 1

tests/models/multimodal/processing/test_phi3v.py

+1 -1

@@ -39,7 +39,7 @@ def test_processor_override(

39 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

40 limit_mm_per_prompt={"image": num_imgs},

41 )

42 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

42 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

43 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

45 # Build the image str / prompt based on the number of images we pass

tests/models/multimodal/processing/test_phi4mm.py

+1 -1

@@ -39,7 +39,7 @@ def test_processor_override(

39 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

40 limit_mm_per_prompt={"image": num_imgs},

41 )

42 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

42 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

43 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

45 # Build the image str / prompt based on the number of images we pass

tests/models/multimodal/processing/test_qwen2_vl.py

+1 -1

@@ -34,7 +34,7 @@ def test_processor_override(

34 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

35 limit_mm_per_prompt={"image": num_imgs},

36 )

37 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

37 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

38 tokenizer = processor.info.get_tokenizer()

39 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

tests/models/multimodal/processing/test_smolvlm.py

+1 -1

@@ -38,7 +38,7 @@ def test_processor_override(

38 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

39 limit_mm_per_prompt={"image": num_imgs},

40 )

41 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

41 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.renderer_config)

42 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

44 # Build the image str / prompt based on the number of images we pass

tests/models/multimodal/processing/test_tensor_schema.py

+8 -16

@@ -11,7 +11,7 @@ import pytest

11 import torch.nn as nn

12 from PIL import Image

14 -from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config

14 +from vllm.config import ModelConfig, RendererConfig, VllmConfig, set_current_vllm_config

15 from vllm.config.multimodal import (

16 AudioDummyOptions,

17 BaseDummyOptions,

@@ -31,7 +31,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs

31 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext

32 from vllm.multimodal.utils import group_mm_kwargs_by_modality

33 from vllm.platforms import current_platform

34 -from vllm.tokenizers import cached_tokenizer_from_config

34 from vllm.utils.collection_utils import is_list_of

35 from vllm.utils.torch_utils import set_default_torch_dtype

@@ -150,7 +149,10 @@ def initialize_dummy_model(

149 backend="nccl",

150 )

151 initialize_model_parallel(tensor_model_parallel_size=1)

153 - vllm_config = VllmConfig(model_config=model_config)

152 + vllm_config = VllmConfig(

153 + model_config=model_config,

154 + renderer_config=RendererConfig(model_config=model_config),

155 + )

156 with set_current_vllm_config(vllm_config=vllm_config):

157 with set_default_torch_dtype(model_config.dtype):

158 model = model_cls(vllm_config=vllm_config)

@@ -182,19 +184,12 @@ def test_model_tensor_schema(model_id: str):

184 else:

185 dtype = model_info.dtype

186

185 - model_config = ModelConfig(

187 + renderer_config = model_info.build_renderer_config(

188 model_id,

187 - tokenizer=model_info.tokenizer or model_id,

188 - tokenizer_mode=model_info.tokenizer_mode,

189 - revision=model_info.revision,

190 - trust_remote_code=model_info.trust_remote_code,

189 hf_overrides=hf_overrides_fn,

192 - skip_tokenizer_init=model_info.require_embed_inputs,

193 - enable_prompt_embeds=model_info.require_embed_inputs,

194 - enable_mm_embeds=model_info.require_embed_inputs,

195 - enforce_eager=model_info.enforce_eager,

190 dtype=dtype,

191 )

192 + model_config = renderer_config.model_config

193

194 model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)

195 assert supports_multimodal(model_cls)

@@ -212,10 +207,7 @@ def test_model_tensor_schema(model_id: str):

207 if not any(inputs_parse_methods):

208 pytest.skip(f"{model_arch} does not support tensor schema validation.")

209

215 - ctx = InputProcessingContext(

216 - model_config,

217 - tokenizer=cached_tokenizer_from_config(model_config),

218 - )

210 + ctx = InputProcessingContext.from_config(renderer_config)

211 processing_info = factories.info(ctx)

212 supported_mm_limits = processing_info.get_supported_mm_limits()

213 limit_mm_per_prompt = {

tests/models/multimodal/processing/test_transformers.py

+3 -2

@@ -3,7 +3,7 @@

3 import pytest

5 from vllm.assets.image import ImageAsset

6 -from vllm.config import ModelConfig

6 +from vllm.config import ModelConfig, RendererConfig

7 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -13,8 +13,9 @@ def test_multimodal_processor(model_id):

13 model=model_id,

14 model_impl="transformers",

15 )

16 + renderer_config = RendererConfig(model_config=model_config)

17 - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)

18 + mm_processor = MULTIMODAL_REGISTRY.create_processor(renderer_config)

20 image_pil = ImageAsset("cherry_blossom").pil_image

21 mm_data = {"image": image_pil}

tests/models/multimodal/test_mapping.py

+3 -30

@@ -7,7 +7,6 @@ import torch

7 import transformers

8 from transformers import AutoConfig, PreTrainedModel

10 -from vllm.config import ModelConfig

10 from vllm.model_executor.models.utils import WeightsMapper

11 from vllm.multimodal import MULTIMODAL_REGISTRY

12 from vllm.transformers_utils.config import try_get_safetensors_metadata

@@ -50,37 +49,11 @@ def test_hf_model_weights_mapper(model_arch: str):

49 model_info.check_available_online(on_fail="skip")

50 model_info.check_transformers_version(on_fail="skip")

53 - is_mistral_model = model_arch in [

54 - "Mistral3ForConditionalGeneration",

55 - "PixtralForConditionalGeneration",

56 - "VoxtralForConditionalGeneration",

57 - ]

58 -

59 - if not is_mistral_model or model_info.tokenizer_mode == "mistral":

60 - tokenizer_mode = model_info.tokenizer_mode

61 - else:

62 - tokenizer_mode = "hf"

63 -

64 - model_id = model_info.default

65 -

66 - model_config = ModelConfig(

67 - model_id,

68 - tokenizer=model_info.tokenizer or model_id,

69 - tokenizer_mode=tokenizer_mode,

70 - config_format="hf",

71 - revision=model_info.revision,

72 - trust_remote_code=model_info.trust_remote_code,

73 - hf_overrides=model_info.hf_overrides,

74 - skip_tokenizer_init=model_info.require_embed_inputs,

75 - enable_prompt_embeds=model_info.require_embed_inputs,

76 - enable_mm_embeds=model_info.require_embed_inputs,

77 - enforce_eager=model_info.enforce_eager,

78 - dtype=model_info.dtype,

79 - )

52 + model_config = model_info.build_model_config(config_format="hf")

53 model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)

82 - original_weights = create_repo_dummy_weights(model_id)

83 - hf_dummy_model = create_dummy_model(model_id, model_arch)

55 + original_weights = create_repo_dummy_weights(model_config.model)

56 + hf_dummy_model = create_dummy_model(model_config.model, model_arch)

57 hf_converted_weights = hf_dummy_model.named_parameters()

58 hf_converted_buffers = hf_dummy_model.named_buffers()

59 mapper: WeightsMapper = model_cls.hf_to_vllm_mapper

tests/models/registry.py

+32 -1

@@ -9,7 +9,8 @@ import pytest

9 from packaging.version import Version

10 from transformers import __version__ as TRANSFORMERS_VERSION

12 -from vllm.config.model import ModelDType, TokenizerMode

12 +from vllm.config.model import ModelConfig, ModelDType

13 +from vllm.config.renderer import RendererConfig, TokenizerMode

16 @dataclass(frozen=True)

@@ -170,6 +171,36 @@ class _HfExamplesInfo:

171 else:

172 pytest.skip(msg)

173

174 + def build_model_config(self, model: str | None = None, **kwargs) -> ModelConfig:

175 + if model is None:

176 + model = self.default

177 +

178 + return ModelConfig(

179 + **{

180 + "model": model,

181 + "revision": self.revision,

182 + "trust_remote_code": self.trust_remote_code,

183 + "hf_overrides": self.hf_overrides,

184 + "enable_prompt_embeds": self.require_embed_inputs,

185 + "enable_mm_embeds": self.require_embed_inputs,

186 + "enforce_eager": self.enforce_eager,

187 + "dtype": self.dtype,

188 + **kwargs,

189 + }

190 + )

191 +

192 + def build_renderer_config(

193 + self, model: str | None = None, **kwargs

194 + ) -> RendererConfig:

195 + model_config = self.build_model_config(model, **kwargs)

196 +

197 + return RendererConfig(

198 + model_config=model_config,

199 + tokenizer=self.tokenizer or model_config.model,

200 + tokenizer_mode=self.tokenizer_mode,

201 + skip_tokenizer_init=self.require_embed_inputs,

202 + )

203 +

204

205 _TEXT_GENERATION_EXAMPLE_MODELS = {

206 # [Decoder-only]

tests/models/utils.py

+2 -15

@@ -13,7 +13,6 @@ from transformers import PretrainedConfig

13 from vllm.config.model import ModelConfig, ModelDType, RunnerOption

14 from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs

15 from vllm.multimodal.processing import InputProcessingContext

16 -from vllm.tokenizers import cached_tokenizer_from_config

17 from .. import ci_envs

18 from .registry import HF_EXAMPLE_MODELS

@@ -296,30 +295,18 @@ def build_model_context(

295

296 model_config_kwargs = model_config_kwargs or {}

297 limit_mm_per_prompt = limit_mm_per_prompt or {}

299 - model_config = ModelConfig(

298 + renderer_config = model_info.build_renderer_config(

299 model_id,

300 runner=runner,

302 - tokenizer=model_info.tokenizer or model_id,

303 - tokenizer_mode=model_info.tokenizer_mode,

304 - revision=model_info.revision,

305 - trust_remote_code=model_info.trust_remote_code,

301 dtype=dtype,

302 seed=0,

303 mm_processor_kwargs=mm_processor_kwargs,

304 limit_mm_per_prompt=limit_mm_per_prompt,

305 mm_processor_cache_gb=mm_processor_cache_gb,

311 - hf_overrides=model_info.hf_overrides,

312 - skip_tokenizer_init=model_info.require_embed_inputs,

313 - enable_prompt_embeds=model_info.require_embed_inputs,

314 - enable_mm_embeds=model_info.require_embed_inputs,

315 - enforce_eager=model_info.enforce_eager,

306 **model_config_kwargs,

307 )

308

319 - return InputProcessingContext(

320 - model_config,

321 - tokenizer=cached_tokenizer_from_config(model_config),

322 - )

309 + return InputProcessingContext.from_config(renderer_config)

310

311

312 def check_embeddings_close(

tests/multimodal/test_cache.py

+16 -11

@@ -6,7 +6,7 @@ import numpy as np

6 import pytest

7 import torch

9 -from vllm.config import ModelConfig, ParallelConfig, VllmConfig

9 +from vllm.config import ModelConfig, ParallelConfig, RendererConfig, VllmConfig

10 from vllm.multimodal import MULTIMODAL_REGISTRY

11 from vllm.multimodal.cache import (

12 BaseMultiModalProcessorCache,

@@ -110,11 +110,14 @@ def _create_vllm_config(

110 mm_processor_cache_gb: float,

111 enable_ipc: bool,

112 ):

113 + model_config = ModelConfig(

114 + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",

115 + mm_processor_cache_gb=mm_processor_cache_gb,

116 + )

117 +

118 return VllmConfig(

114 - model_config=ModelConfig(

115 - model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",

116 - mm_processor_cache_gb=mm_processor_cache_gb,

117 - ),

119 + model_config=model_config,

120 + renderer_config=RendererConfig(model_config=model_config),

121 parallel_config=ParallelConfig(data_parallel_size=1 if enable_ipc else 2),

122 )

123

@@ -506,13 +509,15 @@ def _run_test_cache_eviction_shm(

509

510

511 def test_cache_eviction_shm_cache():

512 + model_config = ModelConfig(

513 + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",

514 + mm_processor_cache_type="shm",

515 + mm_shm_cache_max_object_size_mb=6,

516 + mm_processor_cache_gb=15.2 * MiB_bytes / GiB_bytes,

517 + )

518 vllm_config = VllmConfig(

510 - model_config=ModelConfig(

511 - model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",

512 - mm_processor_cache_type="shm",

513 - mm_shm_cache_max_object_size_mb=6,

514 - mm_processor_cache_gb=15.2 * MiB_bytes / GiB_bytes,

515 - ),

519 + model_config=model_config,

520 + renderer_config=RendererConfig(model_config=model_config),

521 )

522 sender_cache = ShmObjectStoreSenderCache(vllm_config)

523 receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())

tests/multimodal/test_processing.py

+15 -9

@@ -7,7 +7,7 @@ from contextlib import nullcontext

7 import numpy as np

8 import pytest

10 -from vllm.config import ModelConfig

10 +from vllm.config import ModelConfig, RendererConfig

11 from vllm.multimodal import MULTIMODAL_REGISTRY

12 from vllm.multimodal.processing import (

13 InputProcessingContext,

@@ -920,8 +920,9 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):

920 model=model_id,

921 limit_mm_per_prompt=limit_mm_per_prompt,

922 )

923 + renderer_config = RendererConfig(model_config=model_config)

924

924 - processor = MULTIMODAL_REGISTRY.create_processor(model_config)

925 + processor = MULTIMODAL_REGISTRY.create_processor(renderer_config)

926 processor._supported_mm_limits = {"image": num_supported}

927

928 profiler = MultiModalProfiler(processor)

@@ -955,8 +956,9 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):

956 model=model_id,

957 limit_mm_per_prompt=limit_mm_per_prompt,

958 )

959 + renderer_config = RendererConfig(model_config=model_config)

960

959 - processor = MULTIMODAL_REGISTRY.create_processor(model_config)

961 + processor = MULTIMODAL_REGISTRY.create_processor(renderer_config)

962

963 rng = np.random.RandomState(0)

964 image = random_image(rng, min_wh=128, max_wh=256)

@@ -1012,11 +1014,13 @@ def test_hf_processor_init_kwargs(

1014 inference_kwargs,

1015 expected_kwargs,

1016 ):

1015 - ctx = InputProcessingContext(

1016 - model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs),

1017 - tokenizer=None,

1017 + model_config = ModelConfig(model_id, mm_processor_kwargs=config_kwargs)

1018 + renderer_config = RendererConfig(

1019 + model_config=model_config,

1020 + tokenizer=model_id,

1021 )

1022

1023 + ctx = InputProcessingContext.from_config(renderer_config)

1024 processor = ctx.get_hf_processor(

1025 DummyProcessor, # type: ignore[arg-type]

1026 **inference_kwargs,

@@ -1045,11 +1049,13 @@ def test_hf_processor_call_kwargs(

1049 inference_kwargs,

1050 expected_kwargs,

1051 ):

1048 - ctx = InputProcessingContext(

1049 - model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs),

1050 - tokenizer=None,

1052 + model_config = ModelConfig(model_id, mm_processor_kwargs=config_kwargs)

1053 + renderer_config = RendererConfig(

1054 + model_config=model_config,

1055 + tokenizer=model_id,

1056 )

1057

1058 + ctx = InputProcessingContext.from_config(renderer_config)

1059 processor = ctx.get_hf_processor(DummyProcessor) # type: ignore[arg-type]

1060

1061 result = ctx.call_hf_processor(processor, {}, inference_kwargs)

tests/multimodal/test_registry.py

+3 -1

@@ -31,4 +31,6 @@ def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected):

31 model_id,

32 limit_mm_per_prompt=limit_mm_per_prompt,

33 )

34 - assert MULTIMODAL_REGISTRY.supports_multimodal_inputs(ctx.model_config) is expected

34 + assert (

35 + MULTIMODAL_REGISTRY.supports_multimodal_inputs(ctx.renderer_config) is expected

36 + )

tests/test_config.py

+80 -51

@@ -13,6 +13,7 @@ from vllm.config import (

13 CompilationConfig,

14 ModelConfig,

15 PoolerConfig,

16 + RendererConfig,

17 SchedulerConfig,

18 VllmConfig,

19 update_config,

@@ -476,27 +477,41 @@ def test_load_config_pt_load_map_location(pt_load_map_location):

477 ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", 131073, 131072, True),

478 ],

479 )

479 -def test_get_and_verify_max_len(

480 +def test_recalculate_max_model_len(

481 model_id, max_model_len, expected_max_len, should_raise

482 ):

482 - """Test get_and_verify_max_len with different configurations."""

483 + """Test recalculate_max_model_len with different configurations."""

484 model_config = ModelConfig(model_id)

485

486 if should_raise:

487 with pytest.raises(ValueError):

487 - model_config.get_and_verify_max_len(max_model_len)

488 + model_config.recalculate_max_model_len(

489 + max_model_len,

490 + tokenizer=model_id,

491 + tokenizer_revision=None,

492 + )

493 else:

489 - actual_max_len = model_config.get_and_verify_max_len(max_model_len)

490 - assert actual_max_len == expected_max_len

494 + model_config.recalculate_max_model_len(

495 + max_model_len,

496 + tokenizer=model_id,

497 + tokenizer_revision=None,

498 + )

499 + assert model_config.max_model_len == expected_max_len

500

501

493 -class MockConfig:

494 - """Simple mock object for testing maybe_pull_model_tokenizer_for_runai"""

502 +class MockModelConfig:

503 + """Simple mock object for testing maybe_pull_model_for_runai"""

504

496 - def __init__(self, model: str, tokenizer: str):

505 + def __init__(self, model: str):

506 self.model = model

498 - self.tokenizer = tokenizer

499 - self.model_weights = None

507 +

508 +

509 +class MockRendererConfig:

510 + """Simple mock object for testing maybe_pull_tokenizer_for_runai"""

511 +

512 + def __init__(self, model_config: MockModelConfig):

513 + self.model_config = model_config

514 + self.tokenizer = model_config.model

515

516

517 @pytest.mark.parametrize(

@@ -514,59 +529,65 @@ def test_s3_url_model_tokenizer_paths(mock_pull_files, s3_url):

529 mock_pull_files.return_value = None

530

531 # Create first mock and run the method

517 - config1 = MockConfig(model=s3_url, tokenizer=s3_url)

518 - ModelConfig.maybe_pull_model_tokenizer_for_runai(config1, s3_url, s3_url)

532 + model_config1 = MockModelConfig(model=s3_url)

533 + renderer_config1 = MockRendererConfig(model_config=model_config1)

534 + ModelConfig.maybe_pull_model_for_runai(model_config1, s3_url)

535 + RendererConfig.maybe_pull_tokenizer_for_runai(renderer_config1, s3_url)

536

537 # Check that model and tokenizer point to existing directories

521 - assert os.path.exists(config1.model), (

522 - f"Model directory does not exist: {config1.model}"

538 + assert os.path.exists(model_config1.model), (

539 + f"Model directory does not exist: {model_config1.model}"

540 )

524 - assert os.path.isdir(config1.model), (

525 - f"Model path is not a directory: {config1.model}"

541 + assert os.path.isdir(model_config1.model), (

542 + f"Model path is not a directory: {model_config1.model}"

543 )

527 - assert os.path.exists(config1.tokenizer), (

528 - f"Tokenizer directory does not exist: {config1.tokenizer}"

544 + assert os.path.exists(renderer_config1.tokenizer), (

545 + f"Tokenizer directory does not exist: {renderer_config1.tokenizer}"

546 )

530 - assert os.path.isdir(config1.tokenizer), (

531 - f"Tokenizer path is not a directory: {config1.tokenizer}"

547 + assert os.path.isdir(renderer_config1.tokenizer), (

548 + f"Tokenizer path is not a directory: {renderer_config1.tokenizer}"

549 )

550

551 # Verify that the paths are different from the original S3 URL

535 - assert config1.model != s3_url, "Model path should be converted to local directory"

536 - assert config1.tokenizer != s3_url, (

552 + assert model_config1.model != s3_url, (

553 + "Model path should be converted to local directory"

554 + )

555 + assert renderer_config1.tokenizer != s3_url, (

556 "Tokenizer path should be converted to local directory"

557 )

558

559 # Store the original paths

541 - created_model_dir = config1.model

542 - create_tokenizer_dir = config1.tokenizer

560 + created_model_dir = model_config1.model

561 + create_tokenizer_dir = renderer_config1.tokenizer

562

563 # Create a new mock and run the method with the same S3 URL

545 - config2 = MockConfig(model=s3_url, tokenizer=s3_url)

546 - ModelConfig.maybe_pull_model_tokenizer_for_runai(config2, s3_url, s3_url)

564 + model_config2 = MockModelConfig(model=s3_url)

565 + renderer_config2 = MockRendererConfig(model_config=model_config2)

566 + ModelConfig.maybe_pull_model_for_runai(model_config2, s3_url)

567 + RendererConfig.maybe_pull_tokenizer_for_runai(renderer_config2, s3_url)

568

569 # Check that the new directories exist

549 - assert os.path.exists(config2.model), (

550 - f"Model directory does not exist: {config2.model}"

570 + assert os.path.exists(model_config2.model), (

571 + f"Model directory does not exist: {model_config2.model}"

572 )

552 - assert os.path.isdir(config2.model), (

553 - f"Model path is not a directory: {config2.model}"

573 + assert os.path.isdir(model_config2.model), (

574 + f"Model path is not a directory: {model_config2.model}"

575 )

555 - assert os.path.exists(config2.tokenizer), (

556 - f"Tokenizer directory does not exist: {config2.tokenizer}"

576 + assert os.path.exists(renderer_config2.tokenizer), (

577 + f"Tokenizer directory does not exist: {renderer_config2.tokenizer}"

578 )

558 - assert os.path.isdir(config2.tokenizer), (

559 - f"Tokenizer path is not a directory: {config2.tokenizer}"

579 + assert os.path.isdir(renderer_config2.tokenizer), (

580 + f"Tokenizer path is not a directory: {renderer_config2.tokenizer}"

581 )

582

583 # Verify that the paths are deterministic (same as before)

563 - assert config2.model == created_model_dir, (

584 + assert model_config2.model == created_model_dir, (

585 f"Model paths are not deterministic. "

565 - f"Original: {created_model_dir}, New: {config2.model}"

586 + f"Original: {created_model_dir}, New: {model_config2.model}"

587 )

567 - assert config2.tokenizer == create_tokenizer_dir, (

588 + assert renderer_config2.tokenizer == create_tokenizer_dir, (

589 f"Tokenizer paths are not deterministic. "

569 - f"Original: {create_tokenizer_dir}, New: {config2.tokenizer}"

590 + f"Original: {create_tokenizer_dir}, New: {renderer_config2.tokenizer}"

591 )

592

593

@@ -580,28 +601,36 @@ def test_s3_url_different_models_create_different_directories(mock_pull_files):

601 s3_url2 = "s3://example-bucket-2/model/"

602

603 # Create mocks with different S3 URLs and run the method

583 - config1 = MockConfig(model=s3_url1, tokenizer=s3_url1)

584 - ModelConfig.maybe_pull_model_tokenizer_for_runai(config1, s3_url1, s3_url1)

604 + model_config1 = MockModelConfig(model=s3_url1)

605 + renderer_config1 = MockRendererConfig(model_config=model_config1)

606 + ModelConfig.maybe_pull_model_for_runai(model_config1, s3_url1)

607 + RendererConfig.maybe_pull_tokenizer_for_runai(renderer_config1, s3_url1)

608

586 - config2 = MockConfig(model=s3_url2, tokenizer=s3_url2)

587 - ModelConfig.maybe_pull_model_tokenizer_for_runai(config2, s3_url2, s3_url2)

609 + model_config2 = MockModelConfig(model=s3_url2)

610 + renderer_config2 = MockRendererConfig(model_config=model_config2)

611 + ModelConfig.maybe_pull_model_for_runai(model_config2, s3_url2)

612 + RendererConfig.maybe_pull_tokenizer_for_runai(renderer_config2, s3_url2)

613

614 # Verify that different URLs produce different directories

590 - assert config1.model != config2.model, (

615 + assert model_config1.model != model_config2.model, (

616 f"Different S3 URLs should create different model directories. "

592 - f"URL1 model: {config1.model}, URL2 model: {config2.model}"

617 + f"URL1 model: {model_config1.model}, URL2 model: {model_config2.model}"

618 )

594 - assert config1.tokenizer != config2.tokenizer, (

619 + assert renderer_config1.tokenizer != renderer_config2.tokenizer, (

620 f"Different S3 URLs should create different tokenizer directories. "

596 - f"URL1 tokenizer: {config1.tokenizer}, "

597 - f"URL2 tokenizer: {config2.tokenizer}"

621 + f"URL1 tokenizer: {renderer_config1.tokenizer}, "

622 + f"URL2 tokenizer: {renderer_config2.tokenizer}"

623 )

624

625 # Verify that both sets of directories exist

601 - assert os.path.exists(config1.model) and os.path.isdir(config1.model)

602 - assert os.path.exists(config1.tokenizer) and os.path.isdir(config1.tokenizer)

603 - assert os.path.exists(config2.model) and os.path.isdir(config2.model)

604 - assert os.path.exists(config2.tokenizer) and os.path.isdir(config2.tokenizer)

626 + assert os.path.exists(model_config1.model) and os.path.isdir(model_config1.model)

627 + assert os.path.exists(renderer_config1.tokenizer) and os.path.isdir(

628 + renderer_config1.tokenizer

629 + )

630 + assert os.path.exists(model_config2.model) and os.path.isdir(model_config2.model)

631 + assert os.path.exists(renderer_config2.tokenizer) and os.path.isdir(

632 + renderer_config2.tokenizer

633 + )

634

635

636 @pytest.mark.parametrize(

tests/test_inputs.py

+4 -3

@@ -3,7 +3,7 @@

4 import pytest

6 -from vllm.config import ModelConfig

6 +from vllm.config import ModelConfig, RendererConfig

7 from vllm.inputs import zip_enc_dec_prompts

8 from vllm.inputs.parse import parse_raw_prompts

9 from vllm.inputs.preprocess import InputPreprocessor

@@ -108,8 +108,9 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):

108 )

109 def test_preprocessor_always_mm_code_path(model_id, prompt):

110 model_config = ModelConfig(model=model_id)

111 - tokenizer = init_tokenizer_from_config(model_config)

112 - input_preprocessor = InputPreprocessor(model_config, tokenizer)

111 + renderer_config = RendererConfig(model_config=model_config)

112 + tokenizer = init_tokenizer_from_config(renderer_config)

113 + input_preprocessor = InputPreprocessor(renderer_config, tokenizer)

114

115 # HF processor adds sep token

116 sep_token_id = tokenizer.vocab[tokenizer.sep_token]

tests/v1/attention/utils.py

@@ -16,6 +16,7 @@ from vllm.config import (

16 LoadConfig,

17 ModelConfig,

18 ParallelConfig,

19 + RendererConfig,

20 SchedulerConfig,

21 VllmConfig,

22 )

@@ -216,6 +217,7 @@ def create_vllm_config(

217

218 return VllmConfig(

219 model_config=model_config,

220 + renderer_config=RendererConfig(model_config=model_config),

221 cache_config=cache_config,

222 parallel_config=parallel_config,

223 scheduler_config=scheduler_config,

tests/v1/core/test_kv_cache_utils.py

+16 -4

@@ -8,7 +8,7 @@ import pytest

8 import torch

10 import vllm.v1.core.kv_cache_utils as kv_cache_utils

11 -from vllm.config import ModelConfig, SchedulerConfig, VllmConfig

11 +from vllm.config import ModelConfig, RendererConfig, SchedulerConfig, VllmConfig

12 from vllm.lora.request import LoRARequest

13 from vllm.multimodal.inputs import (

14 MultiModalFeatureSpec,

@@ -667,7 +667,10 @@ def test_metrics_empty_stats():

667

668 def test_get_kv_cache_configs_multiple_workers():

669 model_config = ModelConfig(max_model_len=16)

670 - vllm_config = VllmConfig(model_config=model_config)

670 + vllm_config = VllmConfig(

671 + model_config=model_config,

672 + renderer_config=RendererConfig(model_config=model_config),

673 + )

674

675 ref_kv_cache_spec = new_kv_cache_spec()

676 same_kv_cache_specs = [

@@ -1136,6 +1139,7 @@ def test_estimate_max_model_len(model_id, max_model_len, want_estimated_max_len)

1139

1140 vllm_config = VllmConfig(

1141 model_config=model_config,

1142 + renderer_config=RendererConfig(model_config=model_config),

1143 scheduler_config=scheduler_config,

1144 )

1145

@@ -1175,6 +1179,7 @@ def test_get_max_concurrency_for_kv_cache_config():

1179

1180 vllm_config = VllmConfig(

1181 model_config=model_config,

1182 + renderer_config=RendererConfig(model_config=model_config),

1183 scheduler_config=scheduler_config,

1184 )

1185

@@ -1293,7 +1298,10 @@ def test_allocate_with_lookahead():

1298 def test_get_kv_cache_config_one_worker():

1299 # pass max_model_len to pass check_enough_kv_cache_memory

1300 model_config = ModelConfig(max_model_len=16)

1296 - vllm_config = VllmConfig(model_config=model_config)

1301 + vllm_config = VllmConfig(

1302 + model_config=model_config,

1303 + renderer_config=RendererConfig(model_config=model_config),

1304 + )

1305

1306 mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2

1307 # all layers are full attention -> single group

@@ -1584,7 +1592,11 @@ def test_get_kv_cache_config_one_worker():

1592

1593 def test_get_kv_cache_configs_attention_free():

1594 kv_cache_specs: dict[str, KVCacheSpec] = {}

1587 - vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=16))

1595 + model_config = ModelConfig(max_model_len=16)

1596 + vllm_config = VllmConfig(

1597 + model_config=model_config,

1598 + renderer_config=RendererConfig(model_config=model_config),

1599 + )

1600 kv_cache_configs = get_kv_cache_configs(vllm_config, [kv_cache_specs], [0])

1601 assert kv_cache_configs == [

1602 KVCacheConfig(

tests/v1/core/test_scheduler.py

@@ -11,6 +11,7 @@ from vllm.config import (

11 ECTransferConfig,

12 KVTransferConfig,

13 ModelConfig,

14 + RendererConfig,

15 SchedulerConfig,

16 SpeculativeConfig,

17 VllmConfig,

@@ -1563,6 +1564,7 @@ def create_scheduler_with_priority(

1564 vllm_config = VllmConfig(

1565 scheduler_config=scheduler_config,

1566 model_config=model_config,

1567 + renderer_config=RendererConfig(model_config=model_config),

1568 cache_config=cache_config,

1569 kv_transfer_config=kv_transfer_config,

1570 speculative_config=speculative_config,

tests/v1/core/utils.py

@@ -9,6 +9,7 @@ from vllm.config import (

9 ECTransferConfig,

10 KVTransferConfig,

11 ModelConfig,

12 + RendererConfig,

13 SchedulerConfig,

14 SpeculativeConfig,

15 VllmConfig,

@@ -132,6 +133,7 @@ def create_scheduler(

133 vllm_config = VllmConfig(

134 scheduler_config=scheduler_config,

135 model_config=model_config,

136 + renderer_config=RendererConfig(model_config=model_config),

137 cache_config=cache_config,

138 kv_transfer_config=kv_transfer_config,

139 speculative_config=speculative_config,

tests/v1/engine/test_engine_core.py

@@ -15,6 +15,7 @@ from vllm.config import (

15 ECTransferConfig,

16 KVTransferConfig,

17 ModelConfig,

18 + RendererConfig,

19 SchedulerConfig,

20 VllmConfig,

21 )

@@ -522,6 +523,7 @@ def test_encoder_instance_zero_kv_cache(

523

524 vllm_config = VllmConfig(

525 model_config=model_config,

526 + renderer_config=RendererConfig(model_config=model_config),

527 cache_config=cache_config,

528 scheduler_config=scheduler_config,

529 kv_transfer_config=kv_transfer_config,

tests/v1/engine/test_process_multi_modal_uuids.py

+15 -9

@@ -5,7 +5,14 @@ import pytest

6 from vllm.assets.image import ImageAsset

7 from vllm.assets.video import VideoAsset

8 -from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig

8 +from vllm.config import (

9 + CacheConfig,

10 + DeviceConfig,

11 + ModelConfig,

12 + MultiModalConfig,

13 + RendererConfig,

14 + VllmConfig,

15 +)

16 from vllm.sampling_params import SamplingParams

17 from vllm.v1.engine import input_processor as input_processor_mod

18 from vllm.v1.engine.input_processor import InputProcessor

@@ -44,22 +51,21 @@ def _mock_input_processor(

51 monkeypatch.setattr(VllmConfig, "__post_init__", lambda self: None, raising=True)

53 model_config = ModelConfig(

47 - skip_tokenizer_init=True,

54 max_model_len=128,

55 mm_processor_cache_gb=mm_cache_gb,

56 generation_config="vllm",

51 - tokenizer="dummy",

57 )

58 + model_config.multimodal_config = MultiModalConfig(mm_processor_cache_gb=mm_cache_gb)

54 - # Minimal multimodal_config to satisfy references in

55 - # Processor.process_inputs.

56 - class _MockMMConfig:

57 - def __init__(self, gb: float):

58 - self.mm_processor_cache_gb = gb

60 + renderer_config = RendererConfig(

61 + model_config=model_config,

62 + tokenizer="dummy",

63 + skip_tokenizer_init=True,

64 + )

60 - model_config.multimodal_config = _MockMMConfig(mm_cache_gb) # type: ignore[attr-defined]

66 vllm_config = VllmConfig(

67 model_config=model_config,

68 + renderer_config=renderer_config,

69 cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching),

70 device_config=DeviceConfig(device="cpu"),

71 )

tests/v1/kv_connector/unit/utils.py

@@ -15,6 +15,7 @@ from vllm.config import (

15 DeviceConfig,

16 KVTransferConfig,

17 ModelConfig,

18 + RendererConfig,

19 SchedulerConfig,

20 VllmConfig,

21 )

@@ -127,6 +128,7 @@ def create_vllm_config(

128 return VllmConfig(

129 scheduler_config=scheduler_config,

130 model_config=model_config,

131 + renderer_config=RendererConfig(model_config=model_config),

132 cache_config=cache_config,

133 kv_transfer_config=kv_transfer_config,

134 device_config=DeviceConfig("cpu"),

tests/v1/spec_decode/test_eagle.py

@@ -19,6 +19,7 @@ from vllm.config import (

19 DeviceConfig,

20 ModelConfig,

21 ParallelConfig,

22 + RendererConfig,

23 SchedulerConfig,

24 SpeculativeConfig,

25 VllmConfig,

@@ -61,6 +62,7 @@ def _create_proposer(

63 vllm_config = VllmConfig(

64 model_config=model_config,

65 + renderer_config=RendererConfig(model_config=model_config),

66 cache_config=CacheConfig(),

67 speculative_config=speculative_config,

68 device_config=DeviceConfig(device=current_platform.device_type),

tests/v1/spec_decode/test_mtp.py

@@ -18,6 +18,7 @@ from vllm.config import (

18 DeviceConfig,

19 ModelConfig,

20 ParallelConfig,

21 + RendererConfig,

22 SchedulerConfig,

23 SpeculativeConfig,

24 VllmConfig,

@@ -46,6 +47,7 @@ def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer:

48 vllm_config = VllmConfig(

49 model_config=model_config,

50 + renderer_config=RendererConfig(model_config=model_config),

51 cache_config=CacheConfig(),

52 speculative_config=speculative_config,

53 device_config=DeviceConfig(device=current_platform.device_type),

tests/v1/spec_decode/test_ngram.py

@@ -4,6 +4,7 @@ import numpy as np

5 from vllm.config import (

6 ModelConfig,

7 + RendererConfig,

8 SpeculativeConfig,

9 VllmConfig,

10 )

@@ -69,6 +70,7 @@ def test_ngram_proposer():

70 return NgramProposer(

71 vllm_config=VllmConfig(

72 model_config=model_config,

73 + renderer_config=RendererConfig(model_config=model_config),

74 speculative_config=SpeculativeConfig(

75 prompt_lookup_min=min_n,

76 prompt_lookup_max=max_n,

tests/v1/structured_output/test_backend_guidance.py

+9 -3

@@ -6,7 +6,7 @@ from concurrent.futures import Future

6 import pytest

7 from transformers import AutoTokenizer

9 -from vllm.config import StructuredOutputsConfig, VllmConfig

9 +from vllm.config import RendererConfig, StructuredOutputsConfig, VllmConfig

10 from vllm.config.model import ModelConfig

11 from vllm.config.parallel import ParallelConfig

12 from vllm.config.speculative import SpeculativeConfig

@@ -72,8 +72,11 @@ def test_backend_guidance_rollback_terminated():

72 def test_grammar_bitmask_with_specdec():

73 tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)

74 prompt = tokenizer.encode('{"a": "b"}')

75 +

76 + model_config = ModelConfig(tokenizer=TOKENIZER)

77 vllm_config = VllmConfig(

76 - model_config=ModelConfig(tokenizer=TOKENIZER),

78 + model_config=model_config,

79 + renderer_config=RendererConfig(model_config=model_config, tokenizer=TOKENIZER),

80 structured_outputs_config=StructuredOutputsConfig(backend="guidance"),

81 speculative_config=SpeculativeConfig(model="[ngram]", num_speculative_tokens=3),

82 )

@@ -137,8 +140,11 @@ def test_grammar_init_async_and_sync(async_grammar):

140

141 # Use "external_launcher" for sync mode, None for async mode

142 executor_backend = None if async_grammar else "external_launcher"

143 +

144 + model_config = ModelConfig(tokenizer=TOKENIZER)

145 vllm_config = VllmConfig(

141 - model_config=ModelConfig(tokenizer=TOKENIZER),

146 + model_config=model_config,

147 + renderer_config=RendererConfig(model_config=model_config, tokenizer=TOKENIZER),

148 structured_outputs_config=StructuredOutputsConfig(backend="guidance"),

149 parallel_config=ParallelConfig(distributed_executor_backend=executor_backend),

150 )

tests/v1/structured_output/test_reasoning_structured_output.py

+21 -14

@@ -7,7 +7,7 @@ from unittest.mock import Mock

8 import pytest

10 -from vllm.config import ModelConfig, SchedulerConfig, VllmConfig

10 +from vllm.config import ModelConfig, RendererConfig, SchedulerConfig, VllmConfig

11 from vllm.reasoning import ReasoningParser

12 from vllm.v1.request import Request

13 from vllm.v1.structured_output import StructuredOutputManager

@@ -17,19 +17,26 @@ class TestReasoningStructuredOutput:

17 """Test reasoning-aware structured output functionality."""

19 @pytest.fixture

20 - def mock_model_config(self):

21 - """Create a mock ModelConfig."""

22 - config = Mock(spec=ModelConfig)

23 - config.skip_tokenizer_init = True # Skip tokenizer init to avoid network calls

24 - config.get_vocab_size = Mock(return_value=50000)

20 + def mock_renderer_config(self):

21 + """Create a mock RendererConfig."""

22 + renderer_config = Mock(spec=RendererConfig)

23 + renderer_config.skip_tokenizer_init = (

24 + True # Skip tokenizer init to avoid network calls

25 + )

26 +

27 + model_config = Mock(spec=ModelConfig)

28 + model_config.get_vocab_size = Mock(return_value=50000)

29 + model_config.trust_remote_code = False

30 # Add missing runner_type attribute that tokenizer initialization expects

26 - config.runner_type = "generate"

31 + model_config.runner_type = "generate"

32 + renderer_config.model_config = model_config

33 +

34 # Add other attributes that tokenizer initialization might need

28 - config.tokenizer = "test-tokenizer"

29 - config.tokenizer_mode = "auto"

30 - config.trust_remote_code = False

31 - config.tokenizer_revision = None

32 - return config

35 + renderer_config.tokenizer = "test-tokenizer"

36 + renderer_config.tokenizer_mode = "auto"

37 + renderer_config.tokenizer_revision = None

38 +

39 + return renderer_config

41 @pytest.fixture

42 def mock_scheduler_config(self):

@@ -39,10 +46,10 @@ class TestReasoningStructuredOutput:

46 return config

48 @pytest.fixture

42 - def mock_vllm_config(self, mock_model_config, mock_scheduler_config):

49 + def mock_vllm_config(self, mock_renderer_config, mock_scheduler_config):

50 """Create a mock VllmConfig."""

51 config = Mock(spec=VllmConfig)

45 - config.model_config = mock_model_config

52 + config.renderer_config = mock_renderer_config

53 config.scheduler_config = mock_scheduler_config

54 config.structured_outputs_config = Mock()

55 config.structured_outputs_config.reasoning_parser = None

tests/v1/tpu/worker/test_tpu_model_runner.py

@@ -7,6 +7,7 @@ from vllm.attention.layer import Attention

7 from vllm.config import (

8 CacheConfig,

9 ModelConfig,

10 + RendererConfig,

11 SchedulerConfig,

12 VllmConfig,

13 set_current_vllm_config,

@@ -45,6 +46,7 @@ def get_vllm_config():

46 )

47 vllm_config = VllmConfig(

48 model_config=model_config,

49 + renderer_config=RendererConfig(model_config=model_config),

50 cache_config=cache_config,

51 scheduler_config=scheduler_config,

52 )

tests/v1/worker/test_gpu_model_runner.py

@@ -13,6 +13,7 @@ from vllm.config import (

13 CacheConfig,

14 ModelConfig,

15 ParallelConfig,

16 + RendererConfig,

17 SchedulerConfig,

18 VllmConfig,

19 set_current_vllm_config,

@@ -101,6 +102,7 @@ def get_vllm_config():

102 parallel_config = ParallelConfig()

103 vllm_config = VllmConfig(

104 model_config=model_config,

105 + renderer_config=RendererConfig(model_config=model_config),

106 cache_config=cache_config,

107 scheduler_config=scheduler_config,

108 parallel_config=parallel_config,

@@ -811,6 +813,7 @@ def test_hybrid_attention_mamba_tensor_shapes():

813 attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASHINFER)

814 vllm_config = VllmConfig(

815 model_config=model_config,

816 + renderer_config=RendererConfig(model_config=model_config),

817 cache_config=cache_config,

818 scheduler_config=scheduler_config,

819 parallel_config=parallel_config,

vllm/config/__init__.py

@@ -24,6 +24,7 @@ from vllm.config.multimodal import MultiModalConfig

24 from vllm.config.observability import ObservabilityConfig

25 from vllm.config.parallel import EPLBConfig, ParallelConfig

26 from vllm.config.pooler import PoolerConfig

27 +from vllm.config.renderer import RendererConfig

28 from vllm.config.scheduler import SchedulerConfig

29 from vllm.config.speculative import SpeculativeConfig

30 from vllm.config.speech_to_text import SpeechToTextConfig

@@ -81,6 +82,8 @@ __all__ = [

82 "ParallelConfig",

83 # From vllm.config.pooler

84 "PoolerConfig",

85 + # From vllm.config.renderer

86 + "RendererConfig",

87 # From vllm.config.scheduler

88 "SchedulerConfig",

89 # From vllm.config.speculative

vllm/config/model.py

+31 -110

@@ -36,7 +36,6 @@ from vllm.transformers_utils.config import (

36 uses_xdrope_dim,

37 )

38 from vllm.transformers_utils.gguf_utils import (

39 - is_gguf,

39 is_remote_gguf,

40 maybe_patch_hf_config_from_gguf,

41 split_remote_gguf,

@@ -83,7 +82,6 @@ TaskOption = Literal[

82 "transcription",

83 "draft",

84 ]

86 -TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32"]

85 ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]

86 LogprobsMode = Literal[

87 "raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"

@@ -131,18 +129,6 @@ class ModelConfig:

129

130 Note that the model may support other tasks using the same model runner.

131 """

134 - tokenizer: SkipValidation[str] = None # type: ignore

135 - """Name or path of the Hugging Face tokenizer to use. If unspecified, model

136 - name or path will be used."""

137 - tokenizer_mode: TokenizerMode | str = "auto"

138 - """Tokenizer mode:\n

139 - - "auto" will use the tokenizer from `mistral_common` for Mistral models

140 - if available, otherwise it will use the "hf" tokenizer.\n

141 - - "hf" will use the fast tokenizer if available.\n

142 - - "slow" will always use the slow tokenizer.\n

143 - - "mistral" will always use the tokenizer from `mistral_common`.\n

144 - - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n

145 - - Other custom values can be supported via plugins."""

132 trust_remote_code: bool = False

133 """Trust remote code (e.g., from HuggingFace) when downloading the model

134 and tokenizer."""

@@ -168,13 +154,6 @@ class ModelConfig:

154 hf_config_path: str | None = None

155 """Name or path of the Hugging Face config to use. If unspecified, model

156 name or path will be used."""

171 - allowed_local_media_path: str = ""

172 - """Allowing API requests to read local images or videos from directories

173 - specified by the server file system. This is a security risk. Should only

174 - be enabled in trusted environments."""

175 - allowed_media_domains: list[str] | None = None

176 - """If set, only media URLs that belong to this domain can be used for

177 - multi-modal inputs. """

157 revision: str | None = None

158 """The specific model version to use. It can be a branch name, a tag name,

159 or a commit id. If unspecified, will use the default version."""

@@ -182,10 +161,6 @@ class ModelConfig:

161 """The specific revision to use for the model code on the Hugging Face Hub.

162 It can be a branch name, a tag name, or a commit id. If unspecified, will

163 use the default version."""

185 - tokenizer_revision: str | None = None

186 - """The specific revision to use for the tokenizer on the Hugging Face Hub.

187 - It can be a branch name, a tag name, or a commit id. If unspecified, will

188 - use the default version."""

164 max_model_len: SkipValidation[int] = None # type: ignore

165 """Model context length (prompt and output). If unspecified, will be

166 automatically derived from the model config.

@@ -230,10 +205,6 @@ class ModelConfig:

205 preventing potential numerical issues. Note that even if this is set to

206 False, cascade attention will be only used when the heuristic tells that

207 it's beneficial."""

233 - skip_tokenizer_init: bool = False

234 - """Skip initialization of tokenizer and detokenizer. Expects valid

235 - `prompt_token_ids` and `None` for prompt from the input. The generated

236 - output will contain token ids."""

208 enable_prompt_embeds: bool = False

209 """If `True`, enables passing text embeddings as inputs via the

210 `prompt_embeds` key.

@@ -294,8 +265,6 @@ class ModelConfig:

265 logits_processors: list[str | type[LogitsProcessor]] | None = None

266 """One or more logits processors' fully-qualified class names or class

267 definitions"""

297 - io_processor_plugin: str | None = None

298 - """IOProcessor plugin name to load at model startup"""

268

269 # Pooler config

270 pooler_config: PoolerConfig | None = None

@@ -308,7 +277,6 @@ class ModelConfig:

277 from the architecture of `self.model`."""

278 limit_mm_per_prompt: InitVar[dict[str, int | dict[str, int]] | None] = None

279 enable_mm_embeds: InitVar[bool | None] = None

311 - media_io_kwargs: InitVar[dict[str, dict[str, Any]] | None] = None

280 mm_processor_kwargs: InitVar[dict[str, Any] | None] = None

281 mm_processor_cache_gb: InitVar[float | None] = None

282 mm_processor_cache_type: InitVar[MMCacheType | None] = None

@@ -335,18 +303,12 @@ class ModelConfig:

303 "runner",

304 "convert",

305 "task",

338 - "tokenizer",

339 - "tokenizer_mode",

306 "seed",

307 "hf_config_path",

342 - "allowed_local_media_path",

343 - "allowed_media_domains",

344 - "tokenizer_revision",

308 "spec_target_max_model_len",

309 "enforce_eager",

310 "logprobs_mode",

311 "disable_cascade_attn",

349 - "skip_tokenizer_init",

312 "served_model_name",

313 "config_format",

314 "hf_token",

@@ -354,11 +316,9 @@ class ModelConfig:

316 "logits_processor_pattern",

317 "override_attention_dtype",

318 "logits_processors",

357 - "io_processor_plugin",

319 "pooler_config",

320 "multimodal_config",

321 "limit_mm_per_prompt",

361 - "media_io_kwargs",

322 "mm_processor_kwargs",

323 "mm_processor_cache_gb",

324 "mm_processor_cache_type",

@@ -423,7 +383,6 @@ class ModelConfig:

383 # Multimodal config init vars

384 limit_mm_per_prompt: dict[str, int | dict[str, int]] | None,

385 enable_mm_embeds: bool | None,

426 - media_io_kwargs: dict[str, dict[str, Any]] | None,

386 mm_processor_kwargs: dict[str, Any] | None,

387 mm_processor_cache_gb: float | None,

388 mm_processor_cache_type: MMCacheType | None,

@@ -438,13 +397,8 @@ class ModelConfig:

397 self.served_model_name = get_served_model_name(

398 self.model, self.served_model_name

399 )

441 - self.model = maybe_model_redirect(self.model)

442 - # The tokenizer is consistent with the model by default.

443 - if self.tokenizer is None:

444 - self.tokenizer = self.model

445 - if self.tokenizer_revision is None:

446 - self.tokenizer_revision = self.revision

447 - self.tokenizer = maybe_model_redirect(self.tokenizer)

400 + self.original_model = self.model

401 + self.model = maybe_model_redirect(self.original_model)

402

403 if isinstance(self.hf_config_path, str):

404 self.hf_config_path = maybe_model_redirect(self.hf_config_path)

@@ -465,7 +419,7 @@ class ModelConfig:

419 hf_overrides_kw[key] = value

420 hf_overrides_fn = None

421

468 - self.maybe_pull_model_tokenizer_for_runai(self.model, self.tokenizer)

422 + self.maybe_pull_model_for_runai(self.model)

423

424 from vllm.platforms import current_platform

425

@@ -648,7 +602,8 @@ class ModelConfig:

602 )

603

604 self.original_max_model_len = self.max_model_len

651 - self.max_model_len = self.get_and_verify_max_len(self.max_model_len)

605 + self.recalculate_max_model_len(self.original_max_model_len)

606 +

607 # Init multimodal config if needed

608 if self._model_info.supports_multimodal:

609 if (

@@ -664,7 +619,6 @@ class ModelConfig:

619 mm_config_kwargs = dict(

620 limit_per_prompt=limit_mm_per_prompt,

621 enable_mm_embeds=enable_mm_embeds,

667 - media_io_kwargs=media_io_kwargs,

622 mm_processor_kwargs=mm_processor_kwargs,

623 mm_processor_cache_gb=mm_processor_cache_gb,

624 mm_processor_cache_type=mm_processor_cache_type,

@@ -682,16 +636,8 @@ class ModelConfig:

636

637 self.multimodal_config = MultiModalConfig(**mm_config_kwargs)

638

685 - # Multimodal GGUF models must use original repo for mm processing

686 - if is_gguf(self.tokenizer) and self.is_multimodal_model:

687 - raise ValueError(

688 - "Loading a multimodal GGUF model needs to use original "

689 - "tokenizer. Please specify the unquantized hf model's "

690 - "repo name or path using the --tokenizer argument."

691 - )

692 -

639 if self.disable_sliding_window:

694 - # Set after get_and_verify_max_len to ensure that max_model_len

640 + # Set after recalculate_max_model_len to ensure that max_model_len

641 # can be correctly capped to sliding window size

642 self.hf_text_config.sliding_window = None

643

@@ -715,10 +661,9 @@ class ModelConfig:

661

662 @model_validator(mode="after")

663 def validate_model_config_after(self: "ModelConfig") -> "ModelConfig":

718 - if not isinstance(self.tokenizer, str):

719 - raise ValueError("tokenizer must be a string after __post_init__.")

664 if not isinstance(self.max_model_len, int):

665 raise ValueError("max_model_len must be an integer after __post_init__.")

666 +

667 return self

668

669 def _get_transformers_backend_cls(self) -> str:

@@ -767,49 +712,17 @@ class ModelConfig:

712 """The architecture vllm actually used."""

713 return self._architecture

714

770 - def maybe_pull_model_tokenizer_for_runai(self, model: str, tokenizer: str) -> None:

771 - """Pull model/tokenizer from Object Storage to temporary

772 - directory when needed.

773 -

774 - Args:

775 - model: Model name or path

776 - tokenizer: Tokenizer name or path

777 - """

778 -

779 - if not (is_runai_obj_uri(model) or is_runai_obj_uri(tokenizer)):

715 + def maybe_pull_model_for_runai(self, model: str) -> None:

716 + """Pull model from Object Storage to temporary directory when needed."""

717 + if not is_runai_obj_uri(model):

718 return

719

782 - if is_runai_obj_uri(model):

783 - object_storage_model = ObjectStorageModel(url=model)

784 - object_storage_model.pull_files(

785 - model, allow_pattern=["*.model", "*.py", "*.json"]

786 - )

787 - self.model_weights = model

788 - self.model = object_storage_model.dir

789 -

790 - # If tokenizer is same as model, download to same directory

791 - if model == tokenizer:

792 - object_storage_model.pull_files(

793 - model,

794 - ignore_pattern=[

795 - "*.pt",

796 - "*.safetensors",

797 - "*.bin",

798 - "*.tensors",

799 - "*.pth",

800 - ],

801 - )

802 - self.tokenizer = object_storage_model.dir

803 - return

804 -

805 - # Only download tokenizer if needed and not already handled

806 - if is_runai_obj_uri(tokenizer):

807 - object_storage_tokenizer = ObjectStorageModel(url=tokenizer)

808 - object_storage_tokenizer.pull_files(

809 - model,

810 - ignore_pattern=["*.pt", "*.safetensors", "*.bin", "*.tensors", "*.pth"],

811 - )

812 - self.tokenizer = object_storage_tokenizer.dir

720 + object_storage_model = ObjectStorageModel(url=model)

721 + object_storage_model.pull_files(

722 + model, allow_pattern=["*.model", "*.py", "*.json"]

723 + )

724 + self.model_weights = model

725 + self.model = object_storage_model.dir

726

727 def _get_encoder_config(self):

728 model = self.model

@@ -1712,30 +1625,38 @@ class ModelConfig:

1625 return dense_modules[-1]["out_features"]

1626 return self.get_hidden_size()

1627

1715 - def get_and_verify_max_len(self, max_model_len: int):

1628 + def recalculate_max_model_len(

1629 + self,

1630 + original_max_model_len: int | None,

1631 + *,

1632 + tokenizer: str | None = None,

1633 + tokenizer_revision: str | None = None,

1634 + ) -> None:

1635 # Consider max_model_len in tokenizer_config only when

1636 # pooling models use absolute position_embedding.

1637 + # NOTE: For simplicity we assume `args.model == args.tokenizer`

1638 + # since this is

1639 tokenizer_config = None

1640 if (

1641 self.runner_type == "pooling"

1642 and getattr(self.hf_config, "position_embedding_type", "") == "absolute"

1643 ):

1644 tokenizer_config = try_get_tokenizer_config(

1724 - self.tokenizer,

1645 + tokenizer or self.model,

1646 trust_remote_code=self.trust_remote_code,

1726 - revision=self.tokenizer_revision,

1647 + revision=tokenizer_revision or self.revision,

1648 )

1728 - max_model_len = _get_and_verify_max_len(

1649 +

1650 + self.max_model_len = _get_and_verify_max_len(

1651 hf_config=self.hf_text_config,

1652 tokenizer_config=tokenizer_config,

1731 - max_model_len=max_model_len,

1653 + max_model_len=original_max_model_len,

1654 disable_sliding_window=self.disable_sliding_window,

1655 sliding_window=self.get_sliding_window(),

1656 spec_target_max_model_len=self.spec_target_max_model_len,

1657 encoder_config=self.encoder_config,

1658 )

1737 - logger.info("Using max model len %s", max_model_len)

1738 - return max_model_len

1659 + logger.info("Using max model len %s", self.max_model_len)

1660

1661 @property

1662 def attn_type(self) -> AttnTypeStr:

vllm/config/multimodal.py

-4

@@ -79,10 +79,6 @@ class MultiModalConfig:

80 WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed.

81 Only enable this flag for trusted users!"""

82 - media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)

83 - """Additional args passed to process media inputs, keyed by modalities.

84 - For example, to set num_frames for video, set

85 - `--media-io-kwargs '{"video": {"num_frames": 40} }'`"""

82 mm_processor_kwargs: dict[str, object] | None = None

83 """Arguments to be forwarded to the model's processor for multi-modal data,

84 e.g., image processor. Overrides for the multi-modal processor obtained

vllm/config/renderer.py new

+109

@@ -0,0 +1,109 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from typing import Any, Literal

4 +

5 +from pydantic import Field, SkipValidation

6 +from pydantic.dataclasses import dataclass

7 +

8 +from vllm.config.model import ModelConfig

9 +from vllm.config.utils import config

10 +from vllm.transformers_utils.gguf_utils import is_gguf

11 +from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri

12 +from vllm.transformers_utils.utils import maybe_model_redirect

13 +

14 +TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32"]

15 +

16 +

17 +@config

18 +@dataclass

19 +class RendererConfig:

20 + """Configuration for the renderer."""

21 +

22 + # NOTE: In reality, this is a required argument.

23 + # We provide a dummy default value here to generate the CLI args.

24 + model_config: SkipValidation[ModelConfig] = None # type: ignore

25 + """Provides model context to the renderer."""

26 +

27 + tokenizer: str = ""

28 + """Name or path of the Hugging Face tokenizer to use. If unspecified, model

29 + name or path will be used."""

30 + tokenizer_mode: TokenizerMode | str = "auto"

31 + """Tokenizer mode:\n

32 + - "auto" will use the tokenizer from `mistral_common` for Mistral models

33 + if available, otherwise it will use the "hf" tokenizer.\n

34 + - "hf" will use the fast tokenizer if available.\n

35 + - "slow" will always use the slow tokenizer.\n

36 + - "mistral" will always use the tokenizer from `mistral_common`.\n

37 + - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n

38 + - Other custom values can be supported via plugins."""

39 + tokenizer_revision: str | None = None

40 + """The specific revision to use for the tokenizer on the Hugging Face Hub.

41 + It can be a branch name, a tag name, or a commit id. If unspecified, will

42 + use the default version."""

43 + skip_tokenizer_init: bool = False

44 + """Skip initialization of tokenizer and detokenizer. Expects valid

45 + `prompt_token_ids` and `None` for prompt from the input. The generated

46 + output will contain token ids."""

47 +

48 + io_processor_plugin: str | None = None

49 + """IOProcessor plugin name to load at model startup."""

50 +

51 + media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)

52 + """Additional args passed to process media inputs, keyed by modalities.

53 + For example, to set num_frames for video, set

54 + `--media-io-kwargs '{"video": {"num_frames": 40} }'`"""

55 + allowed_local_media_path: str = ""

56 + """Allowing API requests to read local images or videos from directories

57 + specified by the server file system. This is a security risk. Should only

58 + be enabled in trusted environments."""

59 + allowed_media_domains: list[str] | None = None

60 + """If set, only media URLs that belong to this domain can be used for

61 + multi-modal inputs. """

62 +

63 + @property

64 + def trust_remote_code(self) -> bool:

65 + return self.model_config.trust_remote_code

66 +

67 + def __post_init__(self) -> None:

68 + model_config = self.model_config

69 +

70 + # The tokenizer is consistent with the model by default.

71 + if not self.tokenizer:

72 + self.tokenizer = (

73 + ModelConfig.model

74 + if model_config is None

75 + else model_config.original_model

76 + )

77 + if not self.tokenizer_revision:

78 + self.tokenizer_revision = (

79 + ModelConfig.revision if model_config is None else model_config.revision

80 + )

81 +

82 + self.original_tokenizer = self.tokenizer

83 + self.tokenizer = maybe_model_redirect(self.original_tokenizer)

84 + self.maybe_pull_tokenizer_for_runai(self.tokenizer)

85 +

86 + # Multimodal GGUF models must use original repo for mm processing

87 + is_multimodal_model = (

88 + ModelConfig.is_multimodal_model

89 + if model_config is None

90 + else model_config.is_multimodal_model

91 + )

92 + if is_gguf(self.tokenizer) and is_multimodal_model:

93 + raise ValueError(

94 + "Loading a multimodal GGUF model needs to use original "

95 + "tokenizer. Please specify the unquantized hf model's "

96 + "repo name or path using the --tokenizer argument."

97 + )

98 +

99 + def maybe_pull_tokenizer_for_runai(self, tokenizer: str) -> None:

100 + """Pull tokenizer from Object Storage to temporary directory when needed."""

101 + if not is_runai_obj_uri(tokenizer):

102 + return

103 +

104 + object_storage_tokenizer = ObjectStorageModel(url=tokenizer)

105 + object_storage_tokenizer.pull_files(

106 + tokenizer,

107 + ignore_pattern=["*.pt", "*.safetensors", "*.bin", "*.tensors", "*.pth"],

108 + )

109 + self.tokenizer = object_storage_tokenizer.dir

vllm/config/speculative.py

-5

@@ -322,16 +322,11 @@ class SpeculativeConfig:

322 self.draft_model_config = ModelConfig(

323 model=self.model,

324 runner="draft",

325 - tokenizer=self.target_model_config.tokenizer,

326 - tokenizer_mode=self.target_model_config.tokenizer_mode,

325 trust_remote_code=self.target_model_config.trust_remote_code,

328 - allowed_local_media_path=self.target_model_config.allowed_local_media_path,

329 - allowed_media_domains=self.target_model_config.allowed_media_domains,

326 dtype=self.target_model_config.dtype,

327 seed=self.target_model_config.seed,

328 revision=self.revision,

329 code_revision=self.code_revision,

334 - tokenizer_revision=self.target_model_config.tokenizer_revision,

330 spec_target_max_model_len=self.target_model_config.max_model_len,

331 quantization=self.quantization,

332 enforce_eager=self.target_model_config.enforce_eager,

vllm/config/vllm.py

+15 -10

@@ -39,6 +39,7 @@ from .lora import LoRAConfig

39 from .model import ModelConfig

40 from .observability import ObservabilityConfig

41 from .parallel import ParallelConfig

42 +from .renderer import RendererConfig

43 from .scheduler import SchedulerConfig

44 from .speculative import SpeculativeConfig

45 from .structured_outputs import StructuredOutputsConfig

@@ -181,6 +182,8 @@ class VllmConfig:

182 # try to download a model

183 model_config: ModelConfig = Field(default=None)

184 """Model configuration."""

185 + renderer_config: RendererConfig = Field(default_factory=RendererConfig)

186 + """Renderer configuration."""

187 cache_config: CacheConfig = Field(default_factory=CacheConfig)

188 """Cache configuration."""

189 parallel_config: ParallelConfig = Field(default_factory=ParallelConfig)

@@ -741,7 +744,7 @@ class VllmConfig:

744 from vllm.multimodal import MULTIMODAL_REGISTRY

745

746 self.scheduler_config.max_num_encoder_input_tokens = (

744 - MULTIMODAL_REGISTRY.get_encdec_max_encoder_len(self.model_config)

747 + MULTIMODAL_REGISTRY.get_encdec_max_encoder_len(self.renderer_config)

748 )

749 logger.debug(

750 "Encoder-decoder model detected: setting "

@@ -1186,11 +1189,13 @@ class VllmConfig:

1189 computed_compile_ranges_split_points

1190 )

1191

1189 - def recalculate_max_model_len(self, max_model_len: int):

1190 - # Can only be called in try_verify_and_update_config

1191 - model_config = self.model_config

1192 - max_model_len = model_config.get_and_verify_max_len(max_model_len)

1193 - self.model_config.max_model_len = max_model_len

1192 + def recalculate_max_model_len(self, original_max_model_len: int | None) -> None:

1193 + # Can only be called during try_verify_and_update_config

1194 + self.model_config.recalculate_max_model_len(

1195 + original_max_model_len,

1196 + tokenizer=self.renderer_config.tokenizer,

1197 + tokenizer_revision=self.renderer_config.tokenizer_revision,

1198 + )

1199

1200 def try_verify_and_update_config(self):

1201 if self.model_config is None:

@@ -1264,11 +1269,11 @@ class VllmConfig:

1269 return (

1270 f"model={self.model_config.model!r}, "

1271 f"speculative_config={self.speculative_config!r}, "

1267 - f"tokenizer={self.model_config.tokenizer!r}, "

1268 - f"skip_tokenizer_init={self.model_config.skip_tokenizer_init}, "

1269 - f"tokenizer_mode={self.model_config.tokenizer_mode}, "

1272 + f"tokenizer={self.renderer_config.tokenizer!r}, "

1273 + f"skip_tokenizer_init={self.renderer_config.skip_tokenizer_init}, "

1274 + f"tokenizer_mode={self.renderer_config.tokenizer_mode}, "

1275 f"revision={self.model_config.revision}, "

1271 - f"tokenizer_revision={self.model_config.tokenizer_revision}, "

1276 + f"tokenizer_revision={self.renderer_config.tokenizer_revision}, "

1277 f"trust_remote_code={self.model_config.trust_remote_code}, "

1278 f"dtype={self.model_config.dtype}, "

1279 f"max_seq_len={self.model_config.max_model_len}, "

vllm/engine/arg_utils.py

+59 -40

@@ -71,11 +71,11 @@ from vllm.config.model import (

71 ModelDType,

72 RunnerOption,

73 TaskOption,

74 - TokenizerMode,

74 )

75 from vllm.config.multimodal import MMCacheType, MMEncoderTPMode

76 from vllm.config.observability import DetailedTraceModules

77 from vllm.config.parallel import DistributedExecutorBackend, ExpertPlacementStrategy

78 +from vllm.config.renderer import RendererConfig, TokenizerMode

79 from vllm.config.scheduler import SchedulerPolicy

80 from vllm.config.utils import get_field

81 from vllm.config.vllm import OptimizationLevel

@@ -355,17 +355,12 @@ class EngineArgs:

355

356 model: str = ModelConfig.model

357 served_model_name: str | list[str] | None = ModelConfig.served_model_name

358 - tokenizer: str | None = ModelConfig.tokenizer

358 hf_config_path: str | None = ModelConfig.hf_config_path

359 runner: RunnerOption = ModelConfig.runner

360 convert: ConvertOption = ModelConfig.convert

361 task: TaskOption | None = ModelConfig.task

363 - skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init

362 enable_prompt_embeds: bool = ModelConfig.enable_prompt_embeds

365 - tokenizer_mode: TokenizerMode | str = ModelConfig.tokenizer_mode

363 trust_remote_code: bool = ModelConfig.trust_remote_code

367 - allowed_local_media_path: str = ModelConfig.allowed_local_media_path

368 - allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains

364 download_dir: str | None = LoadConfig.download_dir

365 safetensors_load_strategy: str = LoadConfig.safetensors_load_strategy

366 load_format: str | LoadFormats = LoadConfig.load_format

@@ -449,7 +444,6 @@ class EngineArgs:

444 code_revision: str | None = ModelConfig.code_revision

445 hf_token: bool | str | None = ModelConfig.hf_token

446 hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")

452 - tokenizer_revision: str | None = ModelConfig.tokenizer_revision

447 quantization: QuantizationMethods | None = ModelConfig.quantization

448 enforce_eager: bool = ModelConfig.enforce_eager

449 disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce

@@ -458,9 +452,6 @@ class EngineArgs:

452 )

453 enable_mm_embeds: bool = MultiModalConfig.enable_mm_embeds

454 interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings

461 - media_io_kwargs: dict[str, dict[str, Any]] = get_field(

462 - MultiModalConfig, "media_io_kwargs"

463 - )

455 mm_processor_kwargs: dict[str, Any] | None = MultiModalConfig.mm_processor_kwargs

456 disable_mm_preprocessor_cache: bool = False # DEPRECATED

457 mm_processor_cache_gb: float = MultiModalConfig.mm_processor_cache_gb

@@ -474,9 +465,19 @@ class EngineArgs:

465 mm_encoder_attn_backend: AttentionBackendEnum | str | None = (

466 MultiModalConfig.mm_encoder_attn_backend

467 )

477 - io_processor_plugin: str | None = None

468 skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling

469 video_pruning_rate: float = MultiModalConfig.video_pruning_rate

470 + # Renderer fields

471 + tokenizer: str | None = None

472 + tokenizer_mode: TokenizerMode | str = RendererConfig.tokenizer_mode

473 + tokenizer_revision: str | None = RendererConfig.tokenizer_revision

474 + skip_tokenizer_init: bool = RendererConfig.skip_tokenizer_init

475 + io_processor_plugin: str | None = None

476 + media_io_kwargs: dict[str, dict[str, Any]] = get_field(

477 + RendererConfig, "media_io_kwargs"

478 + )

479 + allowed_local_media_path: str = RendererConfig.allowed_local_media_path

480 + allowed_media_domains: list[str] | None = RendererConfig.allowed_media_domains

481 # LoRA fields

482 enable_lora: bool = False

483 max_loras: int = LoRAConfig.max_loras

@@ -627,25 +628,14 @@ class EngineArgs:

628 model_group.add_argument("--runner", **model_kwargs["runner"])

629 model_group.add_argument("--convert", **model_kwargs["convert"])

630 model_group.add_argument("--task", **model_kwargs["task"], deprecated=True)

630 - model_group.add_argument("--tokenizer", **model_kwargs["tokenizer"])

631 - model_group.add_argument("--tokenizer-mode", **model_kwargs["tokenizer_mode"])

631 model_group.add_argument(

632 "--trust-remote-code", **model_kwargs["trust_remote_code"]

633 )

634 model_group.add_argument("--dtype", **model_kwargs["dtype"])

635 model_group.add_argument("--seed", **model_kwargs["seed"])

636 model_group.add_argument("--hf-config-path", **model_kwargs["hf_config_path"])

638 - model_group.add_argument(

639 - "--allowed-local-media-path", **model_kwargs["allowed_local_media_path"]

640 - )

641 - model_group.add_argument(

642 - "--allowed-media-domains", **model_kwargs["allowed_media_domains"]

643 - )

637 model_group.add_argument("--revision", **model_kwargs["revision"])

638 model_group.add_argument("--code-revision", **model_kwargs["code_revision"])

646 - model_group.add_argument(

647 - "--tokenizer-revision", **model_kwargs["tokenizer_revision"]

648 - )

639 model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"])

640 model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"])

641 model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])

@@ -657,9 +647,6 @@ class EngineArgs:

647 model_group.add_argument(

648 "--disable-cascade-attn", **model_kwargs["disable_cascade_attn"]

649 )

660 - model_group.add_argument(

661 - "--skip-tokenizer-init", **model_kwargs["skip_tokenizer_init"]

662 - )

650 model_group.add_argument(

651 "--enable-prompt-embeds", **model_kwargs["enable_prompt_embeds"]

652 )

@@ -698,8 +685,34 @@ class EngineArgs:

685 model_group.add_argument(

686 "--logits-processors", **model_kwargs["logits_processors"]

687 )

701 - model_group.add_argument(

702 - "--io-processor-plugin", **model_kwargs["io_processor_plugin"]

688 +

689 + # Renderer arguments

690 + renderer_kwargs = get_kwargs(RendererConfig)

691 + renderer_group = parser.add_argument_group(

692 + title="RendererConfig",

693 + description=RendererConfig.__doc__,

694 + )

695 + renderer_group.add_argument("--tokenizer", **renderer_kwargs["tokenizer"])

696 + renderer_group.add_argument(

697 + "--tokenizer-mode", **renderer_kwargs["tokenizer_mode"]

698 + )

699 + renderer_group.add_argument(

700 + "--tokenizer-revision", **renderer_kwargs["tokenizer_revision"]

701 + )

702 + renderer_group.add_argument(

703 + "--skip-tokenizer-init", **renderer_kwargs["skip_tokenizer_init"]

704 + )

705 + renderer_group.add_argument(

706 + "--media-io-kwargs", **renderer_kwargs["media_io_kwargs"]

707 + )

708 + renderer_group.add_argument(

709 + "--allowed-local-media-path", **renderer_kwargs["allowed_local_media_path"]

710 + )

711 + renderer_group.add_argument(

712 + "--allowed-media-domains", **renderer_kwargs["allowed_media_domains"]

713 + )

714 + renderer_group.add_argument(

715 + "--io-processor-plugin", **renderer_kwargs["io_processor_plugin"]

716 )

717

718 # Model loading arguments

@@ -949,9 +962,6 @@ class EngineArgs:

962 multimodal_group.add_argument(

963 "--enable-mm-embeds", **multimodal_kwargs["enable_mm_embeds"]

964 )

952 - multimodal_group.add_argument(

953 - "--media-io-kwargs", **multimodal_kwargs["media_io_kwargs"]

954 - )

965 multimodal_group.add_argument(

966 "--mm-processor-kwargs", **multimodal_kwargs["mm_processor_kwargs"]

967 )

@@ -1255,18 +1265,13 @@ class EngineArgs:

1265 runner=self.runner,

1266 convert=self.convert,

1267 task=self.task,

1258 - tokenizer=self.tokenizer,

1259 - tokenizer_mode=self.tokenizer_mode,

1268 trust_remote_code=self.trust_remote_code,

1261 - allowed_local_media_path=self.allowed_local_media_path,

1262 - allowed_media_domains=self.allowed_media_domains,

1269 dtype=self.dtype,

1270 seed=self.seed,

1271 revision=self.revision,

1272 code_revision=self.code_revision,

1273 hf_token=self.hf_token,

1274 hf_overrides=self.hf_overrides,

1269 - tokenizer_revision=self.tokenizer_revision,

1275 max_model_len=self.max_model_len,

1276 quantization=self.quantization,

1277 enforce_eager=self.enforce_eager,

@@ -1274,13 +1279,11 @@ class EngineArgs:

1279 logprobs_mode=self.logprobs_mode,

1280 disable_sliding_window=self.disable_sliding_window,

1281 disable_cascade_attn=self.disable_cascade_attn,

1277 - skip_tokenizer_init=self.skip_tokenizer_init,

1282 enable_prompt_embeds=self.enable_prompt_embeds,

1283 served_model_name=self.served_model_name,

1284 limit_mm_per_prompt=self.limit_mm_per_prompt,

1285 enable_mm_embeds=self.enable_mm_embeds,

1286 interleave_mm_strings=self.interleave_mm_strings,

1283 - media_io_kwargs=self.media_io_kwargs,

1287 skip_mm_profiling=self.skip_mm_profiling,

1288 config_format=self.config_format,

1289 mm_processor_kwargs=self.mm_processor_kwargs,

@@ -1298,7 +1301,6 @@ class EngineArgs:

1301 override_attention_dtype=self.override_attention_dtype,

1302 logits_processors=self.logits_processors,

1303 video_pruning_rate=self.video_pruning_rate,

1301 - io_processor_plugin=self.io_processor_plugin,

1304 )

1305

1306 def validate_tensorizer_args(self):

@@ -1394,9 +1396,25 @@ class EngineArgs:

1396 )

1397

1398 model_config = self.create_model_config()

1397 - self.model = model_config.model

1398 - self.tokenizer = model_config.tokenizer

1399 + renderer_config = RendererConfig(

1400 + model_config=model_config,

1401 + tokenizer=self.tokenizer or "",

1402 + tokenizer_mode=self.tokenizer_mode,

1403 + tokenizer_revision=self.tokenizer_revision,

1404 + skip_tokenizer_init=self.skip_tokenizer_init,

1405 + io_processor_plugin=self.io_processor_plugin,

1406 + media_io_kwargs=self.media_io_kwargs,

1407 + allowed_local_media_path=self.allowed_local_media_path,

1408 + allowed_media_domains=self.allowed_media_domains,

1409 + )

1410

1411 + model_config.recalculate_max_model_len(

1412 + model_config.original_max_model_len,

1413 + tokenizer=renderer_config.tokenizer,

1414 + tokenizer_revision=renderer_config.tokenizer_revision,

1415 + )

1416 +

1417 + self.model = model_config.model

1418 self._check_feature_supported(model_config)

1419 self._set_default_chunked_prefill_and_prefix_caching_args(model_config)

1420 self._set_default_max_num_seqs_and_batched_tokens_args(

@@ -1768,6 +1786,7 @@ class EngineArgs:

1786 )

1787 config = VllmConfig(

1788 model_config=model_config,

1789 + renderer_config=renderer_config,

1790 cache_config=cache_config,

1791 parallel_config=parallel_config,

1792 scheduler_config=scheduler_config,

vllm/engine/protocol.py

+2 -1

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod

5 from collections.abc import AsyncGenerator, Iterable, Mapping

6 from typing import Any

8 -from vllm.config import ModelConfig, VllmConfig

8 +from vllm.config import ModelConfig, RendererConfig, VllmConfig

9 from vllm.inputs.data import PromptType

10 from vllm.lora.request import LoRARequest

11 from vllm.outputs import PoolingRequestOutput, RequestOutput

@@ -22,6 +22,7 @@ class EngineClient(ABC):

22 """Protocol class for Clients to Engine"""

24 vllm_config: VllmConfig

25 + renderer_config: RendererConfig

26 model_config: ModelConfig

27 input_processor: InputProcessor

28 io_processor: IOProcessor | None

vllm/entrypoints/chat_utils.py

+45 -34

@@ -44,7 +44,7 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, Processor

44 from typing_extensions import Required, TypedDict

46 from vllm import envs

47 -from vllm.config import ModelConfig

47 +from vllm.config import ModelConfig, RendererConfig

48 from vllm.logger import init_logger

49 from vllm.model_executor.models import SupportsMultiModal

50 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict

@@ -452,9 +452,10 @@ This is needed because `lru_cache` does not cache when an exception happens.

452

453 def _try_get_processor_chat_template(

454 tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,

455 - model_config: ModelConfig,

455 + *,

456 + trust_remote_code: bool,

457 ) -> str | None:

457 - cache_key = (tokenizer.name_or_path, model_config.trust_remote_code)

458 + cache_key = (tokenizer.name_or_path, trust_remote_code)

459 if cache_key in _PROCESSOR_CHAT_TEMPLATES:

460 return _PROCESSOR_CHAT_TEMPLATES[cache_key]

461

@@ -466,7 +467,7 @@ def _try_get_processor_chat_template(

467 PreTrainedTokenizerFast,

468 ProcessorMixin,

469 ),

469 - trust_remote_code=model_config.trust_remote_code,

470 + trust_remote_code=trust_remote_code,

471 )

472 if (

473 isinstance(processor, ProcessorMixin)

@@ -499,7 +500,10 @@ def resolve_hf_chat_template(

500

501 # 2nd priority: AutoProcessor chat template, unless tool calling is enabled

502 if tools is None:

502 - chat_template = _try_get_processor_chat_template(tokenizer, model_config)

503 + chat_template = _try_get_processor_chat_template(

504 + tokenizer,

505 + trust_remote_code=model_config.trust_remote_code,

506 + )

507 if chat_template is not None:

508 return chat_template

509

@@ -513,10 +517,10 @@ def resolve_hf_chat_template(

517 exc_info=True,

518 )

519

516 - # 4th priority: Predefined fallbacks

520 + # 4th priority: Predefined fallbacks]

521 path = get_chat_template_fallback_path(

522 model_type=model_config.hf_config.model_type,

519 - tokenizer_name_or_path=model_config.tokenizer,

523 + tokenizer_name_or_path=tokenizer.name_or_path,

524 )

525 if path is not None:

526 logger.info_once(

@@ -538,14 +542,14 @@ def _resolve_chat_template_content_format(

542 tools: list[dict[str, Any]] | None,

543 tokenizer: TokenizerLike | None,

544 *,

541 - model_config: ModelConfig,

545 + renderer_config: RendererConfig,

546 ) -> _ChatTemplateContentFormat:

547 if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):

548 hf_chat_template = resolve_hf_chat_template(

549 tokenizer,

550 chat_template=chat_template,

551 tools=tools,

548 - model_config=model_config,

552 + model_config=renderer_config.model_config,

553 )

554 else:

555 hf_chat_template = None

@@ -595,7 +599,7 @@ def resolve_chat_template_content_format(

599 given_format: ChatTemplateContentFormatOption,

600 tokenizer: TokenizerLike | None,

601 *,

598 - model_config: ModelConfig,

602 + renderer_config: RendererConfig,

603 ) -> _ChatTemplateContentFormat:

604 if given_format != "auto":

605 return given_format

@@ -604,7 +608,7 @@ def resolve_chat_template_content_format(

608 chat_template,

609 tools,

610 tokenizer,

607 - model_config=model_config,

611 + renderer_config=renderer_config,

612 )

613

614 _log_chat_template_content_format(

@@ -627,32 +631,32 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

631 maximum per prompt.

632 """

633

630 - def __init__(self, model_config: ModelConfig):

634 + def __init__(self, renderer_config: RendererConfig):

635 super().__init__()

636

633 - self._model_config = model_config

637 + self._renderer_config = renderer_config

638

639 self._items_by_modality = defaultdict[str, list[_T | None]](list)

640 self._uuids_by_modality = defaultdict[str, list[str | None]](list)

641

642 @property

639 - def model_config(self) -> ModelConfig:

640 - return self._model_config

643 + def renderer_config(self) -> RendererConfig:

644 + return self._renderer_config

645

646 @cached_property

647 def model_cls(self) -> type[SupportsMultiModal]:

648 from vllm.model_executor.model_loader import get_model_cls

649

646 - model_cls = get_model_cls(self.model_config)

650 + model_cls = get_model_cls(self.renderer_config.model_config)

651 return cast(type[SupportsMultiModal], model_cls)

652

653 @property

654 def allowed_local_media_path(self):

651 - return self._model_config.allowed_local_media_path

655 + return self._renderer_config.allowed_local_media_path

656

657 @property

658 def allowed_media_domains(self):

655 - return self._model_config.allowed_media_domains

659 + return self._renderer_config.allowed_media_domains

660

661 @property

662 def mm_registry(self):

@@ -660,7 +664,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

664

665 @cached_property

666 def mm_processor(self):

663 - return self.mm_registry.create_processor(self.model_config)

667 + return self.mm_registry.create_processor(self.renderer_config)

668

669 def add(

670 self,

@@ -851,19 +855,20 @@ class MultiModalContentParser(BaseMultiModalContentParser):

855 super().__init__()

856

857 self._tracker = tracker

854 - multimodal_config = self._tracker.model_config.multimodal_config

855 - media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)

856 -

858 self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(

859 envs.VLLM_MEDIA_CONNECTOR,

859 - media_io_kwargs=media_io_kwargs,

860 + media_io_kwargs=self.renderer_config.media_io_kwargs,

861 allowed_local_media_path=tracker.allowed_local_media_path,

862 allowed_media_domains=tracker.allowed_media_domains,

863 )

864

865 + @property

866 + def renderer_config(self) -> RendererConfig:

867 + return self._tracker.renderer_config

868 +

869 @property

870 def model_config(self) -> ModelConfig:

866 - return self._tracker.model_config

871 + return self.renderer_config.model_config

872

873 def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:

874 image = self._connector.fetch_image(image_url) if image_url else None

@@ -963,18 +968,20 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

968 super().__init__()

969

970 self._tracker = tracker

966 - multimodal_config = self._tracker.model_config.multimodal_config

967 - media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)

971 self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(

972 envs.VLLM_MEDIA_CONNECTOR,

970 - media_io_kwargs=media_io_kwargs,

973 + media_io_kwargs=self.renderer_config.media_io_kwargs,

974 allowed_local_media_path=tracker.allowed_local_media_path,

975 allowed_media_domains=tracker.allowed_media_domains,

976 )

977

978 + @property

979 + def renderer_config(self) -> RendererConfig:

980 + return self._tracker.renderer_config

981 +

982 @property

983 def model_config(self) -> ModelConfig:

977 - return self._tracker.model_config

984 + return self.renderer_config.model_config

985

986 def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:

987 image_coro = self._connector.fetch_image_async(image_url) if image_url else None

@@ -1604,15 +1611,17 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None:

1611

1612 def parse_chat_messages(

1613 messages: list[ChatCompletionMessageParam],

1607 - model_config: ModelConfig,

1614 + renderer_config: RendererConfig,

1615 content_format: _ChatTemplateContentFormat,

1616 ) -> tuple[

1617 list[ConversationMessage],

1618 MultiModalDataDict | None,

1619 MultiModalUUIDDict | None,

1620 ]:

1621 + model_config = renderer_config.model_config

1622 +

1623 conversation: list[ConversationMessage] = []

1615 - mm_tracker = MultiModalItemTracker(model_config)

1624 + mm_tracker = MultiModalItemTracker(renderer_config)

1625

1626 for msg in messages:

1627 sub_messages = _parse_chat_message_content(

@@ -1635,15 +1644,17 @@ def parse_chat_messages(

1644

1645 def parse_chat_messages_futures(

1646 messages: list[ChatCompletionMessageParam],

1638 - model_config: ModelConfig,

1647 + renderer_config: RendererConfig,

1648 content_format: _ChatTemplateContentFormat,

1649 ) -> tuple[

1650 list[ConversationMessage],

1651 Awaitable[MultiModalDataDict | None],

1652 MultiModalUUIDDict | None,

1653 ]:

1654 + model_config = renderer_config.model_config

1655 +

1656 conversation: list[ConversationMessage] = []

1646 - mm_tracker = AsyncMultiModalItemTracker(model_config)

1657 + mm_tracker = AsyncMultiModalItemTracker(renderer_config)

1658

1659 for msg in messages:

1660 sub_messages = _parse_chat_message_content(

@@ -1748,14 +1759,14 @@ def apply_hf_chat_template(

1759 chat_template: str | None,

1760 tools: list[dict[str, Any]] | None,

1761 *,

1751 - model_config: ModelConfig,

1762 + renderer_config: RendererConfig,

1763 **kwargs: Any,

1764 ) -> str:

1765 hf_chat_template = resolve_hf_chat_template(

1766 tokenizer,

1767 chat_template=chat_template,

1768 tools=tools,

1758 - model_config=model_config,

1769 + model_config=renderer_config.model_config,

1770 )

1771

1772 if hf_chat_template is None:

vllm/entrypoints/llm.py

+8 -6

@@ -29,8 +29,8 @@ from vllm.config.model import (

29 HfOverrides,

30 ModelDType,

31 RunnerOption,

32 - TokenizerMode,

32 )

33 +from vllm.config.renderer import TokenizerMode

34 from vllm.engine.arg_utils import EngineArgs

35 from vllm.entrypoints.chat_utils import (

36 ChatCompletionMessageParam,

@@ -343,6 +343,7 @@ class LLM:

343 logger.info("Supported tasks: %s", supported_tasks)

344 self.supported_tasks = supported_tasks

345

346 + self.renderer_config = self.llm_engine.renderer_config

347 self.model_config = self.llm_engine.model_config

348 self.input_processor = self.llm_engine.input_processor

349 self.io_processor = self.llm_engine.io_processor

@@ -808,13 +809,13 @@ class LLM:

809 list_of_messages = [cast(list[ChatCompletionMessageParam], messages)]

810

811 tokenizer = self.get_tokenizer()

811 - model_config = self.model_config

812 + renderer_config = self.renderer_config

813 resolved_content_format = resolve_chat_template_content_format(

814 chat_template,

815 tools,

816 chat_template_content_format,

817 tokenizer,

817 - model_config=model_config,

818 + renderer_config=renderer_config,

819 )

820

821 _chat_template_kwargs: dict[str, Any] = dict(

@@ -833,7 +834,7 @@ class LLM:

834 # the chat message parsing for it.

835 conversation, mm_data, mm_uuids = parse_chat_messages(

836 msgs,

836 - model_config,

837 + renderer_config,

838 content_format=resolved_content_format,

839 )

840

@@ -847,7 +848,7 @@ class LLM:

848 prompt_str = apply_hf_chat_template(

849 tokenizer=tokenizer,

850 conversation=conversation,

850 - model_config=model_config,

851 + renderer_config=renderer_config,

852 **_chat_template_kwargs,

853 )

854 # Special tokens are already included in chat templates so

@@ -1290,6 +1291,7 @@ class LLM:

1291 lora_request: list[LoRARequest] | LoRARequest | None = None,

1292 tokenization_kwargs: dict[str, Any] | None = None,

1293 ) -> list[ScoringRequestOutput]:

1294 + renderer_config = self.renderer_config

1295 model_config = self.model_config

1296

1297 if isinstance(tokenizer, MistralTokenizer):

@@ -1317,7 +1319,7 @@ class LLM:

1319

1320 for q, d in input_pairs:

1321 _, engine_prompt = get_score_prompt(

1320 - model_config=model_config,

1322 + renderer_config=renderer_config,

1323 data_1=q,

1324 data_2=d,

1325 tokenizer=tokenizer,

vllm/entrypoints/openai/api_server.py

+1 -1

@@ -1099,7 +1099,7 @@ async def init_app_state(

1099 logger.info("Supported tasks: %s", supported_tasks)

1100

1101 resolved_chat_template = await process_chat_template(

1102 - args.chat_template, engine_client, vllm_config.model_config

1102 + args.chat_template, engine_client, vllm_config.renderer_config

1103 )

1104

1105 if args.tool_server == "demo":

vllm/entrypoints/openai/serving_completion.py

+1 -1

@@ -122,7 +122,7 @@ class OpenAIServingCompletion(OpenAIServing):

122 try:

123 lora_request = self._maybe_get_adapters(request)

124

125 - if self.model_config.skip_tokenizer_init:

125 + if self.renderer_config.skip_tokenizer_init:

126 tokenizer = None

127 else:

128 tokenizer = await self.engine_client.get_tokenizer()

vllm/entrypoints/openai/serving_engine.py

+6 -5

@@ -291,6 +291,7 @@ class OpenAIServing:

291

292 self.input_processor = self.models.input_processor

293 self.io_processor = self.models.io_processor

294 + self.renderer_config = self.models.renderer_config

295 self.model_config = self.models.model_config

296 self.max_model_len = self.model_config.max_model_len

297

@@ -1100,18 +1101,18 @@ class OpenAIServing:

1101 Sequence[RequestPrompt],

1102 list[EngineTokensPrompt],

1103 ]:

1103 - model_config = self.model_config

1104 + renderer_config = self.renderer_config

1105

1106 resolved_content_format = resolve_chat_template_content_format(

1107 chat_template,

1108 tool_dicts,

1109 chat_template_content_format,

1110 tokenizer,

1110 - model_config=model_config,

1111 + renderer_config=renderer_config,

1112 )

1113 conversation, mm_data_future, mm_uuids = parse_chat_messages_futures(

1114 messages,

1114 - model_config,

1115 + renderer_config,

1116 content_format=resolved_content_format,

1117 )

1118

@@ -1138,14 +1139,14 @@ class OpenAIServing:

1139 request_prompt = tokenizer.apply_chat_template(

1140 conversation=conversation,

1141 messages=messages,

1141 - model_config=model_config,

1142 + model_config=renderer_config.model_config,

1143 **_chat_template_kwargs,

1144 )

1145 else:

1146 request_prompt = apply_hf_chat_template(

1147 tokenizer=tokenizer,

1148 conversation=conversation,

1148 - model_config=model_config,

1149 + renderer_config=renderer_config,

1150 **_chat_template_kwargs,

1151 )

1152

vllm/entrypoints/openai/serving_models.py

@@ -71,6 +71,7 @@ class OpenAIServingModels:

72 self.input_processor = self.engine_client.input_processor

73 self.io_processor = self.engine_client.io_processor

74 + self.renderer_config = self.engine_client.renderer_config

75 self.model_config = self.engine_client.model_config

76 self.max_model_len = self.model_config.max_model_len

vllm/entrypoints/openai/speech_to_text.py

+5 -5

@@ -91,7 +91,7 @@ class OpenAISpeechToText(OpenAIServing):

91 self.task_type = task_type

93 self.asr_config = self.model_cls.get_speech_to_text_config(

94 - self.model_config, task_type

94 + self.renderer_config, task_type

95 )

97 self.enable_force_include_usage = enable_force_include_usage

@@ -101,8 +101,8 @@ class OpenAISpeechToText(OpenAIServing):

101 self.tokenizer = cast(

102 PreTrainedTokenizerBase,

103 get_tokenizer(

104 - tokenizer_name=self.model_config.tokenizer,

105 - tokenizer_mode=self.model_config.tokenizer_mode,

104 + tokenizer_name=self.renderer_config.tokenizer,

105 + tokenizer_mode=self.renderer_config.tokenizer_mode,

106 ),

107 )

108

@@ -154,7 +154,7 @@ class OpenAISpeechToText(OpenAIServing):

154 prompt = self.model_cls.get_generation_prompt(

155 audio=chunk,

156 stt_config=self.asr_config,

157 - model_config=self.model_config,

157 + renderer_config=self.renderer_config,

158 language=language,

159 task_type=self.task_type,

160 request_prompt=request.prompt,

@@ -428,7 +428,7 @@ class OpenAISpeechToText(OpenAIServing):

428 if res.prompt_token_ids is not None:

429 num_prompt_tokens = len(res.prompt_token_ids)

430 if audio_tokens := self.model_cls.get_num_audio_tokens(

431 - audio_duration_s, self.asr_config, self.model_config

431 + audio_duration_s, self.asr_config, self.renderer_config

432 ):

433 num_prompt_tokens += audio_tokens

434

vllm/entrypoints/pooling/pooling/serving.py

+1 -1

@@ -94,7 +94,7 @@ class OpenAIServingPooling(OpenAIServing):

94 try:

95 lora_request = self._maybe_get_adapters(request)

97 - if self.model_config.skip_tokenizer_init:

97 + if self.renderer_config.skip_tokenizer_init:

98 tokenizer = None

99 else:

100 tokenizer = await self.engine_client.get_tokenizer()

vllm/entrypoints/pooling/score/serving.py

+1 -3

@@ -160,10 +160,8 @@ class ServingScores(OpenAIServing):

160 data_1: str | ScoreContentPartParam,

161 data_2: str | ScoreContentPartParam,

162 ) -> tuple[str, TokensPrompt]:

163 - model_config = self.model_config

164 -

163 full_prompt, engine_prompt = get_score_prompt(

166 - model_config=model_config,

164 + renderer_config=self.renderer_config,

165 data_1=data_1,

166 data_2=data_2,

167 tokenizer=tokenizer,

vllm/entrypoints/score_utils.py

+8 -5

@@ -5,7 +5,7 @@ from typing import Any, TypeAlias, cast

5 from torch.nn import CosineSimilarity

6 from typing_extensions import Required, TypedDict

8 -from vllm.config import ModelConfig

8 +from vllm.config import ModelConfig, RendererConfig

9 from vllm.entrypoints.chat_utils import (

10 BaseMultiModalItemTracker,

11 ChatCompletionContentPartImageEmbedsParam,

@@ -88,9 +88,9 @@ def _validate_score_input_lens(

88 def parse_score_data(

89 data_1: str | ScoreContentPartParam,

90 data_2: str | ScoreContentPartParam,

91 - model_config: ModelConfig,

91 + renderer_config: RendererConfig,

92 ) -> tuple[str, str, MultiModalDataDict | None]:

93 - mm_tracker = MultiModalItemTracker(model_config)

93 + mm_tracker = MultiModalItemTracker(renderer_config)

95 content_1 = _parse_score_content(data_1, mm_tracker)

96 content_2 = _parse_score_content(data_2, mm_tracker)

@@ -176,7 +176,7 @@ def post_process_tokens(

176

177

178 def get_score_prompt(

179 - model_config: ModelConfig,

179 + renderer_config: RendererConfig,

180 tokenizer: TokenizerLike,

181 tokenization_kwargs: dict[str, Any],

182 data_1: str | ScoreContentPartParam,

@@ -185,11 +185,14 @@ def get_score_prompt(

185 prompt_1, prompt_2, mm_data = parse_score_data(

186 data_1,

187 data_2,

188 - model_config,

188 + renderer_config,

189 )

190 +

191 from vllm.model_executor.model_loader import get_model_cls

192

193 + model_config = renderer_config.model_config

194 model = get_model_cls(model_config)

195 +

196 if supports_score_template(model):

197 full_prompt = apply_score_template(model_config, prompt_1, prompt_2)

198 prompt_inputs = tokenizer(full_prompt, **tokenization_kwargs)

vllm/entrypoints/utils.py

+4 -4

@@ -13,7 +13,7 @@ from fastapi import Request

13 from fastapi.responses import JSONResponse, StreamingResponse

14 from starlette.background import BackgroundTask, BackgroundTasks

16 -from vllm.config import ModelConfig

16 +from vllm.config import RendererConfig

17 from vllm.engine.arg_utils import EngineArgs

18 from vllm.engine.protocol import EngineClient

19 from vllm.entrypoints.chat_utils import (

@@ -288,7 +288,7 @@ def process_lora_modules(

288 async def process_chat_template(

289 args_chat_template: Path | str | None,

290 engine_client: EngineClient,

291 - model_config: ModelConfig,

291 + renderer_config: RendererConfig,

292 ) -> str | None:

293 resolved_chat_template = load_chat_template(args_chat_template)

294 if resolved_chat_template is not None:

@@ -305,7 +305,7 @@ async def process_chat_template(

305 tokenizer=tokenizer,

306 chat_template=None,

307 tools=None,

308 - model_config=model_config,

308 + model_config=renderer_config.model_config,

309 )

310

311 if hf_chat_template != resolved_chat_template:

@@ -314,6 +314,6 @@ async def process_chat_template(

314 "It is different from official chat template '%s'. "

315 "This discrepancy may lead to performance degradation.",

316 resolved_chat_template,

317 - model_config.model,

317 + renderer_config.model_config.model,

318 )

319 return resolved_chat_template

vllm/inputs/preprocess.py

+5 -4

@@ -6,7 +6,7 @@ from typing import Any, cast

7 from typing_extensions import assert_never

9 -from vllm.config import ModelConfig

9 +from vllm.config import RendererConfig

10 from vllm.logger import init_logger

11 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

12 from vllm.multimodal.cache import BaseMultiModalProcessorCache

@@ -45,14 +45,15 @@ logger = init_logger(__name__)

45 class InputPreprocessor:

46 def __init__(

47 self,

48 - model_config: ModelConfig,

48 + renderer_config: RendererConfig,

49 tokenizer: TokenizerLike | None,

50 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

51 mm_processor_cache: BaseMultiModalProcessorCache | None = None,

52 ) -> None:

53 super().__init__()

55 - self.model_config = model_config

55 + self.renderer_config = renderer_config

56 + self.model_config = renderer_config.model_config

57 self.tokenizer = tokenizer

58 self.mm_registry = mm_registry

59 self.mm_processor_cache = mm_processor_cache

@@ -231,7 +232,7 @@ class InputPreprocessor:

232 def _get_mm_processor(self) -> BaseMultiModalProcessor:

233 if not hasattr(self, "_mm_processor"):

234 self._mm_processor = self.mm_registry.create_processor(

234 - self.model_config,

235 + self.renderer_config,

236 tokenizer=self.tokenizer,

237 cache=self.mm_processor_cache,

238 )

vllm/model_executor/models/adapters.py

+10 -10

@@ -415,7 +415,7 @@ def load_weights_using_from_2_way_softmax(

415 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

416 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

417

418 - model_config = model.vllm_config.model_config

418 + renderer_config = model.vllm_config.renderer_config

419 quant_config = model.vllm_config.quant_config

420 text_config = model.config.get_text_config()

421

@@ -447,10 +447,10 @@ def load_weights_using_from_2_way_softmax(

447 from vllm.tokenizers import get_tokenizer

448

449 tokenizer = get_tokenizer(

450 - model_config.tokenizer,

451 - revision=model_config.tokenizer_revision,

452 - tokenizer_mode=model_config.tokenizer_mode,

453 - trust_remote_code=model_config.trust_remote_code,

450 + renderer_config.tokenizer,

451 + revision=renderer_config.tokenizer_revision,

452 + tokenizer_mode=renderer_config.tokenizer_mode,

453 + trust_remote_code=renderer_config.trust_remote_code,

454 )

455

456 false_id = tokenizer.convert_tokens_to_ids(tokens[0])

@@ -473,7 +473,7 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te

473 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

474 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

475

476 - model_config = model.vllm_config.model_config

476 + renderer_config = model.vllm_config.renderer_config

477 quant_config = model.vllm_config.quant_config

478 text_config = model.config.get_text_config()

479

@@ -501,10 +501,10 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te

501 from vllm.tokenizers import get_tokenizer

502

503 tokenizer = get_tokenizer(

504 - model_config.tokenizer,

505 - revision=model_config.tokenizer_revision,

506 - tokenizer_mode=model_config.tokenizer_mode,

507 - trust_remote_code=model_config.trust_remote_code,

504 + renderer_config.tokenizer,

505 + revision=renderer_config.tokenizer_revision,

506 + tokenizer_mode=renderer_config.tokenizer_mode,

507 + trust_remote_code=renderer_config.trust_remote_code,

508 )

509

510 token_ids = [tokenizer.convert_tokens_to_ids(t) for t in tokens]

vllm/model_executor/models/deepseek_ocr.py

+2 -2

@@ -377,8 +377,8 @@ class DeepseekOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

377 self.projector_config = config.projector_config

378 self.text_config = config.text_config

379

380 - model_config = vllm_config.model_config

381 - tokenizer = cached_tokenizer_from_config(model_config)

380 + renderer_config = vllm_config.renderer_config

381 + tokenizer = cached_tokenizer_from_config(renderer_config)

382 self.image_token_id = tokenizer.vocab[_IMAGE_TOKEN]

383

384 self.sam_model = build_sam_vit_b()

vllm/model_executor/models/deepseek_vl2.py

+2 -2

@@ -370,8 +370,8 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

370 self.projector_config = config.projector_config

371 self.text_config = config.text_config

372

373 - model_config = vllm_config.model_config

374 - tokenizer = cached_tokenizer_from_config(model_config)

373 + renderer_config = vllm_config.renderer_config

374 + tokenizer = cached_tokenizer_from_config(renderer_config)

375 self.image_token_id: int = tokenizer.vocab[_IMAGE_TOKEN]

376

377 self.vision = self._init_vision_module(

vllm/model_executor/models/gemma3n_mm.py

+5 -3

@@ -18,7 +18,7 @@ from transformers.models.gemma3n import (

18 )

19 from transformers.models.siglip import SiglipImageProcessorFast

21 -from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

21 +from vllm.config import RendererConfig, SpeechToTextConfig, VllmConfig

22 from vllm.config.multimodal import BaseDummyOptions

23 from vllm.inputs.data import PromptType

24 from vllm.logger import init_logger

@@ -760,7 +760,7 @@ class Gemma3nForConditionalGeneration(

760 cls,

761 audio: np.ndarray,

762 stt_config: SpeechToTextConfig,

763 - model_config: ModelConfig,

763 + renderer_config: RendererConfig,

764 language: Optional[str],

765 task_type: Literal["transcribe", "translate"],

766 request_prompt: str,

@@ -798,7 +798,9 @@ class Gemma3nForConditionalGeneration(

798

799 @classmethod

800 def get_speech_to_text_config(

801 - cls, model_config: ModelConfig, task_type: str

801 + cls,

802 + renderer_config: RendererConfig,

803 + task_type: str,

804 ) -> SpeechToTextConfig:

805 return SpeechToTextConfig(

806 # Let's set this to 30 as suggested in the docs for now, although

vllm/model_executor/models/granite_speech.py

+8 -6

@@ -34,7 +34,7 @@ import torch.nn.functional as F

34 from torch import nn

35 from transformers import BatchFeature, PretrainedConfig

37 -from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig

37 +from vllm.config import CacheConfig, RendererConfig, SpeechToTextConfig, VllmConfig

38 from vllm.config.multimodal import BaseDummyOptions

39 from vllm.inputs.data import PromptType

40 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

@@ -840,7 +840,7 @@ class GraniteSpeechForConditionalGeneration(

840 def get_generation_prompt(

841 cls,

842 audio: np.ndarray,

843 - model_config: ModelConfig,

843 + renderer_config: RendererConfig,

844 stt_config: SpeechToTextConfig,

845 language: str | None,

846 task_type: Literal["transcribe", "translate"],

@@ -861,7 +861,7 @@ class GraniteSpeechForConditionalGeneration(

861 else:

862 raise ValueError(f"Unsupported task type {task_type}")

863

864 - tokenizer = cached_tokenizer_from_config(model_config)

864 + tokenizer = cached_tokenizer_from_config(renderer_config)

865 chat = [dict(role="user", content=user_prompt)]

866 prompt = tokenizer.apply_chat_template(

867 chat,

@@ -882,10 +882,10 @@ class GraniteSpeechForConditionalGeneration(

882 cls,

883 audio_duration_s: float,

884 stt_config: SpeechToTextConfig,

885 - model_config: ModelConfig,

885 + renderer_config: RendererConfig,

886 ) -> int | None:

887 """Get the number of audio tokens for an audio duration in sec."""

888 - processor = cached_processor_from_config(model_config)

888 + processor = cached_processor_from_config(renderer_config)

889 hop_length = processor.audio_processor.melspec_kwargs["hop_length"]

890 proj_win_size = processor.audio_processor.projector_window_size

891 ds_rate = processor.audio_processor.projector_downsample_rate

@@ -903,7 +903,9 @@ class GraniteSpeechForConditionalGeneration(

903

904 @classmethod

905 def get_speech_to_text_config(

906 - cls, model_config: ModelConfig, task_type: str

906 + cls,

907 + renderer_config: RendererConfig,

908 + task_type: str,

909 ) -> SpeechToTextConfig:

910 """Get the stt config for this model."""

911 # Default settings are reasonable for this model and we don't currently

vllm/model_executor/models/gritlm.py

+7 -7

@@ -6,7 +6,7 @@ import numpy as np

6 import torch

7 import torch.nn as nn

9 -from vllm.config import ModelConfig, VllmConfig

9 +from vllm.config import RendererConfig, VllmConfig

10 from vllm.logger import init_logger

11 from vllm.model_executor.layers.pooler import (

12 DispatchPooler,

@@ -29,12 +29,12 @@ logger = init_logger(__name__)

29 class GritLMMeanPool(nn.Module):

30 """As `MeanPool`, but only includes non-instruction tokens."""

32 - def __init__(self, model_config: ModelConfig):

32 + def __init__(self, renderer_config: RendererConfig):

33 super().__init__()

35 - self.model_config = model_config

35 + self.renderer_config = renderer_config

37 - tokenizer = cached_tokenizer_from_config(self.model_config)

37 + tokenizer = cached_tokenizer_from_config(self.renderer_config)

39 # Collect the tokens needed for pattern matching.

40 # "▁<" is different from "_<". The former uses "▁" to indicate that

@@ -174,10 +174,10 @@ class GritLMMeanPool(nn.Module):

174

175

176 class GritLMPooler(Pooler):

177 - def __init__(self, model_config: ModelConfig):

177 + def __init__(self, renderer_config: RendererConfig):

178 super().__init__()

179

180 - self.pooling = GritLMMeanPool(model_config)

180 + self.pooling = GritLMMeanPool(renderer_config)

181 self.head = PoolerHead(PoolerNormalize())

182

183 def get_supported_tasks(self) -> Set[PoolingTask]:

@@ -238,6 +238,6 @@ class GritLM(LlamaForCausalLM):

238 self.pooler = DispatchPooler(

239 {

240 "token_embed": Pooler.for_token_embed(pooler_config),

241 - "embed": GritLMPooler(vllm_config.model_config),

241 + "embed": GritLMPooler(vllm_config.renderer_config),

242 }

243 )

vllm/model_executor/models/interfaces.py

+6 -4

@@ -19,7 +19,7 @@ from torch import Tensor

19 from transformers.models.whisper.tokenization_whisper import LANGUAGES

20 from typing_extensions import Self, TypeIs

22 -from vllm.config import ModelConfig, SpeechToTextConfig

22 +from vllm.config import RendererConfig, SpeechToTextConfig

23 from vllm.inputs import TokensPrompt

24 from vllm.inputs.data import PromptType

25 from vllm.logger import init_logger

@@ -887,7 +887,7 @@ class SupportsTranscription(Protocol):

887 cls,

888 audio: np.ndarray,

889 stt_config: SpeechToTextConfig,

890 - model_config: ModelConfig,

890 + renderer_config: RendererConfig,

891 language: str | None,

892 task_type: Literal["transcribe", "translate"],

893 request_prompt: str,

@@ -930,7 +930,9 @@ class SupportsTranscription(Protocol):

930

931 @classmethod

932 def get_speech_to_text_config(

933 - cls, model_config: ModelConfig, task_type: Literal["transcribe", "translate"]

933 + cls,

934 + renderer_config: RendererConfig,

935 + task_type: Literal["transcribe", "translate"],

936 ) -> SpeechToTextConfig:

937 """Get the speech to text config for the ASR model."""

938 ...

@@ -940,7 +942,7 @@ class SupportsTranscription(Protocol):

942 cls,

943 audio_duration_s: float,

944 stt_config: SpeechToTextConfig,

943 - model_config: ModelConfig,

945 + renderer_config: RendererConfig,

946 ) -> int | None:

947 """

948 Map from audio duration to number of audio tokens produced by the ASR

vllm/model_executor/models/interns1.py

+1 -1

@@ -182,7 +182,7 @@ class InternS1ProcessingInfo(BaseProcessingInfo):

182 def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:

183 hf_processor = self.ctx.get_hf_processor(InternVLProcessor, **kwargs)

184 hf_processor.video_processor = cached_video_processor_from_config(

185 - self.ctx.model_config,

185 + self.ctx.renderer_config,

186 processor_cls=InternVLVideoProcessor,

187 size=hf_processor.image_processor.size,

188 **kwargs,

vllm/model_executor/models/nano_nemotron_vl.py

+7 -6

@@ -1169,16 +1169,17 @@ class NemotronH_Nano_VL_V2(

1169 self.mlp1 = self.mlp1.to(self.language_model.config.dtype)

1170

1171 self.config = config

1172 - self.model_config = vllm_config.model_config

1172

1173 # Pre-tokenize special tokens for video processing

1174 # to avoid repeated tokenization

1176 - tokenizer = cached_tokenizer_from_config(vllm_config.model_config)

1177 - self._img_start_token_ids = tokenizer.encode(

1175 + self._tokenizer = cached_tokenizer_from_config(vllm_config.renderer_config)

1176 + self._img_start_token_ids = self._tokenizer.encode(

1177 IMG_START, add_special_tokens=False

1178 )

1180 - self._img_end_token_ids = tokenizer.encode(IMG_END, add_special_tokens=False)

1181 - self._img_context_token_ids = tokenizer.encode(

1179 + self._img_end_token_ids = self._tokenizer.encode(

1180 + IMG_END, add_special_tokens=False

1181 + )

1182 + self._img_context_token_ids = self._tokenizer.encode(

1183 IMG_CONTEXT, add_special_tokens=False

1184 )

1185

@@ -1364,7 +1365,7 @@ class NemotronH_Nano_VL_V2(

1365 input_embeds for the LLM.

1366 """

1367 device = video_embeddings.device

1367 - tokenizer = cached_tokenizer_from_config(self.model_config)

1368 + tokenizer = self._tokenizer

1369

1370 # Generate video replacement token IDs using get_video_repl

1371 # This tokenizes each frame separator independently, then uses pre-tokenized

vllm/model_executor/models/nemotron_vl.py

+1 -1

@@ -347,7 +347,7 @@ class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):

347

348 def get_image_processor(self, **kwargs: object):

349 return cached_image_processor_from_config(

350 - self.ctx.model_config,

350 + self.ctx.renderer_config,

351 **kwargs,

352 )

353

vllm/model_executor/models/pixtral.py

+1 -1

@@ -193,7 +193,7 @@ class PixtralProcessorAdapter:

193

194 class PixtralProcessingInfo(BaseProcessingInfo):

195 def get_tokenizer(self) -> MistralTokenizer:

196 - tokenizer = cached_tokenizer_from_config(self.ctx.model_config)

196 + tokenizer = cached_tokenizer_from_config(self.ctx.renderer_config)

197 if not isinstance(tokenizer, MistralTokenizer):

198 raise ValueError("This model requires `--tokenizer-mode mistral`")

199

vllm/model_executor/models/voxtral.py

+12 -10

@@ -20,7 +20,7 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioEncoder

20 from transformers import BatchFeature, TensorType, WhisperConfig

21 from transformers.tokenization_utils_base import TextInput

23 -from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

23 +from vllm.config import RendererConfig, SpeechToTextConfig, VllmConfig

24 from vllm.config.multimodal import BaseDummyOptions

25 from vllm.inputs.data import PromptType

26 from vllm.logger import init_logger

@@ -176,7 +176,7 @@ class VoxtralProcessorAdapter:

176

177 class VoxtralProcessingInfo(BaseProcessingInfo):

178 def get_tokenizer(self) -> MistralTokenizer:

179 - tokenizer = cached_tokenizer_from_config(self.ctx.model_config)

179 + tokenizer = cached_tokenizer_from_config(self.ctx.renderer_config)

180 if not isinstance(tokenizer, MistralTokenizer):

181 raise ValueError("This model requires `--tokenizer-mode mistral`")

182

@@ -339,7 +339,7 @@ class VoxtralForConditionalGeneration(

339

340 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

341 super().__init__()

342 - self.tokenizer = cached_tokenizer_from_config(vllm_config.model_config)

342 + self.tokenizer = cached_tokenizer_from_config(vllm_config.renderer_config)

343

344 # update quant config to so that ignored module and target module names

345 # match the vLLM model names

@@ -450,9 +450,11 @@ class VoxtralForConditionalGeneration(

450

451 @classmethod

452 def get_speech_to_text_config(

453 - cls, model_config: ModelConfig, task_type: str

453 + cls,

454 + renderer_config: RendererConfig,

455 + task_type: str,

456 ) -> SpeechToTextConfig:

455 - tokenizer = cached_tokenizer_from_config(model_config)

457 + tokenizer = cached_tokenizer_from_config(renderer_config)

458 audio_config = tokenizer.instruct.audio_encoder.audio_config

459 max_audio_clip_s = audio_config.chunk_length_s

460 sample_rate = audio_config.sampling_rate

@@ -468,17 +470,17 @@ class VoxtralForConditionalGeneration(

470 def get_generation_prompt(

471 cls,

472 audio: np.ndarray,

471 - model_config: ModelConfig,

473 + renderer_config: RendererConfig, # not needed here

474 stt_config: SpeechToTextConfig,

475 language: str | None,

476 task_type: Literal["transcribe", "translate"],

477 request_prompt: str,

478 to_language: str | None,

479 ) -> PromptType:

478 - tokenizer = cached_tokenizer_from_config(model_config)

480 + tokenizer = cached_tokenizer_from_config(renderer_config)

481 audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless

482 req = TranscriptionRequest(

481 - model=model_config.model,

483 + model=renderer_config.model_config.model,

484 audio=RawAudio.from_audio(audio),

485 language=language,

486 )

@@ -494,14 +496,14 @@ class VoxtralForConditionalGeneration(

496 cls,

497 audio_duration_s: float,

498 stt_config: SpeechToTextConfig,

497 - model_config: ModelConfig,

499 + renderer_config: RendererConfig,

500 ) -> int | None:

501 """

502 Map from audio duration to number of audio tokens produced by the ASR

503 model, without running a forward pass.

504 This is used for estimating the amount of processing for this audio.

505 """

504 - tokenizer = cached_tokenizer_from_config(model_config)

506 + tokenizer = cached_tokenizer_from_config(renderer_config)

507 adapter = VoxtralProcessorAdapter(tokenizer)

508 return adapter.get_num_audio_tokens(

509 int(audio_duration_s * stt_config.sample_rate)

vllm/model_executor/models/whisper.py

+8 -6

@@ -19,7 +19,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids

19 from vllm.attention.backends.abstract import AttentionType

20 from vllm.attention.layer import Attention, MultiHeadAttention

21 from vllm.attention.layers.cross_attention import CrossAttention

22 -from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig

22 +from vllm.config import CacheConfig, RendererConfig, SpeechToTextConfig, VllmConfig

23 from vllm.config.multimodal import BaseDummyOptions

24 from vllm.distributed import get_tensor_model_parallel_world_size

25 from vllm.inputs.data import PromptType

@@ -811,7 +811,7 @@ class WhisperForConditionalGeneration(

811 def get_generation_prompt(

812 cls,

813 audio: np.ndarray,

814 - model_config: ModelConfig, # not needed here

814 + renderer_config: RendererConfig, # not needed here

815 stt_config: SpeechToTextConfig,

816 language: str | None,

817 task_type: Literal["transcribe", "translate"],

@@ -847,9 +847,11 @@ class WhisperForConditionalGeneration(

847

848 @classmethod

849 def get_speech_to_text_config(

850 - cls, model_config: ModelConfig, task_type: str

850 + cls,

851 + renderer_config: RendererConfig,

852 + task_type: str,

853 ) -> SpeechToTextConfig:

852 - processor = cached_processor_from_config(model_config)

854 + processor = cached_processor_from_config(renderer_config)

855

856 return SpeechToTextConfig(

857 max_audio_clip_s=processor.feature_extractor.chunk_length,

@@ -861,9 +863,9 @@ class WhisperForConditionalGeneration(

863 cls,

864 audio_duration_s: float,

865 stt_config: SpeechToTextConfig,

864 - model_config: ModelConfig,

866 + renderer_config: RendererConfig,

867 ) -> int | None:

866 - processor = cached_processor_from_config(model_config)

868 + processor = cached_processor_from_config(renderer_config)

869 hop_length = processor.feature_extractor.hop_length

870 assert hop_length is not None

871 # NOTE(NickLucche) user can't pass encoder

vllm/multimodal/cache.py

+10 -12

@@ -31,7 +31,7 @@ from .inputs import (

31 )

33 if TYPE_CHECKING:

34 - from vllm.config import ModelConfig, VllmConfig

34 + from vllm.config import ModelConfig, RendererConfig, VllmConfig

36 from .processing import ResolvedPromptUpdate

37 from .registry import MultiModalRegistry

@@ -561,13 +561,13 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):

561

562

563 def _enable_processor_cache(

564 - model_config: "ModelConfig",

564 + renderer_config: "RendererConfig",

565 mm_registry: "MultiModalRegistry",

566 ) -> bool:

567 - if not mm_registry.supports_multimodal_inputs(model_config):

567 + if not mm_registry.supports_multimodal_inputs(renderer_config):

568 return False

569

570 - mm_config = model_config.get_multimodal_config()

570 + mm_config = renderer_config.model_config.get_multimodal_config()

571 return mm_config.mm_processor_cache_gb > 0

572

573

@@ -599,7 +599,7 @@ def processor_cache_from_config(

599 """Return a `BaseMultiModalProcessorCache`, if enabled."""

600 model_config = vllm_config.model_config

601

602 - if not _enable_processor_cache(model_config, mm_registry):

602 + if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):

603 return None

604

605 if not _enable_ipc_cache(vllm_config):

@@ -611,14 +611,14 @@ def processor_cache_from_config(

611

612

613 def processor_only_cache_from_config(

614 - model_config: "ModelConfig",

614 + renderer_config: "RendererConfig",

615 mm_registry: "MultiModalRegistry",

616 ):

617 """Return a `MultiModalProcessorOnlyCache`, if enabled."""

618 - if not _enable_processor_cache(model_config, mm_registry):

618 + if not _enable_processor_cache(renderer_config, mm_registry):

619 return None

620

621 - return MultiModalProcessorOnlyCache(model_config)

621 + return MultiModalProcessorOnlyCache(renderer_config.model_config)

622

623

624 class BaseMultiModalReceiverCache(

@@ -787,7 +787,7 @@ def engine_receiver_cache_from_config(

787 """

788 model_config = vllm_config.model_config

789

790 - if not _enable_processor_cache(model_config, mm_registry):

790 + if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):

791 return None

792

793 if not _enable_ipc_cache(vllm_config):

@@ -809,9 +809,7 @@ def worker_receiver_cache_from_config(

809 Return a `BaseMultiModalReceiverCache` only when IPC caching is enabled and

810 mm_processor_cache_type=="shm".

811 """

812 - model_config = vllm_config.model_config

813 -

814 - if not _enable_processor_cache(model_config, mm_registry):

812 + if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):

813 return None

814

815 if not _enable_ipc_cache(vllm_config):

vllm/multimodal/processing.py

+23 -5

@@ -23,7 +23,7 @@ import torch

23 from typing_extensions import TypeVar, assert_never

25 from vllm.logger import init_logger

26 -from vllm.tokenizers import TokenizerLike

26 +from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config

27 from vllm.transformers_utils.processor import cached_processor_from_config

28 from vllm.utils.collection_utils import flatten_2d_lists, full_groupby

29 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides

@@ -53,7 +53,7 @@ if TYPE_CHECKING:

53 from transformers.feature_extraction_utils import BatchFeature

54 from transformers.processing_utils import ProcessorMixin

56 - from vllm.config import ModelConfig

56 + from vllm.config import ModelConfig, RendererConfig

58 from .cache import BaseMultiModalProcessorCache

59 from .profiling import BaseDummyInputsBuilder

@@ -63,6 +63,7 @@ else:

63 ProcessorMixin = object

65 ModelConfig = object

66 + RendererConfig = object

68 BaseMultiModalProcessorCache = object

@@ -945,12 +946,29 @@ class InputProcessingContext:

946 modify the inputs.

947 """

948

948 - model_config: ModelConfig

949 - """The configuration of the model."""

949 + renderer_config: RendererConfig

950 + """The configuration of the renderer."""

951

952 tokenizer: TokenizerLike | None

953 """The tokenizer used to tokenize the inputs."""

954

955 + @classmethod

956 + def from_config(

957 + cls,

958 + renderer_config: RendererConfig,

959 + *,

960 + tokenizer: TokenizerLike | None = None,

961 + ):

962 + if tokenizer is None and not renderer_config.skip_tokenizer_init:

963 + tokenizer = cached_tokenizer_from_config(renderer_config)

964 +

965 + return cls(renderer_config, tokenizer)

966 +

967 + @property

968 + def model_config(self) -> ModelConfig:

969 + """The configuration of the model."""

970 + return self.renderer_config.model_config

971 +

972 def get_tokenizer(self) -> TokenizerLike:

973 if self.tokenizer is None:

974 raise ValueError(

@@ -1047,7 +1065,7 @@ class InputProcessingContext:

1065 typ = ProcessorMixin

1066

1067 return cached_processor_from_config(

1050 - self.model_config,

1068 + self.renderer_config,

1069 processor_cls=typ,

1070 tokenizer=self.tokenizer,

1071 **kwargs,

vllm/multimodal/registry.py

+30 -34

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast

7 from vllm.config.multimodal import BaseDummyOptions

8 from vllm.logger import init_logger

9 -from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config

9 +from vllm.tokenizers import TokenizerLike

11 from .cache import BaseMultiModalProcessorCache

12 from .processing import (

@@ -22,7 +22,7 @@ from .profiling import (

22 )

24 if TYPE_CHECKING:

25 - from vllm.config import ModelConfig

25 + from vllm.config import ModelConfig, RendererConfig

26 from vllm.model_executor.models.interfaces import SupportsMultiModal

28 logger = init_logger(__name__)

@@ -114,17 +114,18 @@ class MultiModalRegistry:

114

115 return mm_options if len(mm_options) > 0 else None

116

117 - def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:

117 + def supports_multimodal_inputs(self, renderer_config: "RendererConfig") -> bool:

118 """

119 Checks if the model supports multimodal inputs.

120 Returns True if the model is multimodal with any non-zero supported

121 modalities, otherwise returns False, effectively running in

122 text-only mode.

123 """

124 + model_config = renderer_config.model_config

125 if not model_config.is_multimodal_model:

126 return False

127

127 - info = self._create_processing_info(model_config, tokenizer=None)

128 + info = self._create_processing_info(renderer_config, tokenizer=None)

129 supported_modalities = info.get_supported_mm_limits()

130

131 mm_config = model_config.get_multimodal_config()

@@ -144,7 +145,7 @@ class MultiModalRegistry:

145

146 def get_max_tokens_per_item_by_modality(

147 self,

147 - model_config: "ModelConfig",

148 + renderer_config: "RendererConfig",

149 *,

150 cache: BaseMultiModalProcessorCache | None = None,

151 profiler_limits: Mapping[str, int] | None = None,

@@ -153,10 +154,11 @@ class MultiModalRegistry:

154 Get the maximum number of tokens per data item from each modality based

155 on underlying model configuration.

156 """

157 + model_config = renderer_config.model_config

158 if not model_config.is_multimodal_model:

159 return {}

160

159 - processor = self.create_processor(model_config, cache=cache)

161 + processor = self.create_processor(renderer_config, cache=cache)

162 profiler: MultiModalProfiler = MultiModalProfiler(processor)

163

164 seq_len = model_config.max_model_len

@@ -171,7 +173,7 @@ class MultiModalRegistry:

173

174 def get_mm_limits_per_prompt(

175 self,

174 - model_config: "ModelConfig",

176 + renderer_config: "RendererConfig",

177 *,

178 cache: BaseMultiModalProcessorCache | None = None,

179 ) -> Mapping[str, int]:

@@ -179,10 +181,11 @@ class MultiModalRegistry:

181 Get the maximum number of multi-modal input instances for each modality

182 that are allowed per prompt for a model class.

183 """

184 + model_config = renderer_config.model_config

185 if not model_config.is_multimodal_model:

186 return {}

187

185 - processor = self.create_processor(model_config, cache=cache)

188 + processor = self.create_processor(renderer_config, cache=cache)

189 profiler: MultiModalProfiler = MultiModalProfiler(processor)

190 return profiler.get_mm_limits()

191

@@ -228,30 +231,21 @@ class MultiModalRegistry:

231 assert hasattr(model_cls, "_processor_factory")

232 return cast("SupportsMultiModal", model_cls)

233

231 - def _create_processing_ctx(

232 - self,

233 - model_config: "ModelConfig",

234 - tokenizer: TokenizerLike | None = None,

235 - ) -> InputProcessingContext:

236 - if tokenizer is None and not model_config.skip_tokenizer_init:

237 - tokenizer = cached_tokenizer_from_config(model_config)

238 -

239 - return InputProcessingContext(model_config, tokenizer)

240 -

234 def _create_processing_info(

235 self,

243 - model_config: "ModelConfig",

236 + renderer_config: "RendererConfig",

237 *,

238 tokenizer: TokenizerLike | None = None,

239 ) -> BaseProcessingInfo:

247 - model_cls = self._get_model_cls(model_config)

240 + model_cls = self._get_model_cls(renderer_config.model_config)

241 factories = model_cls._processor_factory

249 - ctx = self._create_processing_ctx(model_config, tokenizer)

242 +

243 + ctx = InputProcessingContext.from_config(renderer_config, tokenizer=tokenizer)

244 return factories.info(ctx)

245

246 def create_processor(

247 self,

254 - model_config: "ModelConfig",

248 + renderer_config: "RendererConfig",

249 *,

250 tokenizer: TokenizerLike | None = None,

251 cache: BaseMultiModalProcessorCache | None = None,

@@ -259,19 +253,19 @@ class MultiModalRegistry:

253 """

254 Create a multi-modal processor for a specific model and tokenizer.

255 """

256 + model_config = renderer_config.model_config

257 if not model_config.is_multimodal_model:

258 raise ValueError(f"{model_config.model} is not a multimodal model")

259

260 model_cls = self._get_model_cls(model_config)

261 factories = model_cls._processor_factory

262

268 - ctx = self._create_processing_ctx(model_config, tokenizer)

269 -

263 + ctx = InputProcessingContext.from_config(renderer_config, tokenizer=tokenizer)

264 return factories.build_processor(ctx, cache=cache)

265

266 def get_decoder_dummy_data(

267 self,

274 - model_config: "ModelConfig",

268 + renderer_config: "RendererConfig",

269 seq_len: int,

270 mm_counts: Mapping[str, int] | None = None,

271 *,

@@ -280,15 +274,15 @@ class MultiModalRegistry:

274 """

275 Create dummy data for profiling the memory usage of a model.

276

283 - The model is identified by `model_config`.

277 + The model is identified by `renderer_config`.

278 """

285 - processor = self.create_processor(model_config, cache=cache)

279 + processor = self.create_processor(renderer_config, cache=cache)

280 profiler: MultiModalProfiler = MultiModalProfiler(processor)

281

282 # Extract configurable options from multimodal config.

283 # Only include modalities that use advanced option types so legacy

284 # count-only behavior remains unchanged.

291 - mm_options = self._extract_mm_options(model_config)

285 + mm_options = self._extract_mm_options(renderer_config.model_config)

286

287 dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts, mm_options)

288

@@ -304,7 +298,7 @@ class MultiModalRegistry:

298

299 def get_encoder_dummy_data(

300 self,

307 - model_config: "ModelConfig",

301 + renderer_config: "RendererConfig",

302 seq_len: int,

303 mm_counts: Mapping[str, int] | None = None,

304 *,

@@ -313,15 +307,15 @@ class MultiModalRegistry:

307 """

308 Create dummy data for profiling the memory usage of a model.

309

316 - The model is identified by `model_config`.

310 + The model is identified by `renderer_config`.

311 """

318 - processor = self.create_processor(model_config, cache=cache)

312 + processor = self.create_processor(renderer_config, cache=cache)

313 profiler: MultiModalProfiler = MultiModalProfiler(processor)

314

315 # Extract configurable options from multimodal config.

316 # Only include modalities that use advanced option types so legacy

317 # count-only behavior remains unchanged.

324 - mm_options = self._extract_mm_options(model_config)

318 + mm_options = self._extract_mm_options(renderer_config.model_config)

319

320 dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts, mm_options)

321

@@ -336,13 +330,15 @@ class MultiModalRegistry:

330

331 return dummy_data

332

339 - def get_encdec_max_encoder_len(self, model_config: "ModelConfig") -> int:

333 + def get_encdec_max_encoder_len(self, renderer_config: "RendererConfig") -> int:

334 """

335 Get the maximum length of the encoder input for encoder-decoder models.

336 """

337 + model_config = renderer_config.model_config

338 if not model_config.is_encoder_decoder:

339 return 0

345 - max_tokens = self.get_max_tokens_per_item_by_modality(model_config)

340 +

341 + max_tokens = self.get_max_tokens_per_item_by_modality(renderer_config)

342 if not max_tokens:

343 # TODO - this function assumes encoder-decoder models are

344 # multimodal. This will need to change when adding support for more

vllm/tokenizers/registry.py

+12 -12

@@ -24,7 +24,7 @@ from vllm.utils.import_utils import resolve_obj_by_qualname

24 from .protocol import TokenizerLike

26 if TYPE_CHECKING:

27 - from vllm.config import ModelConfig

27 + from vllm.config import RendererConfig

29 logger = init_logger(__name__)

@@ -205,18 +205,18 @@ def get_tokenizer(

205 cached_get_tokenizer = lru_cache(get_tokenizer)

206

207

208 -def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):

208 +def cached_tokenizer_from_config(renderer_config: "RendererConfig", **kwargs):

209 return cached_get_tokenizer(

210 - model_config.tokenizer,

211 - tokenizer_mode=model_config.tokenizer_mode,

212 - revision=model_config.tokenizer_revision,

213 - trust_remote_code=model_config.trust_remote_code,

210 + renderer_config.tokenizer,

211 + tokenizer_mode=renderer_config.tokenizer_mode,

212 + revision=renderer_config.tokenizer_revision,

213 + trust_remote_code=renderer_config.trust_remote_code,

214 **kwargs,

215 )

216

217

218 -def init_tokenizer_from_config(model_config: "ModelConfig"):

219 - runner_type = model_config.runner_type

218 +def init_tokenizer_from_config(renderer_config: "RendererConfig"):

219 + runner_type = renderer_config.model_config.runner_type

220 if runner_type == "generate" or runner_type == "draft":

221 truncation_side = "left"

222 elif runner_type == "pooling":

@@ -225,9 +225,9 @@ def init_tokenizer_from_config(model_config: "ModelConfig"):

225 assert_never(runner_type)

226

227 return get_tokenizer(

228 - model_config.tokenizer,

229 - tokenizer_mode=model_config.tokenizer_mode,

230 - trust_remote_code=model_config.trust_remote_code,

231 - revision=model_config.tokenizer_revision,

228 + renderer_config.tokenizer,

229 + tokenizer_mode=renderer_config.tokenizer_mode,

230 + trust_remote_code=renderer_config.trust_remote_code,

231 + revision=renderer_config.tokenizer_revision,

232 truncation_side=truncation_side,

233 )

vllm/transformers_utils/processor.py

+17 -11

@@ -23,7 +23,7 @@ from vllm.transformers_utils.utils import convert_model_repo_to_path

23 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides

25 if TYPE_CHECKING:

26 - from vllm.config import ModelConfig

26 + from vllm.config import ModelConfig, RendererConfig

28 _P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)

29 _V = TypeVar("_V", bound=BaseVideoProcessor, default=BaseVideoProcessor)

@@ -233,17 +233,18 @@ def cached_get_processor_without_dynamic_kwargs(

233

234

235 def cached_processor_from_config(

236 - model_config: "ModelConfig",

236 + renderer_config: "RendererConfig",

237 processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,

238 **kwargs: Any,

239 ) -> _P:

240 + model_config = renderer_config.model_config

241 if is_gguf(model_config.model):

241 - assert not is_gguf(model_config.tokenizer), (

242 + assert not is_gguf(renderer_config.tokenizer), (

243 "For multimodal GGUF models, the original tokenizer "

244 "should be used to correctly load processor."

245 )

245 - model = model_config.tokenizer

246 - revision = model_config.tokenizer_revision

246 + model = renderer_config.tokenizer

247 + revision = renderer_config.tokenizer_revision

248 else:

249 model = model_config.model

250 revision = model_config.revision

@@ -297,9 +298,11 @@ cached_get_feature_extractor = lru_cache(get_feature_extractor)

298

299

300 def cached_feature_extractor_from_config(

300 - model_config: "ModelConfig",

301 + renderer_config: "RendererConfig",

302 **kwargs: Any,

303 ):

304 + model_config = renderer_config.model_config

305 +

306 return cached_get_feature_extractor(

307 model_config.model,

308 revision=model_config.revision,

@@ -348,16 +351,17 @@ cached_get_image_processor = lru_cache(get_image_processor)

351

352

353 def cached_image_processor_from_config(

351 - model_config: "ModelConfig",

354 + renderer_config: "RendererConfig",

355 **kwargs: Any,

356 ):

357 + model_config = renderer_config.model_config

358 if is_gguf(model_config.model):

355 - assert not is_gguf(model_config.tokenizer), (

359 + assert not is_gguf(renderer_config.tokenizer), (

360 "For multimodal GGUF models, the original tokenizer "

361 "should be used to correctly load image processor."

362 )

359 - model = model_config.tokenizer

360 - revision = model_config.tokenizer_revision

363 + model = renderer_config.tokenizer

364 + revision = renderer_config.tokenizer_revision

365 else:

366 model = model_config.model

367 revision = model_config.revision

@@ -411,10 +415,12 @@ cached_get_video_processor = lru_cache(get_video_processor)

415

416

417 def cached_video_processor_from_config(

414 - model_config: "ModelConfig",

418 + renderer_config: "RendererConfig",

419 processor_cls: type[_V] | None = None,

420 **kwargs: Any,

421 ):

422 + model_config = renderer_config.model_config

423 +

424 return cached_get_video_processor(

425 model_config.model,

426 revision=model_config.revision,

vllm/v1/core/encoder_cache_manager.py

+4 -4

@@ -10,7 +10,7 @@ from vllm.multimodal import MultiModalRegistry

10 from vllm.v1.request import Request

12 if TYPE_CHECKING:

13 - from vllm.config import ModelConfig, SchedulerConfig

13 + from vllm.config import RendererConfig, SchedulerConfig

15 logger = init_logger(__name__)

@@ -250,7 +250,7 @@ class EncoderCacheManager:

250

251

252 def compute_encoder_budget(

253 - model_config: "ModelConfig",

253 + renderer_config: "RendererConfig",

254 scheduler_config: "SchedulerConfig",

255 mm_registry: MultiModalRegistry,

256 ) -> tuple[int, int]:

@@ -263,9 +263,9 @@ def compute_encoder_budget(

263 - Space budget for encoder cache size, measured in number of tokens

264 from the input sequence.

265 """

266 - if mm_registry.supports_multimodal_inputs(model_config):

266 + if mm_registry.supports_multimodal_inputs(renderer_config):

267 max_tokens_by_modality = mm_registry.get_max_tokens_per_item_by_modality(

268 - model_config

268 + renderer_config

269 )

270

271 return compute_mm_encoder_budget(

vllm/v1/core/sched/scheduler.py

+1 -1

@@ -164,7 +164,7 @@ class Scheduler(SchedulerInterface):

164 # This can be changed when we make encoder cache for embedding caching

165 # across requests.

166 encoder_compute_budget, encoder_cache_size = compute_encoder_budget(

167 - model_config=vllm_config.model_config,

167 + renderer_config=vllm_config.renderer_config,

168 scheduler_config=vllm_config.scheduler_config,

169 mm_registry=mm_registry,

170 )

vllm/v1/engine/async_llm.py

+4 -3

@@ -91,6 +91,7 @@ class AsyncLLM(EngineClient):

91 # Ensure we can serialize custom transformer configs

92 maybe_register_config_serialize_by_value()

94 + self.renderer_config = vllm_config.renderer_config

95 self.model_config = vllm_config.model_config

96 self.vllm_config = vllm_config

97 self.observability_config = vllm_config.observability_config

@@ -108,15 +109,15 @@ class AsyncLLM(EngineClient):

109 "enabling logging without default stat loggers."

110 )

111

111 - if self.model_config.skip_tokenizer_init:

112 + if self.renderer_config.skip_tokenizer_init:

113 tokenizer = None

114 else:

114 - tokenizer = init_tokenizer_from_config(self.model_config)

115 + tokenizer = init_tokenizer_from_config(self.renderer_config)

116

117 self.input_processor = InputProcessor(self.vllm_config, tokenizer)

118 self.io_processor = get_io_processor(

119 self.vllm_config,

119 - self.model_config.io_processor_plugin,

120 + self.renderer_config.io_processor_plugin,

121 )

122

123 # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).

vllm/v1/engine/input_processor.py

+4 -3

@@ -43,6 +43,7 @@ class InputProcessor:

43 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

44 ) -> None:

45 self.vllm_config = vllm_config

46 + self.renderer_config = vllm_config.renderer_config

47 self.model_config = vllm_config.model_config

48 self.cache_config = vllm_config.cache_config

49 self.lora_config = vllm_config.lora_config

@@ -54,7 +55,7 @@ class InputProcessor:

55 self.mm_processor_cache = processor_cache_from_config(vllm_config, mm_registry)

57 self.input_preprocessor = InputPreprocessor(

57 - self.model_config,

58 + self.renderer_config,

59 tokenizer,

60 mm_registry,

61 mm_processor_cache=self.mm_processor_cache,

@@ -252,7 +253,7 @@ class InputProcessor:

253 if not params.structured_outputs or not self.structured_outputs_config:

254 return

255

255 - if self.model_config.skip_tokenizer_init and params.structured_outputs:

256 + if self.renderer_config.skip_tokenizer_init and params.structured_outputs:

257 raise ValueError(

258 "Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501

259 )

@@ -582,7 +583,7 @@ class InputProcessor:

583 if prompt_type == "encoder" and model_config.is_multimodal_model:

584 mm_registry = self.input_preprocessor.mm_registry

585 mm_processor = mm_registry.create_processor(

585 - model_config,

586 + self.renderer_config,

587 tokenizer=tokenizer,

588 )

589 assert isinstance(mm_processor, EncDecMultiModalProcessor)

vllm/v1/engine/llm_engine.py

+4 -3

@@ -60,6 +60,7 @@ class LLMEngine:

60 ) -> None:

61 self.vllm_config = vllm_config

62 self.observability_config = vllm_config.observability_config

63 + self.renderer_config = vllm_config.renderer_config

64 self.model_config = vllm_config.model_config

65 self.cache_config = vllm_config.cache_config

@@ -83,15 +84,15 @@ class LLMEngine:

84 self.dp_group = None

85 self.should_execute_dummy_batch = False

86 - if self.model_config.skip_tokenizer_init:

87 + if self.renderer_config.skip_tokenizer_init:

88 tokenizer = None

89 else:

89 - tokenizer = init_tokenizer_from_config(self.model_config)

90 + tokenizer = init_tokenizer_from_config(self.renderer_config)

92 self.input_processor = InputProcessor(self.vllm_config, tokenizer)

93 self.io_processor = get_io_processor(

94 self.vllm_config,

94 - self.model_config.io_processor_plugin,

95 + self.renderer_config.io_processor_plugin,

96 )

98 # OutputProcessor (convert EngineCoreOutputs --> RequestOutput).

vllm/v1/spec_decode/eagle.py

+1 -1

@@ -85,7 +85,7 @@ class EagleProposer:

85 # Multi-modal data support

86 self.mm_registry = MULTIMODAL_REGISTRY

87 self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(

88 - vllm_config.model_config

88 + vllm_config.renderer_config

89 )

91 self.attn_metadata_builder: AttentionMetadataBuilder | None = None

vllm/v1/structured_output/__init__.py

+6 -12

@@ -63,7 +63,7 @@ class StructuredOutputManager:

63 max_workers = max(1, min(multiprocessing.cpu_count() // 2, 8))

64 self.executor_for_fillmask = ThreadPoolExecutor(max_workers=max_workers)

66 - if not self.vllm_config.model_config.skip_tokenizer_init:

66 + if not vllm_config.renderer_config.skip_tokenizer_init:

67 # The default max_workers if not specified is the number of

68 # CPUs * 5, which is way too high since these tasks are CPU-bound,

69 # not I/O bound. We also know we would never dominate CPU usage

@@ -71,21 +71,15 @@ class StructuredOutputManager:

71 # of CPUs.

72 max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)

73 self.executor = ThreadPoolExecutor(max_workers=max_workers)

74 - self.tokenizer = init_tokenizer_from_config(

75 - model_config=self.vllm_config.model_config

76 - )

77 - reasoning_parser = (

78 - self.vllm_config.structured_outputs_config.reasoning_parser

79 - )

74 + self.tokenizer = init_tokenizer_from_config(vllm_config.renderer_config)

75 + reasoning_parser = vllm_config.structured_outputs_config.reasoning_parser

76 reasoning_parser_plugin = (

81 - self.vllm_config.structured_outputs_config.reasoning_parser_plugin

77 + vllm_config.structured_outputs_config.reasoning_parser_plugin

78 )

79 if reasoning_parser_plugin and len(reasoning_parser_plugin) > 3:

80 ReasoningParserManager.import_reasoning_parser(reasoning_parser_plugin)

86 - reasoning_parser = (

87 - self.vllm_config.structured_outputs_config.reasoning_parser

88 - )

82 + reasoning_parser = vllm_config.structured_outputs_config.reasoning_parser

83 if reasoning_parser:

84 reasoner_cls = ReasoningParserManager.get_reasoning_parser(

85 reasoning_parser

@@ -93,7 +87,7 @@ class StructuredOutputManager:

87 self.reasoner = reasoner_cls(tokenizer=self.tokenizer)

89 self.enable_in_reasoning = (

96 - self.vllm_config.structured_outputs_config.enable_in_reasoning

90 + vllm_config.structured_outputs_config.enable_in_reasoning

91 )

93 def grammar_init(self, request: Request) -> None:

vllm/v1/worker/gpu_model_runner.py

+4 -3

@@ -271,6 +271,7 @@ class GPUModelRunner(

271 device: torch.device,

272 ):

273 self.vllm_config = vllm_config

274 + self.renderer_config = vllm_config.renderer_config

275 self.model_config = vllm_config.model_config

276 self.cache_config = vllm_config.cache_config

277 self.compilation_config = vllm_config.compilation_config

@@ -335,7 +336,7 @@ class GPUModelRunner(

336 self.uses_mrope = model_config.uses_mrope

337 self.uses_xdrope_dim = model_config.uses_xdrope_dim

338 self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(

338 - model_config

339 + self.renderer_config

340 )

341

342 if self.model_config.is_encoder_decoder:

@@ -558,7 +559,7 @@ class GPUModelRunner(

559

560 self.mm_budget = (

561 MultiModalBudget(

561 - self.model_config,

562 + self.renderer_config,

563 self.scheduler_config,

564 self.mm_registry,

565 )

@@ -3873,7 +3874,7 @@ class GPUModelRunner(

3874 assert self.mm_budget is not None

3875

3876 dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(

3876 - model_config=self.model_config,

3877 + renderer_config=self.renderer_config,

3878 seq_len=self.max_model_len,

3879 mm_counts={modality: 1},

3880 cache=self.mm_budget.cache,

vllm/v1/worker/tpu_model_runner.py

+4 -3

@@ -143,6 +143,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

143 original_parallel_config: ParallelConfig | None = None,

144 ):

145 self.vllm_config = vllm_config

146 + self.renderer_config = vllm_config.renderer_config

147 self.model_config = vllm_config.model_config

148 self.cache_config = vllm_config.cache_config

149 self.lora_config = vllm_config.lora_config

@@ -222,7 +223,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

223 self.mm_registry = MULTIMODAL_REGISTRY

224 self.uses_mrope = model_config.uses_mrope

225 self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(

225 - model_config

226 + self.renderer_config

227 )

228 # TODO: Support M-RoPE (e.g, Qwen2-VL)

229 assert not self.uses_mrope, "TPU does not support M-RoPE yet."

@@ -353,7 +354,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

354

355 self.mm_budget = (

356 MultiModalBudget(

356 - self.model_config,

357 + self.renderer_config,

358 self.scheduler_config,

359 self.mm_registry,

360 )

@@ -2038,7 +2039,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

2039 assert self.mm_budget is not None

2040

2041 dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(

2041 - model_config=self.model_config,

2042 + renderer_config=self.renderer_config,

2043 seq_len=self.max_model_len,

2044 mm_counts={modality: 1},

2045 cache=self.mm_budget.cache,

vllm/v1/worker/utils.py

+12 -7

@@ -7,7 +7,7 @@ import torch

8 from vllm.attention.backends.abstract import AttentionBackend

9 from vllm.attention.layer import Attention

10 -from vllm.config import ModelConfig, SchedulerConfig, VllmConfig

10 +from vllm.config import RendererConfig, SchedulerConfig, VllmConfig

11 from vllm.model_executor.models.interfaces import MultiModalEmbeddings

12 from vllm.model_executor.models.utils import extract_layer_index

13 from vllm.multimodal.cache import processor_only_cache_from_config

@@ -23,24 +23,29 @@ class MultiModalBudget:

24 def __init__(

25 self,

26 - model_config: ModelConfig,

26 + renderer_config: RendererConfig,

27 scheduler_config: SchedulerConfig,

28 mm_registry: MultiModalRegistry,

29 ) -> None:

30 super().__init__()

32 - self.model_config = model_config

32 + self.renderer_config = renderer_config

33 + self.model_config = renderer_config.model_config

34 self.scheduler_config = scheduler_config

35 self.mm_registry = mm_registry

35 - self.cache = cache = processor_only_cache_from_config(model_config, mm_registry)

36 + self.cache = cache = processor_only_cache_from_config(

37 + renderer_config, mm_registry

38 + )

37 - self.max_model_len = model_config.max_model_len

40 + self.max_model_len = self.model_config.max_model_len

41 self.max_num_reqs = scheduler_config.max_num_seqs

40 - self.mm_limits = mm_registry.get_mm_limits_per_prompt(model_config, cache=cache)

43 + self.mm_limits = mm_registry.get_mm_limits_per_prompt(

44 + renderer_config, cache=cache

45 + )

47 max_tokens_by_modality = mm_registry.get_max_tokens_per_item_by_modality(

43 - model_config,

48 + renderer_config,

49 cache=cache,

50 profiler_limits=self.mm_limits,

51 )