A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 08:45 UTC
[Refactor] Pass full VllmConfig to Renderer (#34485)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 13, 2026 at 14:48 UTC
2f308214c0ff6cfa849879c5beb884192714f429
18 files changed
+137
-86
tests/entrypoints/openai/test_chat_error.py
+6
-1
@@ -59,11 +59,16 @@ class MockModelConfig:
59
return self.diff_sampling_param or {}
60
61
62
+@dataclass
63
+class MockVllmConfig:
64
+ model_config: MockModelConfig
65
+
66
+
67
def _build_renderer(model_config: MockModelConfig):
68
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
69
70
return HfRenderer(
66
- model_config,
71
+ MockVllmConfig(model_config),
72
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
73
)
74
tests/entrypoints/openai/test_completion_error.py
+6
-1
@@ -58,6 +58,11 @@ class MockModelConfig:
58
return self.diff_sampling_param or {}
59
60
61
+@dataclass
62
+class MockVllmConfig:
63
+ model_config: MockModelConfig
64
+
65
+
66
def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
67
models = OpenAIServingModels(
68
engine_client=engine,
@@ -74,7 +79,7 @@ def _build_renderer(model_config: MockModelConfig):
79
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
80
81
return HfRenderer(
77
- model_config,
82
+ MockVllmConfig(model_config),
83
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
84
)
85
tests/entrypoints/openai/test_lora_resolvers.py
+6
-1
@@ -57,6 +57,11 @@ class MockModelConfig:
57
return self.diff_sampling_param or {}
58
59
60
+@dataclass
61
+class MockVllmConfig:
62
+ model_config: MockModelConfig
63
+
64
+
65
class MockLoRAResolver(LoRAResolver):
66
async def resolve_lora(
67
self, base_model_name: str, lora_name: str
@@ -91,7 +96,7 @@ def _build_renderer(model_config: MockModelConfig):
96
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
97
98
return HfRenderer(
94
- model_config,
99
+ MockVllmConfig(model_config),
100
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
101
)
102
tests/entrypoints/openai/test_serving_chat.py
+14
-3
@@ -534,11 +534,16 @@ class MockModelConfig:
534
return self.diff_sampling_param or {}
535
536
537
+@dataclass
538
+class MockVllmConfig:
539
+ model_config: MockModelConfig
540
+
541
+
542
def _build_renderer(model_config: MockModelConfig):
543
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
544
545
return HfRenderer(
541
- model_config,
546
+ MockVllmConfig(model_config),
547
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
548
)
549
@@ -749,7 +754,10 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
754
mock_engine.io_processor = MagicMock()
755
756
mock_tokenizer = MagicMock(spec=MistralTokenizer)
752
- mock_renderer = MistralRenderer(mock_engine.model_config, tokenizer_kwargs={})
757
+ mock_renderer = MistralRenderer(
758
+ MockVllmConfig(mock_engine.model_config),
759
+ tokenizer_kwargs={},
760
+ )
761
mock_renderer._tokenizer = mock_tokenizer
762
# Force the Mistral chat template renderer to return token IDs.
763
# Choose a prompt length that is < max_model_len, but large enough that
@@ -788,7 +796,10 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():
796
mock_engine.io_processor = MagicMock()
797
798
mock_tokenizer = MagicMock(spec=MistralTokenizer)
791
- mock_renderer = MistralRenderer(mock_engine.model_config, tokenizer_kwargs={})
799
+ mock_renderer = MistralRenderer(
800
+ MockVllmConfig(mock_engine.model_config),
801
+ tokenizer_kwargs={},
802
+ )
803
mock_renderer._tokenizer = mock_tokenizer
804
# prompt_token_ids length == max_model_len should be rejected for
805
# completion-like requests (ChatCompletionRequest).
tests/renderers/test_completions.py
+32
-23
@@ -40,6 +40,11 @@ class MockModelConfig:
40
is_encoder_decoder: bool = False
41
42
43
+@dataclass
44
+class MockVllmConfig:
45
+ model_config: MockModelConfig
46
+
47
+
48
@dataclass
49
class DummyTokenizer:
50
truncation_side: str = "left"
@@ -72,7 +77,7 @@ def _build_renderer(
77
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
78
79
renderer = HfRenderer(
75
- model_config,
80
+ MockVllmConfig(model_config),
81
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
82
)
83
@@ -104,14 +109,14 @@ class TestValidatePrompt:
109
renderer = _build_renderer(MockModelConfig())
110
111
with pytest.raises(ValueError, match="at least one prompt"):
107
- renderer.render_prompts(_preprocess_prompt(renderer.config, []))
112
+ renderer.render_prompts(_preprocess_prompt(renderer.model_config, []))
113
114
def test_invalid_type(self):
115
renderer = _build_renderer(MockModelConfig())
116
117
with pytest.raises(TypeError, match="should be a list of integers"):
118
renderer.render_prompts(
114
- _preprocess_prompt(renderer.config, [[1, 2], ["foo", "bar"]]) # type: ignore[arg-type]
119
+ _preprocess_prompt(renderer.model_config, [[1, 2], ["foo", "bar"]]) # type: ignore[arg-type]
120
)
121
122
@@ -120,7 +125,9 @@ class TestRenderPrompt:
125
renderer = _build_renderer(MockModelConfig())
126
127
tokens = [101, 7592, 2088]
123
- prompts = renderer.render_prompts(_preprocess_prompt(renderer.config, tokens))
128
+ prompts = renderer.render_prompts(
129
+ _preprocess_prompt(renderer.model_config, tokens)
130
+ )
131
results = renderer.tokenize_prompts(
132
prompts,
133
TokenizeParams(max_total_tokens=100),
@@ -134,7 +141,7 @@ class TestRenderPrompt:
141
142
token_lists = [[101, 7592, 2088], [102, 1234, 5678, 9012], [103, 4567]]
143
prompts = renderer.render_prompts(
137
- _preprocess_prompt(renderer.config, token_lists)
144
+ _preprocess_prompt(renderer.model_config, token_lists)
145
)
146
results = renderer.tokenize_prompts(
147
prompts,
@@ -151,7 +158,7 @@ class TestRenderPrompt:
158
159
text_input = "x" * 10
160
prompts = renderer.render_prompts(
154
- _preprocess_prompt(renderer.config, text_input)
161
+ _preprocess_prompt(renderer.model_config, text_input)
162
)
163
results = renderer.tokenize_prompts(
164
prompts,
@@ -166,7 +173,7 @@ class TestRenderPrompt:
173
174
text_list_input = ["x" * 10, "x" * 12, "x" * 14]
175
prompts = renderer.render_prompts(
169
- _preprocess_prompt(renderer.config, text_list_input)
176
+ _preprocess_prompt(renderer.model_config, text_list_input)
177
)
178
results = renderer.tokenize_prompts(
179
prompts,
@@ -181,7 +188,7 @@ class TestRenderPrompt:
188
renderer = _build_renderer(MockModelConfig())
189
190
prompts = renderer.render_prompts(
184
- _preprocess_prompt(renderer.config, "x" * 200)
191
+ _preprocess_prompt(renderer.model_config, "x" * 200)
192
)
193
results = renderer.tokenize_prompts(
194
prompts,
@@ -195,7 +202,7 @@ class TestRenderPrompt:
202
renderer = _build_renderer(MockModelConfig())
203
204
prompts = renderer.render_prompts(
198
- _preprocess_prompt(renderer.config, "x" * 200)
205
+ _preprocess_prompt(renderer.model_config, "x" * 200)
206
)
207
results = renderer.tokenize_prompts(
208
prompts,
@@ -209,7 +216,7 @@ class TestRenderPrompt:
216
renderer = _build_renderer(MockModelConfig())
217
218
prompts = renderer.render_prompts(
212
- _preprocess_prompt(renderer.config, "x" * 200)
219
+ _preprocess_prompt(renderer.model_config, "x" * 200)
220
)
221
results = renderer.tokenize_prompts(
222
prompts,
@@ -224,7 +231,7 @@ class TestRenderPrompt:
231
232
long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens
233
prompts = renderer.render_prompts(
227
- _preprocess_prompt(renderer.config, long_tokens)
234
+ _preprocess_prompt(renderer.model_config, long_tokens)
235
)
236
results = renderer.tokenize_prompts(
237
prompts,
@@ -240,7 +247,7 @@ class TestRenderPrompt:
247
248
long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens
249
prompts = renderer.render_prompts(
243
- _preprocess_prompt(renderer.config, long_tokens)
250
+ _preprocess_prompt(renderer.model_config, long_tokens)
251
)
252
results = renderer.tokenize_prompts(
253
prompts,
@@ -257,7 +264,7 @@ class TestRenderPrompt:
264
# Exceeds max_total_tokens and max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN
265
long_tokens = "x" * 150
266
prompts = renderer.render_prompts(
260
- _preprocess_prompt(renderer.config, long_tokens)
267
+ _preprocess_prompt(renderer.model_config, long_tokens)
268
)
269
270
with pytest.raises(
@@ -278,7 +285,7 @@ class TestRenderPrompt:
285
# Exceeds max_total_tokens but not max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN
286
long_tokens = "x" * 150
287
prompts = renderer.render_prompts(
281
- _preprocess_prompt(renderer.config, long_tokens)
288
+ _preprocess_prompt(renderer.model_config, long_tokens)
289
)
290
291
with pytest.raises(
@@ -299,7 +306,7 @@ class TestRenderPrompt:
306
307
long_tokens = list(range(150)) # Exceeds max_total_tokens=100
308
prompts = renderer.render_prompts(
302
- _preprocess_prompt(renderer.config, long_tokens)
309
+ _preprocess_prompt(renderer.model_config, long_tokens)
310
)
311
312
with pytest.raises(
@@ -315,7 +322,7 @@ class TestRenderPrompt:
322
renderer = _build_renderer(MockModelConfig(skip_tokenizer_init=True))
323
324
prompts = renderer.render_prompts(
318
- _preprocess_prompt(renderer.config, "Hello world")
325
+ _preprocess_prompt(renderer.model_config, "Hello world")
326
)
327
328
with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"):
@@ -328,7 +335,9 @@ class TestRenderPrompt:
335
renderer = _build_renderer(MockModelConfig())
336
337
tokens = [1, 2, 3, 4]
331
- prompts = renderer.render_prompts(_preprocess_prompt(renderer.config, tokens))
338
+ prompts = renderer.render_prompts(
339
+ _preprocess_prompt(renderer.model_config, tokens)
340
+ )
341
results = renderer.tokenize_prompts(
342
prompts,
343
TokenizeParams(
@@ -358,7 +367,7 @@ class TestRenderEmbedPrompt:
367
embed_bytes = self._create_test_embed_bytes(tensor_input)
368
369
prompts = renderer.render_prompts(
361
- _preprocess_prompt(renderer.config, embed_bytes)
370
+ _preprocess_prompt(renderer.model_config, embed_bytes)
371
)
372
results = renderer.tokenize_prompts(
373
prompts,
@@ -379,7 +388,7 @@ class TestRenderEmbedPrompt:
388
389
prompts = renderer.render_prompts(
390
_preprocess_prompt(
382
- renderer.config,
391
+ renderer.model_config,
392
[self._create_test_embed_bytes(t) for t in tensor_inputs],
393
)
394
)
@@ -400,7 +409,7 @@ class TestRenderEmbedPrompt:
409
410
prompts = renderer.render_prompts(
411
_preprocess_prompt(
403
- renderer.config, self._create_test_embed_bytes(tensor_input)
412
+ renderer.model_config, self._create_test_embed_bytes(tensor_input)
413
)
414
)
415
results = renderer.tokenize_prompts(
@@ -427,7 +436,7 @@ class TestRenderEmbedPrompt:
436
437
prompts = renderer.render_prompts(
438
_preprocess_prompt(
430
- renderer.config, self._create_test_embed_bytes(tensor_input)
439
+ renderer.model_config, self._create_test_embed_bytes(tensor_input)
440
)
441
)
442
results = renderer.tokenize_prompts(
@@ -446,7 +455,7 @@ class TestRenderEmbedPrompt:
455
456
prompts = renderer.render_prompts(
457
_preprocess_prompt(
449
- renderer.config, self._create_test_embed_bytes(tensor_input)
458
+ renderer.model_config, self._create_test_embed_bytes(tensor_input)
459
)
460
)
461
results = renderer.tokenize_prompts(
@@ -466,7 +475,7 @@ class TestRenderEmbedPrompt:
475
476
prompts = renderer.render_prompts(
477
_preprocess_prompt(
469
- renderer.config,
478
+ renderer.model_config,
479
[text_input, self._create_test_embed_bytes(tensor_input)],
480
)
481
)
tests/renderers/test_mistral.py
+9
-1
@@ -38,6 +38,11 @@ class MockModelConfig:
38
is_encoder_decoder: bool = False
39
40
41
+@dataclass
42
+class MockVllmConfig:
43
+ model_config: MockModelConfig
44
+
45
+
46
@pytest.mark.asyncio
47
async def test_async_mistral_tokenizer_does_not_block_event_loop():
48
expected_tokens = [1, 2, 3]
@@ -50,7 +55,10 @@ async def test_async_mistral_tokenizer_does_not_block_event_loop():
55
mock_model_config = MockModelConfig(skip_tokenizer_init=True)
56
mock_tokenizer = Mock(spec=MistralTokenizer)
57
mock_tokenizer.apply_chat_template = mocked_apply_chat_template
53
- mock_renderer = MistralRenderer(mock_model_config, tokenizer_kwargs={})
58
+ mock_renderer = MistralRenderer(
59
+ MockVllmConfig(mock_model_config),
60
+ tokenizer_kwargs={},
61
+ )
62
mock_renderer._tokenizer = mock_tokenizer
63
64
task = mock_renderer.render_messages_async([], ChatParams())
tests/test_inputs.py
+3
-2
@@ -3,7 +3,7 @@
3
4
import pytest
5
6
-from vllm.config import ModelConfig
6
+from vllm.config import ModelConfig, VllmConfig
7
from vllm.inputs.preprocess import InputPreprocessor
8
9
pytestmark = pytest.mark.cpu_test
@@ -20,7 +20,8 @@ pytestmark = pytest.mark.cpu_test
20
)
21
def test_preprocessor_always_mm_code_path(model_id, prompt):
22
model_config = ModelConfig(model=model_id)
23
- input_preprocessor = InputPreprocessor(model_config)
23
+ vllm_config = VllmConfig(model_config=model_config)
24
+ input_preprocessor = InputPreprocessor(vllm_config)
25
26
# HF processor adds sep token
27
tokenizer = input_preprocessor.get_tokenizer()
vllm/inputs/preprocess.py
+5
-6
@@ -6,7 +6,7 @@ from typing import Any, overload
6
7
from typing_extensions import assert_never
8
9
-from vllm.config import ModelConfig, ObservabilityConfig
9
+from vllm.config import VllmConfig
10
from vllm.logger import init_logger
11
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
12
from vllm.multimodal.cache import BaseMultiModalProcessorCache
@@ -54,17 +54,16 @@ logger = init_logger(__name__)
54
class InputPreprocessor:
55
def __init__(
56
self,
57
- model_config: ModelConfig,
58
- observability_config: ObservabilityConfig | None = None,
57
+ vllm_config: VllmConfig,
58
renderer: BaseRenderer | None = None,
59
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
60
mm_processor_cache: BaseMultiModalProcessorCache | None = None,
61
) -> None:
62
super().__init__()
63
65
- self.model_config = model_config
66
- self.observability_config = observability_config
67
- self.renderer = renderer or renderer_from_config(model_config)
64
+ self.model_config = vllm_config.model_config
65
+ self.observability_config = vllm_config.observability_config
66
+ self.renderer = renderer or renderer_from_config(vllm_config)
67
self.mm_registry = mm_registry
68
self.mm_processor_cache = mm_processor_cache
69
vllm/renderers/base.py
+7
-7
@@ -21,7 +21,7 @@ from .inputs.preprocess import extract_target_prompt
21
from .params import ChatParams, TokenizeParams
22
23
if TYPE_CHECKING:
24
- from vllm.config import ModelConfig
24
+ from vllm.config import VllmConfig
25
from vllm.entrypoints.chat_utils import (
26
ChatCompletionMessageParam,
27
ConversationMessage,
@@ -35,15 +35,15 @@ class BaseRenderer(ABC):
35
@abstractmethod
36
def from_config(
37
cls,
38
- config: "ModelConfig",
38
+ config: "VllmConfig",
39
tokenizer_kwargs: dict[str, Any],
40
) -> "BaseRenderer":
41
raise NotImplementedError
42
43
- def __init__(self, config: "ModelConfig") -> None:
43
+ def __init__(self, config: "VllmConfig") -> None:
44
super().__init__()
45
46
- self.config = config
46
+ self.model_config = config.model_config
47
48
# Lazy initialization since offline LLM doesn't use async
49
self._async_tokenizer: AsyncMicrobatchTokenizer | None = None
@@ -90,7 +90,7 @@ class BaseRenderer(ABC):
90
prompt: DictPrompt | bytes,
91
) -> DictPrompt:
92
if isinstance(prompt, bytes):
93
- embeds = safe_load_prompt_embeds(self.config, prompt)
93
+ embeds = safe_load_prompt_embeds(self.model_config, prompt)
94
prompt = EmbedsPrompt(prompt_embeds=embeds)
95
96
return prompt
@@ -310,7 +310,7 @@ class BaseRenderer(ABC):
310
return
311
312
for prompt in prompts:
313
- target_prompt = extract_target_prompt(self.config, prompt)
313
+ target_prompt = extract_target_prompt(self.model_config, prompt)
314
target_prompt.update(prompt_extras) # type: ignore[arg-type]
315
316
# Top-level methods
@@ -325,7 +325,7 @@ class BaseRenderer(ABC):
325
326
# NOTE: Some MM models have non-default `add_special_tokens`
327
# so we handle tokenization in multi-modal processor
328
- if self.config.is_multimodal_model:
328
+ if self.model_config.is_multimodal_model:
329
self._apply_prompt_extras(dict_prompts, prompt_extras)
330
return dict_prompts
331
vllm/renderers/deepseek_v32.py
+7
-6
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from typing import Any
4
5
-from vllm.config import ModelConfig
5
+from vllm.config import VllmConfig
6
from vllm.entrypoints.chat_utils import (
7
ChatCompletionMessageParam,
8
ConversationMessage,
@@ -26,19 +26,20 @@ class DeepseekV32Renderer(BaseRenderer):
26
@classmethod
27
def from_config(
28
cls,
29
- config: ModelConfig,
29
+ config: VllmConfig,
30
tokenizer_kwargs: dict[str, Any],
31
) -> "BaseRenderer":
32
return cls(config, tokenizer_kwargs)
33
34
def __init__(
35
self,
36
- config: ModelConfig,
36
+ config: VllmConfig,
37
tokenizer_kwargs: dict[str, Any],
38
) -> None:
39
super().__init__(config)
40
41
- if config.skip_tokenizer_init:
41
+ model_config = self.model_config
42
+ if model_config.skip_tokenizer_init:
43
tokenizer = None
44
else:
45
tokenizer = cached_get_tokenizer(
@@ -67,7 +68,7 @@ class DeepseekV32Renderer(BaseRenderer):
68
tokenizer = self.get_tokenizer()
69
conversation, mm_data, mm_uuids = parse_chat_messages(
70
messages,
70
- self.config,
71
+ self.model_config,
72
content_format="string",
73
)
74
@@ -93,7 +94,7 @@ class DeepseekV32Renderer(BaseRenderer):
94
tokenizer = self.get_tokenizer()
95
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
96
messages,
96
- self.config,
97
+ self.model_config,
98
content_format="string",
99
)
100
vllm/renderers/grok2.py
+7
-6
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from typing import Any
4
5
-from vllm.config import ModelConfig
5
+from vllm.config import VllmConfig
6
from vllm.entrypoints.chat_utils import (
7
ChatCompletionMessageParam,
8
ConversationMessage,
@@ -25,19 +25,20 @@ class Grok2Renderer(BaseRenderer):
25
@classmethod
26
def from_config(
27
cls,
28
- config: ModelConfig,
28
+ config: VllmConfig,
29
tokenizer_kwargs: dict[str, Any],
30
) -> "BaseRenderer":
31
return cls(config, tokenizer_kwargs)
32
33
def __init__(
34
self,
35
- config: ModelConfig,
35
+ config: VllmConfig,
36
tokenizer_kwargs: dict[str, Any],
37
) -> None:
38
super().__init__(config)
39
40
- if config.skip_tokenizer_init:
40
+ model_config = self.model_config
41
+ if model_config.skip_tokenizer_init:
42
tokenizer = None
43
else:
44
tokenizer = cached_get_tokenizer(
@@ -66,7 +67,7 @@ class Grok2Renderer(BaseRenderer):
67
tokenizer = self.get_tokenizer()
68
conversation, mm_data, mm_uuids = parse_chat_messages(
69
messages,
69
- self.config,
70
+ self.model_config,
71
content_format="string",
72
)
73
@@ -92,7 +93,7 @@ class Grok2Renderer(BaseRenderer):
93
tokenizer = self.get_tokenizer()
94
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
95
messages,
95
- self.config,
96
+ self.model_config,
97
content_format="string",
98
)
99
vllm/renderers/hf.py
+8
-7
@@ -14,7 +14,7 @@ import jinja2.nodes
14
import jinja2.parser
15
import jinja2.sandbox
16
17
-from vllm.config import ModelConfig
17
+from vllm.config import ModelConfig, VllmConfig
18
from vllm.entrypoints.chat_utils import (
19
ChatCompletionMessageParam,
20
ChatTemplateContentFormat,
@@ -589,23 +589,24 @@ class HfRenderer(BaseRenderer):
589
@classmethod
590
def from_config(
591
cls,
592
- config: ModelConfig,
592
+ config: VllmConfig,
593
tokenizer_kwargs: dict[str, Any],
594
) -> "BaseRenderer":
595
return cls(config, tokenizer_kwargs)
596
597
def __init__(
598
self,
599
- config: ModelConfig,
599
+ config: VllmConfig,
600
tokenizer_kwargs: dict[str, Any],
601
) -> None:
602
super().__init__(config)
603
604
+ model_config = self.model_config
605
self.use_unified_vision_chunk = getattr(
605
- config.hf_config, "use_unified_vision_chunk", False
606
+ model_config.hf_config, "use_unified_vision_chunk", False
607
)
608
608
- if config.skip_tokenizer_init:
609
+ if model_config.skip_tokenizer_init:
610
tokenizer = None
611
else:
612
tokenizer = cast(
@@ -634,7 +635,7 @@ class HfRenderer(BaseRenderer):
635
messages: list[ChatCompletionMessageParam],
636
params: ChatParams,
637
) -> tuple[list[ConversationMessage], DictPrompt]:
637
- model_config = self.config
638
+ model_config = self.model_config
639
tokenizer = self.get_tokenizer()
640
641
conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -688,7 +689,7 @@ class HfRenderer(BaseRenderer):
689
messages: list[ChatCompletionMessageParam],
690
params: ChatParams,
691
) -> tuple[list[ConversationMessage], DictPrompt]:
691
- model_config = self.config
692
+ model_config = self.model_config
693
tokenizer = self.get_tokenizer()
694
695
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
vllm/renderers/mistral.py
+7
-6
@@ -3,7 +3,7 @@
3
from concurrent.futures import ThreadPoolExecutor
4
from typing import Any
5
6
-from vllm.config import ModelConfig
6
+from vllm.config import VllmConfig
7
from vllm.entrypoints.chat_utils import (
8
ChatCompletionMessageParam,
9
ConversationMessage,
@@ -54,19 +54,20 @@ class MistralRenderer(BaseRenderer):
54
@classmethod
55
def from_config(
56
cls,
57
- config: ModelConfig,
57
+ config: VllmConfig,
58
tokenizer_kwargs: dict[str, Any],
59
) -> "BaseRenderer":
60
return cls(config, tokenizer_kwargs)
61
62
def __init__(
63
self,
64
- config: ModelConfig,
64
+ config: VllmConfig,
65
tokenizer_kwargs: dict[str, Any],
66
) -> None:
67
super().__init__(config)
68
69
- if config.skip_tokenizer_init:
69
+ model_config = self.model_config
70
+ if model_config.skip_tokenizer_init:
71
tokenizer = None
72
else:
73
tokenizer = cached_get_tokenizer(
@@ -100,7 +101,7 @@ class MistralRenderer(BaseRenderer):
101
tokenizer = self.get_tokenizer()
102
conversation, mm_data, mm_uuids = parse_chat_messages(
103
messages,
103
- self.config,
104
+ self.model_config,
105
content_format="string",
106
)
107
@@ -126,7 +127,7 @@ class MistralRenderer(BaseRenderer):
127
tokenizer = self.get_tokenizer()
128
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
129
messages,
129
- self.config,
130
+ self.model_config,
131
content_format="string",
132
)
133
vllm/renderers/registry.py
+9
-5
@@ -10,7 +10,7 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
10
from .base import BaseRenderer
11
12
if TYPE_CHECKING:
13
- from vllm.config import ModelConfig
13
+ from vllm.config import VllmConfig
14
15
logger = init_logger(__name__)
16
@@ -55,7 +55,7 @@ class RendererRegistry:
55
def load_renderer(
56
self,
57
renderer_mode: str,
58
- config: "ModelConfig",
58
+ config: "VllmConfig",
59
tokenizer_kwargs: dict[str, Any],
60
) -> BaseRenderer:
61
renderer_cls = self.load_renderer_cls(renderer_mode)
@@ -71,12 +71,16 @@ RENDERER_REGISTRY = RendererRegistry(
71
"""The global `RendererRegistry` instance."""
72
73
74
-def renderer_from_config(config: "ModelConfig", **kwargs):
74
+def renderer_from_config(config: "VllmConfig", **kwargs):
75
+ model_config = config.model_config
76
tokenizer_mode, tokenizer_name, args, kwargs = tokenizer_args_from_config(
76
- config, **kwargs
77
+ model_config, **kwargs
78
)
79
79
- if config.tokenizer_mode == "auto" and config.model_impl == "terratorch":
80
+ if (
81
+ model_config.tokenizer_mode == "auto"
82
+ and model_config.model_impl == "terratorch"
83
+ ):
84
renderer_mode = "terratorch"
85
else:
86
renderer_mode = tokenizer_mode
vllm/renderers/terratorch.py
+7
-6
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from typing import Any
4
5
-from vllm.config import ModelConfig
5
+from vllm.config import VllmConfig
6
from vllm.entrypoints.chat_utils import (
7
ChatCompletionMessageParam,
8
ConversationMessage,
@@ -24,15 +24,16 @@ class TerratorchRenderer(BaseRenderer):
24
@classmethod
25
def from_config(
26
cls,
27
- config: "ModelConfig",
27
+ config: VllmConfig,
28
tokenizer_kwargs: dict[str, Any],
29
) -> "BaseRenderer":
30
return cls(config)
31
32
- def __init__(self, config: ModelConfig) -> None:
32
+ def __init__(self, config: VllmConfig) -> None:
33
super().__init__(config)
34
35
- if not config.skip_tokenizer_init:
35
+ model_config = self.model_config
36
+ if not model_config.skip_tokenizer_init:
37
raise ValueError("Terratorch renderer requires `skip_tokenizer_init=True`")
38
39
@property
@@ -47,7 +48,7 @@ class TerratorchRenderer(BaseRenderer):
48
messages: list[ChatCompletionMessageParam],
49
params: ChatParams,
50
) -> tuple[list[ConversationMessage], DictPrompt]:
50
- model_config = self.config
51
+ model_config = self.model_config
52
53
conversation, mm_data, mm_uuids = parse_chat_messages(
54
messages,
@@ -68,7 +69,7 @@ class TerratorchRenderer(BaseRenderer):
69
messages: list[ChatCompletionMessageParam],
70
params: ChatParams,
71
) -> tuple[list[ConversationMessage], DictPrompt]:
71
- model_config = self.config
72
+ model_config = self.model_config
73
74
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
75
messages,
vllm/v1/engine/async_llm.py
+1
-1
@@ -132,7 +132,7 @@ class AsyncLLM(EngineClient):
132
"enabling logging without default stat loggers."
133
)
134
135
- self.renderer = renderer = renderer_from_config(self.model_config)
135
+ self.renderer = renderer = renderer_from_config(self.vllm_config)
136
self.io_processor = get_io_processor(
137
self.vllm_config,
138
self.model_config.io_processor_plugin,
vllm/v1/engine/input_processor.py
+2
-3
@@ -59,7 +59,7 @@ class InputProcessor:
59
60
self.generation_config_fields = model_config.try_get_generation_config()
61
62
- self.renderer = renderer or renderer_from_config(model_config)
62
+ self.renderer = renderer or renderer_from_config(vllm_config)
63
self.mm_registry = mm_registry
64
self.mm_processor_cache = mm_registry.processor_cache_from_config(vllm_config)
65
@@ -75,8 +75,7 @@ class InputProcessor:
75
mm_budget.reset_cache() # Not used anymore
76
77
self.input_preprocessor = InputPreprocessor(
78
- model_config,
79
- self.observability_config,
78
+ vllm_config,
79
renderer=renderer,
80
mm_registry=mm_registry,
81
mm_processor_cache=self.mm_processor_cache,
vllm/v1/engine/llm_engine.py
+1
-1
@@ -90,7 +90,7 @@ class LLMEngine:
90
self.dp_group = None
91
self.should_execute_dummy_batch = False
92
93
- self.renderer = renderer = renderer_from_config(self.model_config)
93
+ self.renderer = renderer = renderer_from_config(self.vllm_config)
94
self.io_processor = get_io_processor(
95
self.vllm_config,
96
self.model_config.io_processor_plugin,