@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4bdf7ac593
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Bugfix] Fix SHM cache initialization (#26427)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Oct 9, 2025 at 17:48 UTC 4bdf7ac59338e00f2899fd72541d80d09a59e03c
30 files changed +357 -417
tests/entrypoints/openai/test_lora_resolvers.py
+5 -3
@@ -113,15 +113,17 @@ def mock_serving_setup():
113 mock_engine.generate.reset_mock()
114 mock_engine.add_lora.reset_mock()
115
116 - mock_model_config = MockModelConfig()
116 + mock_engine.model_config = MockModelConfig()
117 + mock_engine.processor = MagicMock()
118 + mock_engine.io_processor = MagicMock()
119 +
120 models = OpenAIServingModels(
121 engine_client=mock_engine,
122 base_model_paths=BASE_MODEL_PATHS,
120 - model_config=mock_model_config,
123 )
124
125 serving_completion = OpenAIServingCompletion(
124 - mock_engine, mock_model_config, models, request_logger=None
126 + mock_engine, models, request_logger=None
127 )
128
129 serving_completion._process_inputs = AsyncMock(
tests/entrypoints/openai/test_serving_chat.py
+29 -16
@@ -245,17 +245,13 @@ class MockModelConfig:
245 return self.diff_sampling_param or {}
246
247
248 -def _build_serving_chat(
249 - engine: AsyncLLM, model_config: MockModelConfig
250 -) -> OpenAIServingChat:
248 +def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
249 models = OpenAIServingModels(
250 engine_client=engine,
251 base_model_paths=BASE_MODEL_PATHS,
254 - model_config=model_config,
252 )
253 serving_chat = OpenAIServingChat(
254 engine,
258 - model_config,
255 models,
256 response_role="assistant",
257 chat_template=CHAT_TEMPLATE,
@@ -280,18 +276,17 @@ def _build_serving_chat(
276
277 @dataclass
278 class MockEngine:
283 - async def get_model_config(self):
284 - return MockModelConfig()
279 + model_config: MockModelConfig = field(default_factory=MockModelConfig)
280 + processor: MagicMock = field(default_factory=MagicMock)
281 + io_processor: MagicMock = field(default_factory=MagicMock)
282
283
284 async def _async_serving_chat_init():
285 engine = MockEngine()
289 - model_config = await engine.get_model_config()
286
291 - models = OpenAIServingModels(engine, model_config, BASE_MODEL_PATHS)
287 + models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
288 serving_completion = OpenAIServingChat(
289 engine,
294 - model_config,
290 models,
291 response_role="assistant",
292 chat_template=CHAT_TEMPLATE,
@@ -311,8 +306,11 @@ async def test_serving_chat_returns_correct_model_name():
306 mock_engine = MagicMock(spec=AsyncLLM)
307 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
308 mock_engine.errored = False
309 + mock_engine.model_config = MockModelConfig()
310 + mock_engine.processor = MagicMock()
311 + mock_engine.io_processor = MagicMock()
312
315 - serving_chat = _build_serving_chat(mock_engine, MockModelConfig())
313 + serving_chat = _build_serving_chat(mock_engine)
314 messages = [{"role": "user", "content": "what is 1+1?"}]
315
316 async def return_model_name(*args):
@@ -338,8 +336,11 @@ async def test_serving_chat_should_set_correct_max_tokens():
336 mock_engine = MagicMock(spec=AsyncLLM)
337 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
338 mock_engine.errored = False
339 + mock_engine.model_config = MockModelConfig()
340 + mock_engine.processor = MagicMock()
341 + mock_engine.io_processor = MagicMock()
342
342 - serving_chat = _build_serving_chat(mock_engine, MockModelConfig())
343 + serving_chat = _build_serving_chat(mock_engine)
344
345 req = ChatCompletionRequest(
346 model=MODEL_NAME,
@@ -368,9 +369,12 @@ async def test_serving_chat_should_set_correct_max_tokens():
369 mock_engine = MagicMock(spec=AsyncLLM)
370 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
371 mock_engine.errored = False
372 + mock_engine.model_config = mock_model_config
373 + mock_engine.processor = MagicMock()
374 + mock_engine.io_processor = MagicMock()
375
376 # Initialize the serving chat
373 - serving_chat = _build_serving_chat(mock_engine, mock_model_config)
377 + serving_chat = _build_serving_chat(mock_engine)
378
379 # Test Case 1: No max_tokens specified in request
380 req = ChatCompletionRequest(
@@ -410,9 +414,12 @@ async def test_serving_chat_should_set_correct_max_tokens():
414 mock_engine = MagicMock(spec=AsyncLLM)
415 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
416 mock_engine.errored = False
417 + mock_engine.model_config = mock_model_config
418 + mock_engine.processor = MagicMock()
419 + mock_engine.io_processor = MagicMock()
420
421 # Initialize the serving chat
415 - serving_chat = _build_serving_chat(mock_engine, mock_model_config)
422 + serving_chat = _build_serving_chat(mock_engine)
423
424 # Test case 1: No max_tokens specified, defaults to context_window
425 req = ChatCompletionRequest(
@@ -453,9 +460,12 @@ async def test_serving_chat_could_load_correct_generation_config():
460 mock_engine = MagicMock(spec=AsyncLLM)
461 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
462 mock_engine.errored = False
463 + mock_engine.model_config = mock_model_config
464 + mock_engine.processor = MagicMock()
465 + mock_engine.io_processor = MagicMock()
466
467 # Initialize the serving chat
458 - serving_chat = _build_serving_chat(mock_engine, mock_model_config)
468 + serving_chat = _build_serving_chat(mock_engine)
469
470 req = ChatCompletionRequest(
471 model=MODEL_NAME,
@@ -496,8 +506,11 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):
506 mock_engine = MagicMock(spec=AsyncLLM)
507 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
508 mock_engine.errored = False
509 + mock_engine.model_config = mock_model_config
510 + mock_engine.processor = MagicMock()
511 + mock_engine.io_processor = MagicMock()
512
500 - serving_chat = _build_serving_chat(mock_engine, mock_model_config)
513 + serving_chat = _build_serving_chat(mock_engine)
514
515 # Test cache_salt
516 req = ChatCompletionRequest(
tests/entrypoints/openai/test_serving_engine.py
+3 -1
@@ -22,10 +22,12 @@ def serving() -> OpenAIServing:
22 model_config = Mock(spec=ModelConfig)
23 model_config.max_model_len = 32768
24 models = Mock(spec=OpenAIServingModels)
25 + models.model_config = model_config
26 + models.processor = Mock()
27 + models.io_processor = Mock()
28
29 serving = OpenAIServing(
30 engine_client=engine_client,
28 - model_config=model_config,
31 models=models,
32 request_logger=None,
33 )
tests/entrypoints/openai/test_serving_models.py
+4 -2
@@ -25,15 +25,17 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
25
26
27 async def _async_serving_models_init() -> OpenAIServingModels:
28 - mock_model_config = MagicMock(spec=ModelConfig)
28 mock_engine_client = MagicMock(spec=EngineClient)
29 # Set the max_model_len attribute to avoid missing attribute
30 + mock_model_config = MagicMock(spec=ModelConfig)
31 mock_model_config.max_model_len = 2048
32 + mock_engine_client.model_config = mock_model_config
33 + mock_engine_client.processor = MagicMock()
34 + mock_engine_client.io_processor = MagicMock()
35
36 serving_models = OpenAIServingModels(
37 engine_client=mock_engine_client,
38 base_model_paths=BASE_MODEL_PATHS,
36 - model_config=mock_model_config,
39 lora_modules=None,
40 )
41 await serving_models.init_static_loras()
tests/entrypoints/openai/test_serving_responses.py
+9 -5
@@ -2,7 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from contextlib import AsyncExitStack
5 -from unittest.mock import AsyncMock, MagicMock
5 +from unittest.mock import MagicMock
6
7 import pytest
8 import pytest_asyncio
@@ -70,11 +70,14 @@ class TestInitializeToolSessions:
70 """Create a real OpenAIServingResponses instance for testing"""
71 # Create minimal mocks for required dependencies
72 engine_client = MagicMock()
73 - engine_client.get_model_config = AsyncMock()
73
74 model_config = MagicMock()
75 model_config.hf_config.model_type = "test"
76 model_config.get_diff_sampling_param.return_value = {}
77 + engine_client.model_config = model_config
78 +
79 + engine_client.processor = MagicMock()
80 + engine_client.io_processor = MagicMock()
81
82 models = MagicMock()
83
@@ -83,7 +86,6 @@ class TestInitializeToolSessions:
86 # Create the actual instance
87 instance = OpenAIServingResponses(
88 engine_client=engine_client,
86 - model_config=model_config,
89 models=models,
90 request_logger=None,
91 chat_template=None,
@@ -132,18 +134,20 @@ class TestValidateGeneratorInput:
134 """Create a real OpenAIServingResponses instance for testing"""
135 # Create minimal mocks for required dependencies
136 engine_client = MagicMock()
135 - engine_client.get_model_config = AsyncMock()
137
138 model_config = MagicMock()
139 model_config.hf_config.model_type = "test"
140 model_config.get_diff_sampling_param.return_value = {}
141 + engine_client.model_config = model_config
142 +
143 + engine_client.processor = MagicMock()
144 + engine_client.io_processor = MagicMock()
145
146 models = MagicMock()
147
148 # Create the actual instance
149 instance = OpenAIServingResponses(
150 engine_client=engine_client,
146 - model_config=model_config,
151 models=models,
152 request_logger=None,
153 chat_template=None,
tests/test_inputs.py
+5 -3
@@ -7,6 +7,7 @@ from vllm.config import ModelConfig
7 from vllm.inputs import zip_enc_dec_prompts
8 from vllm.inputs.parse import parse_raw_prompts
9 from vllm.inputs.preprocess import InputPreprocessor
10 +from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
11
12 pytestmark = pytest.mark.cpu_test
13
@@ -106,7 +107,8 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
107 )
108 def test_preprocessor_text_no_mm_inputs(model_id, prompt):
109 model_config = ModelConfig(model=model_id)
109 - input_preprocessor = InputPreprocessor(model_config)
110 + tokenizer = init_tokenizer_from_configs(model_config)
111 + input_preprocessor = InputPreprocessor(model_config, tokenizer)
112
113 with pytest.raises(ValueError, match="does not support multimodal inputs"):
114 input_preprocessor.preprocess(prompt)
@@ -127,8 +129,8 @@ def test_preprocessor_text_no_mm_inputs(model_id, prompt):
129 )
130 def test_preprocessor_always_mm_code_path(model_id, prompt):
131 model_config = ModelConfig(model=model_id)
130 - input_preprocessor = InputPreprocessor(model_config)
131 - tokenizer = input_preprocessor.tokenizer
132 + tokenizer = init_tokenizer_from_configs(model_config)
133 + input_preprocessor = InputPreprocessor(model_config, tokenizer)
134
135 # HF processor adds sep token
136 sep_token_id = tokenizer.vocab[tokenizer.sep_token]
tests/v1/engine/test_processor_multi_modal_uuids.py
+1 -1
@@ -65,7 +65,7 @@ def _mk_processor(
65 device_config=DeviceConfig(device="cpu"),
66 )
67
68 - return Processor(vllm_config)
68 + return Processor(vllm_config, tokenizer=None)
69
70
71 def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):
tests/v1/sample/test_logprobs.py
+1 -1
@@ -459,7 +459,7 @@ def test_all_logprobs(example_prompts):
459 results_logprobs_all = runner.llm.generate(
460 example_prompts, sampling_params=sampling_params_logprobs_all
461 )
462 - vocab_size = runner.llm.llm_engine.get_model_config().get_vocab_size()
462 + vocab_size = runner.llm.llm_engine.model_config.get_vocab_size()
463
464 for i in range(len(results_logprobs_all)):
465 logprobs = results_logprobs_all[i].outputs[0].logprobs
vllm/benchmarks/throughput.py
+1 -1
@@ -186,7 +186,7 @@ async def run_vllm_async(
186 engine_args,
187 disable_frontend_multiprocessing=disable_frontend_multiprocessing,
188 ) as llm:
189 - model_config = await llm.get_model_config()
189 + model_config = llm.model_config
190 assert all(
191 model_config.max_model_len
192 >= (request.prompt_len + request.expected_output_len)
vllm/engine/protocol.py
+11 -201
@@ -1,26 +1,23 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -import asyncio
4 from abc import ABC, abstractmethod
5 from collections.abc import AsyncGenerator, Iterable, Mapping
6 from typing import Any, Optional, Union
7
9 -from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
8 from vllm.config import ModelConfig, VllmConfig
11 -from vllm.inputs.data import PromptType, TokensPrompt
12 -from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
13 -from vllm.inputs.preprocess import InputPreprocessor
9 +from vllm.inputs.data import PromptType
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
16 -from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
17 -from vllm.plugins.io_processors.interface import IOProcessor
12 +from vllm.outputs import PoolingRequestOutput, RequestOutput
13 +from vllm.plugins.io_processors import IOProcessor
14 from vllm.pooling_params import PoolingParams
19 -from vllm.sampling_params import BeamSearchParams, SamplingParams
15 +from vllm.sampling_params import SamplingParams
16 from vllm.tasks import SupportedTask
17 from vllm.transformers_utils.tokenizer import AnyTokenizer
22 -from vllm.utils import Device, collect_from_async_generator, random_uuid
18 +from vllm.utils import Device
19 from vllm.v1.engine import EngineCoreRequest
20 +from vllm.v1.engine.processor import Processor
21
22 logger = init_logger(__name__)
23
@@ -28,6 +25,11 @@ logger = init_logger(__name__)
25 class EngineClient(ABC):
26 """Protocol class for Clients to Engine"""
27
28 + vllm_config: VllmConfig
29 + model_config: ModelConfig
30 + processor: Processor
31 + io_processor: Optional[IOProcessor]
32 +
33 @property
34 @abstractmethod
35 def is_running(self) -> bool: ...
@@ -61,180 +63,6 @@ class EngineClient(ABC):
63 """Generate outputs for a request."""
64 ...
65
64 - async def beam_search(
65 - self,
66 - prompt: PromptType,
67 - request_id: str,
68 - params: BeamSearchParams,
69 - lora_request: Optional[LoRARequest] = None,
70 - ) -> AsyncGenerator[RequestOutput, None]:
71 - beam_width = params.beam_width
72 - max_tokens = params.max_tokens
73 - ignore_eos = params.ignore_eos
74 - temperature = params.temperature
75 - length_penalty = params.length_penalty
76 - include_stop_str_in_output = params.include_stop_str_in_output
77 -
78 - preprocessor = await self.get_input_preprocessor()
79 - tokenizer = preprocessor.get_tokenizer()
80 - eos_token_id = tokenizer.eos_token_id
81 -
82 - if is_explicit_encoder_decoder_prompt(prompt):
83 - raise NotImplementedError
84 - else:
85 - processed_inputs = preprocessor._prompt_to_llm_inputs(prompt)
86 -
87 - if processed_inputs["type"] == "embeds":
88 - raise NotImplementedError
89 -
90 - # This is a workaround to fix multimodal beam search; this is a
91 - # bandaid fix for 2 small problems:
92 - # 1. Multi_modal_data on the processed_inputs currently resolves to
93 - # `None`.
94 - # 2. preprocessing above expands the multimodal placeholders. However,
95 - # this happens again in generation, so the double expansion causes
96 - # a mismatch.
97 - # TODO - would be ideal to handle this more gracefully.
98 - if isinstance(prompt, str):
99 - prompt_text = prompt
100 - prompt_token_ids = []
101 - multi_modal_data = None
102 - else:
103 - prompt_text = prompt.get("prompt")
104 - prompt_token_ids = prompt.get("prompt_token_ids", [])
105 - multi_modal_data = prompt.get("multi_modal_data")
106 -
107 - mm_processor_kwargs = processed_inputs.get("mm_processor_kwargs")
108 -
109 - tokenized_length = len(prompt_token_ids)
110 -
111 - sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
112 -
113 - beam_search_params = SamplingParams(
114 - logprobs=2 * beam_width,
115 - max_tokens=1,
116 - temperature=temperature,
117 - )
118 - all_beams = [
119 - BeamSearchSequence(
120 - tokens=prompt_token_ids,
121 - cum_logprob=0,
122 - logprobs=[],
123 - multi_modal_data=multi_modal_data,
124 - mm_processor_kwargs=mm_processor_kwargs,
125 - lora_request=lora_request,
126 - )
127 - ]
128 - completed = []
129 -
130 - for _ in range(max_tokens):
131 - prompts_batch, lora_req_batch = zip(
132 - *[
133 - (
134 - TokensPrompt(
135 - prompt_token_ids=beam.tokens,
136 - multi_modal_data=beam.multi_modal_data,
137 - mm_processor_kwargs=beam.mm_processor_kwargs,
138 - ),
139 - beam.lora_request,
140 - )
141 - for beam in all_beams
142 - ]
143 - )
144 -
145 - tasks = []
146 -
147 - request_id = f"beam_search-{random_uuid()}"
148 - for i, (individual_prompt, lora_req) in enumerate(
149 - zip(prompts_batch, lora_req_batch)
150 - ):
151 - request_id_item = f"{request_id}-{i}"
152 - task = asyncio.create_task(
153 - collect_from_async_generator(
154 - self.generate(
155 - individual_prompt,
156 - beam_search_params,
157 - request_id_item,
158 - lora_request=lora_req,
159 - )
160 - )
161 - )
162 - tasks.append(task)
163 -
164 - output = await asyncio.gather(*tasks)
165 -
166 - output = [x[0] for x in output]
167 -
168 - new_beams = []
169 - for i, current_beam in enumerate(all_beams):
170 - result = output[i]
171 -
172 - if result.outputs[0].logprobs is not None:
173 - logprobs = result.outputs[0].logprobs[0]
174 - for token_id, logprob_obj in logprobs.items():
175 - if token_id == eos_token_id and not ignore_eos:
176 - completed.append(
177 - BeamSearchSequence(
178 - tokens=current_beam.tokens + [token_id]
179 - if include_stop_str_in_output
180 - else current_beam.tokens,
181 - logprobs=current_beam.logprobs + [logprobs],
182 - cum_logprob=current_beam.cum_logprob
183 - + logprob_obj.logprob,
184 - finish_reason="stop",
185 - stop_reason=eos_token_id,
186 - )
187 - )
188 - else:
189 - new_beams.append(
190 - BeamSearchSequence(
191 - tokens=current_beam.tokens + [token_id],
192 - logprobs=current_beam.logprobs + [logprobs],
193 - lora_request=current_beam.lora_request,
194 - cum_logprob=current_beam.cum_logprob
195 - + logprob_obj.logprob,
196 - multi_modal_data=current_beam.multi_modal_data,
197 - mm_processor_kwargs=current_beam.mm_processor_kwargs,
198 - )
199 - )
200 -
201 - sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
202 - all_beams = sorted_beams[:beam_width]
203 -
204 - completed.extend(all_beams)
205 - sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
206 - best_beams = sorted_completed[:beam_width]
207 -
208 - for beam in best_beams:
209 - if beam.tokens[-1] == eos_token_id and not ignore_eos:
210 - # Skip the eos token in the text.
211 - tokens = beam.tokens[tokenized_length:-1]
212 - else:
213 - tokens = beam.tokens[tokenized_length:]
214 - beam.text = tokenizer.decode(tokens)
215 -
216 - yield RequestOutput(
217 - request_id=request_id,
218 - prompt=prompt_text,
219 - outputs=[
220 - CompletionOutput(
221 - text=beam.text,
222 - cumulative_logprob=beam.cum_logprob,
223 - token_ids=beam.tokens[tokenized_length:],
224 - index=i,
225 - logprobs=beam.logprobs,
226 - finish_reason=beam.finish_reason
227 - if beam.finish_reason is not None
228 - else "length",
229 - stop_reason=beam.stop_reason,
230 - )
231 - for (i, beam) in enumerate(best_beams)
232 - ],
233 - finished=True,
234 - prompt_token_ids=prompt_token_ids,
235 - prompt_logprobs=None,
236 - )
237 -
66 @abstractmethod
67 def encode(
68 self,
@@ -259,29 +87,11 @@ class EngineClient(ABC):
87 """
88 ...
89
262 - @abstractmethod
263 - async def get_vllm_config(self) -> VllmConfig:
264 - """Get the vllm configuration of the vLLM engine."""
265 - ...
266 -
267 - @abstractmethod
268 - async def get_model_config(self) -> ModelConfig:
269 - """Get the model configuration of the vLLM engine."""
270 - ...
271 -
272 - @abstractmethod
273 - async def get_input_preprocessor(self) -> InputPreprocessor:
274 - """Get the input processor of the vLLM engine."""
275 - ...
276 -
90 @abstractmethod
91 async def get_tokenizer(self) -> AnyTokenizer:
92 """Get the tokenizer"""
93 ...
94
282 - async def get_io_processor(self) -> IOProcessor:
283 - raise NotImplementedError
284 -
95 @abstractmethod
96 async def is_tracing_enabled(self) -> bool: ...
97
vllm/entrypoints/llm.py
+17 -42
@@ -66,7 +66,6 @@ from vllm.outputs import (
66 RequestOutput,
67 ScoringRequestOutput,
68 )
69 -from vllm.plugins.io_processors import get_io_processor
69 from vllm.pooling_params import PoolingParams
70 from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams
71 from vllm.tasks import PoolingTask
@@ -79,7 +78,6 @@ from vllm.usage.usage_lib import UsageContext
78 from vllm.utils import Counter, Device, as_iter, is_list_of
79 from vllm.v1.engine import EngineCoreRequest
80 from vllm.v1.engine.llm_engine import LLMEngine
82 -from vllm.v1.engine.processor import Processor
81 from vllm.v1.sample.logits_processor import LogitsProcessor
82
83 if TYPE_CHECKING:
@@ -335,21 +333,13 @@ class LLM:
333 self.request_counter = Counter()
334 self.default_sampling_params: Union[dict[str, Any], None] = None
335
338 - supported_tasks = self.llm_engine.get_supported_tasks() # type: ignore
339 -
340 - logger.info("Supported_tasks: %s", supported_tasks)
341 -
336 + supported_tasks = self.llm_engine.get_supported_tasks()
337 + logger.info("Supported tasks: %s", supported_tasks)
338 self.supported_tasks = supported_tasks
339
344 - # Load the Input/Output processor plugin if any
345 - io_processor_plugin = self.llm_engine.model_config.io_processor_plugin
346 - self.io_processor = get_io_processor(
347 - self.llm_engine.vllm_config, io_processor_plugin
348 - )
349 -
350 - @property
351 - def model_config(self):
352 - return self.llm_engine.model_config
340 + self.model_config = self.llm_engine.model_config
341 + self.processor = self.llm_engine.processor
342 + self.io_processor = self.llm_engine.io_processor
343
344 def get_tokenizer(self) -> AnyTokenizer:
345 return self.llm_engine.get_tokenizer()
@@ -364,18 +354,9 @@ class LLM:
354 else:
355 self.llm_engine.tokenizer = get_cached_tokenizer(tokenizer)
356
367 - def _get_processor(self) -> Processor:
368 - if not hasattr(self, "_processor"):
369 - vllm_config = self.llm_engine.vllm_config
370 - self._processor = Processor(vllm_config)
371 -
372 - return self._processor
373 -
357 def get_default_sampling_params(self) -> SamplingParams:
358 if self.default_sampling_params is None:
376 - self.default_sampling_params = (
377 - self.llm_engine.model_config.get_diff_sampling_param()
378 - )
359 + self.default_sampling_params = self.model_config.get_diff_sampling_param()
360 if self.default_sampling_params:
361 return SamplingParams.from_optional(**self.default_sampling_params)
362 return SamplingParams()
@@ -423,7 +404,7 @@ class LLM:
404 considered legacy and may be deprecated in the future. You should
405 instead pass them via the `inputs` parameter.
406 """
426 - model_config = self.llm_engine.model_config
407 + model_config = self.model_config
408 runner_type = model_config.runner_type
409 if runner_type != "generate":
410 raise ValueError(
@@ -463,7 +444,7 @@ class LLM:
444 # isn't multimodal, leave the lora as is.
445 if (
446 lora_config is None
466 - or not self.llm_engine.model_config.is_multimodal_model
447 + or not self.model_config.is_multimodal_model
448 or (lora_config and lora_config.default_mm_loras is None)
449 ):
450 return lora_request
@@ -495,15 +476,13 @@ class LLM:
476 if (
477 not default_mm_loras
478 or not isinstance(prompt, dict)
498 - or "multi_modal_data" not in prompt
479 + or not (mm_data := prompt.get("multi_modal_data") or {})
480 ):
481 return lora_request
482
502 - prompt = cast(Union[TextPrompt, TokensPrompt], prompt)
503 -
504 - intersection = set(prompt["multi_modal_data"].keys()).intersection(
505 - default_mm_loras.keys()
506 - )
483 + intersection = set(
484 + mm_data.keys() # type: ignore
485 + ).intersection(default_mm_loras.keys())
486 if not intersection:
487 return lora_request
488 if len(intersection) > 1:
@@ -819,7 +798,7 @@ class LLM:
798 list_of_messages = [cast(list[ChatCompletionMessageParam], messages)]
799
800 tokenizer = self.get_tokenizer()
822 - model_config = self.llm_engine.get_model_config()
801 + model_config = self.model_config
802 resolved_content_format = resolve_chat_template_content_format(
803 chat_template,
804 tools,
@@ -1031,7 +1010,7 @@ class LLM:
1010 pooling_task,
1011 )
1012
1034 - model_config = self.llm_engine.model_config
1013 + model_config = self.model_config
1014 runner_type = model_config.runner_type
1015 if runner_type != "pooling":
1016 raise ValueError(
@@ -1276,7 +1255,7 @@ class LLM:
1255 pooling_params: Optional[PoolingParams] = None,
1256 lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1257 ) -> list[ScoringRequestOutput]:
1279 - model_config = self.llm_engine.model_config
1258 + model_config = self.model_config
1259
1260 if isinstance(tokenizer, MistralTokenizer):
1261 raise ValueError("Score API is not supported for Mistral tokenizer")
@@ -1287,7 +1266,6 @@ class LLM:
1266 if pooling_params is None:
1267 pooling_params = PoolingParams(task="score")
1268
1290 - model_config = self.llm_engine.model_config
1269 pooling_params.verify("score", model_config)
1270 pooling_params_list = list[PoolingParams]()
1271
@@ -1301,8 +1279,6 @@ class LLM:
1279
1280 input_pairs = [(t1, t2) for t1, t2 in zip(data_1, data_2)]
1281
1304 - model_config = self.llm_engine.model_config
1305 -
1282 for q, d in input_pairs:
1283 _, engine_prompt = get_score_prompt(
1284 model_config=model_config,
@@ -1380,7 +1356,7 @@ class LLM:
1356 A list of `ScoringRequestOutput` objects containing the
1357 generated scores in the same order as the input prompts.
1358 """
1383 - model_config = self.llm_engine.model_config
1359 + model_config = self.model_config
1360 runner_type = model_config.runner_type
1361 if runner_type != "pooling":
1362 raise ValueError(
@@ -1658,8 +1634,7 @@ class LLM:
1634 tokenization_kwargs,
1635 )
1636
1661 - processor = self._get_processor()
1662 - engine_request = processor.process_inputs(
1637 + engine_request = self.processor.process_inputs(
1638 request_id,
1639 engine_prompt,
1640 params,
vllm/entrypoints/openai/api_server.py
+5 -18
@@ -1601,10 +1601,11 @@ def build_app(args: Namespace) -> FastAPI:
1601
1602 async def init_app_state(
1603 engine_client: EngineClient,
1604 - vllm_config: VllmConfig,
1604 state: State,
1605 args: Namespace,
1606 ) -> None:
1607 + vllm_config = engine_client.vllm_config
1608 +
1609 if args.served_model_name is not None:
1610 served_model_names = args.served_model_name
1611 else:
@@ -1622,11 +1623,9 @@ async def init_app_state(
1623 state.engine_client = engine_client
1624 state.log_stats = not args.disable_log_stats
1625 state.vllm_config = vllm_config
1625 - model_config = vllm_config.model_config
1626
1627 supported_tasks = await engine_client.get_supported_tasks()
1628 -
1629 - logger.info("Supported_tasks: %s", supported_tasks)
1628 + logger.info("Supported tasks: %s", supported_tasks)
1629
1630 resolved_chat_template = load_chat_template(args.chat_template)
1631 if resolved_chat_template is not None:
@@ -1688,7 +1687,6 @@ async def init_app_state(
1687
1688 state.openai_serving_models = OpenAIServingModels(
1689 engine_client=engine_client,
1691 - model_config=model_config,
1690 base_model_paths=base_model_paths,
1691 lora_modules=lora_modules,
1692 )
@@ -1696,7 +1694,6 @@ async def init_app_state(
1694 state.openai_serving_responses = (
1695 OpenAIServingResponses(
1696 engine_client,
1699 - model_config,
1697 state.openai_serving_models,
1698 request_logger=request_logger,
1699 chat_template=resolved_chat_template,
@@ -1717,7 +1714,6 @@ async def init_app_state(
1714 state.openai_serving_chat = (
1715 OpenAIServingChat(
1716 engine_client,
1720 - model_config,
1717 state.openai_serving_models,
1718 args.response_role,
1719 request_logger=request_logger,
@@ -1740,7 +1736,6 @@ async def init_app_state(
1736 state.openai_serving_completion = (
1737 OpenAIServingCompletion(
1738 engine_client,
1743 - model_config,
1739 state.openai_serving_models,
1740 request_logger=request_logger,
1741 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
@@ -1754,7 +1749,6 @@ async def init_app_state(
1749 state.openai_serving_pooling = (
1750 OpenAIServingPooling(
1751 engine_client,
1757 - vllm_config,
1752 state.openai_serving_models,
1753 request_logger=request_logger,
1754 chat_template=resolved_chat_template,
@@ -1768,7 +1762,6 @@ async def init_app_state(
1762 state.openai_serving_embedding = (
1763 OpenAIServingEmbedding(
1764 engine_client,
1771 - model_config,
1765 state.openai_serving_models,
1766 request_logger=request_logger,
1767 chat_template=resolved_chat_template,
@@ -1782,7 +1775,6 @@ async def init_app_state(
1775 state.openai_serving_classification = (
1776 ServingClassification(
1777 engine_client,
1785 - model_config,
1778 state.openai_serving_models,
1779 request_logger=request_logger,
1780 log_error_stack=args.log_error_stack,
@@ -1793,7 +1785,6 @@ async def init_app_state(
1785 state.openai_serving_scores = (
1786 ServingScores(
1787 engine_client,
1796 - model_config,
1788 state.openai_serving_models,
1789 request_logger=request_logger,
1790 log_error_stack=args.log_error_stack,
@@ -1803,7 +1794,6 @@ async def init_app_state(
1794 )
1795 state.openai_serving_tokenization = OpenAIServingTokenization(
1796 engine_client,
1806 - model_config,
1797 state.openai_serving_models,
1798 request_logger=request_logger,
1799 chat_template=resolved_chat_template,
@@ -1814,7 +1804,6 @@ async def init_app_state(
1804 state.openai_serving_transcription = (
1805 OpenAIServingTranscription(
1806 engine_client,
1817 - model_config,
1807 state.openai_serving_models,
1808 request_logger=request_logger,
1809 log_error_stack=args.log_error_stack,
@@ -1825,7 +1814,6 @@ async def init_app_state(
1814 state.openai_serving_translation = (
1815 OpenAIServingTranslation(
1816 engine_client,
1828 - model_config,
1817 state.openai_serving_models,
1818 request_logger=request_logger,
1819 log_error_stack=args.log_error_stack,
@@ -1946,12 +1934,11 @@ async def run_server_worker(
1934 maybe_register_tokenizer_info_endpoint(args)
1935 app = build_app(args)
1936
1949 - vllm_config = await engine_client.get_vllm_config()
1950 - await init_app_state(engine_client, vllm_config, app.state, args)
1937 + await init_app_state(engine_client, app.state, args)
1938
1939 logger.info(
1940 "Starting vLLM API server %d on %s",
1954 - vllm_config.parallel_config._api_process_rank,
1941 + engine_client.vllm_config.parallel_config._api_process_rank,
1942 listen_address,
1943 )
1944 shutdown_task = await serve_http(
vllm/entrypoints/openai/run_batch.py
+3 -12
@@ -14,7 +14,6 @@ import torch
14 from prometheus_client import start_http_server
15 from tqdm import tqdm
16
17 -from vllm.config import VllmConfig
17 from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
18 from vllm.engine.protocol import EngineClient
19 from vllm.entrypoints.logger import RequestLogger
@@ -328,7 +327,6 @@ async def run_request(
327
328 async def run_batch(
329 engine_client: EngineClient,
331 - vllm_config: VllmConfig,
330 args: Namespace,
331 ) -> None:
332 if args.served_model_name is not None:
@@ -345,22 +343,19 @@ async def run_batch(
343 BaseModelPath(name=name, model_path=args.model) for name in served_model_names
344 ]
345
348 - model_config = vllm_config.model_config
349 -
346 + model_config = engine_client.model_config
347 supported_tasks = await engine_client.get_supported_tasks()
351 - logger.info("Supported_tasks: %s", supported_tasks)
348 + logger.info("Supported tasks: %s", supported_tasks)
349
350 # Create the openai serving objects.
351 openai_serving_models = OpenAIServingModels(
352 engine_client=engine_client,
356 - model_config=model_config,
353 base_model_paths=base_model_paths,
354 lora_modules=None,
355 )
356 openai_serving_chat = (
357 OpenAIServingChat(
358 engine_client,
363 - model_config,
359 openai_serving_models,
360 args.response_role,
361 request_logger=request_logger,
@@ -374,7 +369,6 @@ async def run_batch(
369 openai_serving_embedding = (
370 OpenAIServingEmbedding(
371 engine_client,
377 - model_config,
372 openai_serving_models,
373 request_logger=request_logger,
374 chat_template=None,
@@ -392,7 +386,6 @@ async def run_batch(
386 openai_serving_scores = (
387 ServingScores(
388 engine_client,
395 - model_config,
389 openai_serving_models,
390 request_logger=request_logger,
391 )
@@ -509,9 +502,7 @@ async def main(args: Namespace):
502 usage_context=UsageContext.OPENAI_BATCH_RUNNER,
503 disable_frontend_multiprocessing=False,
504 ) as engine_client:
512 - vllm_config = await engine_client.get_vllm_config()
513 -
514 - await run_batch(engine_client, vllm_config, args)
505 + await run_batch(engine_client, args)
506
507
508 if __name__ == "__main__":
vllm/entrypoints/openai/serving_chat.py
+3 -6
@@ -15,7 +15,6 @@ from fastapi import Request
15 from openai_harmony import Message as OpenAIMessage
16 from pydantic import TypeAdapter
17
18 -from vllm.config import ModelConfig
18 from vllm.engine.protocol import EngineClient
19 from vllm.entrypoints.chat_utils import (
20 ChatTemplateContentFormatOption,
@@ -81,7 +80,6 @@ class OpenAIServingChat(OpenAIServing):
80 def __init__(
81 self,
82 engine_client: EngineClient,
84 - model_config: ModelConfig,
83 models: OpenAIServingModels,
84 response_role: str,
85 *,
@@ -101,7 +99,6 @@ class OpenAIServingChat(OpenAIServing):
99 ) -> None:
100 super().__init__(
101 engine_client=engine_client,
104 - model_config=model_config,
102 models=models,
103 request_logger=request_logger,
104 return_tokens_as_token_ids=return_tokens_as_token_ids,
@@ -138,7 +135,7 @@ class OpenAIServingChat(OpenAIServing):
135 self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
136 if self.enable_auto_tools:
137 try:
141 - if tool_parser == "pythonic" and model_config.model.startswith(
138 + if tool_parser == "pythonic" and self.model_config.model.startswith(
139 "meta-llama/Llama-3.2"
140 ):
141 logger.warning(
@@ -169,7 +166,7 @@ class OpenAIServingChat(OpenAIServing):
166 else:
167 self.tool_call_id_type = "random"
168
172 - self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
169 + self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
170 if self.use_harmony:
171 if "stop_token_ids" not in self.default_sampling_params:
172 self.default_sampling_params["stop_token_ids"] = []
@@ -338,7 +335,7 @@ class OpenAIServingChat(OpenAIServing):
335 )
336
337 if isinstance(sampling_params, BeamSearchParams):
341 - generator = self.engine_client.beam_search(
338 + generator = self.beam_search(
339 prompt=engine_prompt,
340 request_id=request_id,
341 params=sampling_params,
vllm/entrypoints/openai/serving_classification.py
-3
@@ -8,7 +8,6 @@ import numpy as np
8 from fastapi import Request
9 from typing_extensions import override
10
11 -from vllm.config import ModelConfig
11 from vllm.engine.protocol import EngineClient
12 from vllm.entrypoints.logger import RequestLogger
13 from vllm.entrypoints.openai.protocol import (
@@ -128,7 +127,6 @@ class ServingClassification(ClassificationMixin):
127 def __init__(
128 self,
129 engine_client: EngineClient,
131 - model_config: ModelConfig,
130 models: OpenAIServingModels,
131 *,
132 request_logger: Optional[RequestLogger],
@@ -136,7 +134,6 @@ class ServingClassification(ClassificationMixin):
134 ) -> None:
135 super().__init__(
136 engine_client=engine_client,
139 - model_config=model_config,
137 models=models,
138 request_logger=request_logger,
139 log_error_stack=log_error_stack,
vllm/entrypoints/openai/serving_completion.py
+1 -4
@@ -10,7 +10,6 @@ from typing import Optional, Union, cast
10 import jinja2
11 from fastapi import Request
12
13 -from vllm.config import ModelConfig
13 from vllm.engine.protocol import EngineClient
14 from vllm.entrypoints.logger import RequestLogger
15 from vllm.entrypoints.openai.protocol import (
@@ -44,7 +43,6 @@ class OpenAIServingCompletion(OpenAIServing):
43 def __init__(
44 self,
45 engine_client: EngineClient,
47 - model_config: ModelConfig,
46 models: OpenAIServingModels,
47 *,
48 request_logger: Optional[RequestLogger],
@@ -55,7 +53,6 @@ class OpenAIServingCompletion(OpenAIServing):
53 ):
54 super().__init__(
55 engine_client=engine_client,
58 - model_config=model_config,
56 models=models,
57 request_logger=request_logger,
58 return_tokens_as_token_ids=return_tokens_as_token_ids,
@@ -201,7 +198,7 @@ class OpenAIServingCompletion(OpenAIServing):
198 # but pre-commit in CI fails without it.
199 engine_prompt = cast(Union[EmbedsPrompt, TokensPrompt], engine_prompt)
200 if isinstance(sampling_params, BeamSearchParams):
204 - generator = self.engine_client.beam_search(
201 + generator = self.beam_search(
202 prompt=engine_prompt,
203 request_id=request_id,
204 params=sampling_params,
vllm/entrypoints/openai/serving_embedding.py
-3
@@ -10,7 +10,6 @@ import torch
10 from fastapi import Request
11 from typing_extensions import assert_never, override
12
13 -from vllm.config import ModelConfig
13 from vllm.engine.protocol import EngineClient
14 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
15 from vllm.entrypoints.logger import RequestLogger
@@ -597,7 +596,6 @@ class OpenAIServingEmbedding(EmbeddingMixin):
596 def __init__(
597 self,
598 engine_client: EngineClient,
600 - model_config: ModelConfig,
599 models: OpenAIServingModels,
600 *,
601 request_logger: Optional[RequestLogger],
@@ -608,7 +606,6 @@ class OpenAIServingEmbedding(EmbeddingMixin):
606 ) -> None:
607 super().__init__(
608 engine_client=engine_client,
611 - model_config=model_config,
609 models=models,
610 request_logger=request_logger,
611 log_error_stack=log_error_stack,
vllm/entrypoints/openai/serving_engine.py
+199 -17
@@ -1,5 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import asyncio
4 import json
5 import sys
6 import time
@@ -15,17 +16,13 @@ from pydantic import BaseModel, ConfigDict, Field
16 from starlette.datastructures import Headers
17 from typing_extensions import TypeIs
18
18 -from vllm.entrypoints.utils import _validate_truncation_size
19 -from vllm.v1.engine import EngineCoreRequest
20 -from vllm.v1.engine.processor import Processor
21 -
19 if sys.version_info >= (3, 12):
20 from typing import TypedDict
21 else:
22 from typing_extensions import TypedDict
23
24 import vllm.envs as envs
28 -from vllm.config import ModelConfig
25 +from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
26 from vllm.engine.protocol import EngineClient
27 from vllm.entrypoints.chat_utils import (
28 ChatCompletionMessageParam,
@@ -68,9 +65,14 @@ from vllm.entrypoints.openai.protocol import (
65 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
66 from vllm.entrypoints.openai.tool_parsers import ToolParser
67 from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
68 +from vllm.entrypoints.utils import _validate_truncation_size
69 from vllm.inputs.data import PromptType
70 from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
73 -from vllm.inputs.parse import PromptComponents, get_prompt_components
71 +from vllm.inputs.parse import (
72 + PromptComponents,
73 + get_prompt_components,
74 + is_explicit_encoder_decoder_prompt,
75 +)
76 from vllm.logger import init_logger
77 from vllm.logprobs import Logprob, PromptLogprobs
78 from vllm.lora.request import LoRARequest
@@ -78,7 +80,7 @@ from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error
80 MultiModalDataDict,
81 MultiModalUUIDDict,
82 )
81 -from vllm.outputs import PoolingRequestOutput, RequestOutput
83 +from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
84 from vllm.pooling_params import PoolingParams
85 from vllm.sampling_params import BeamSearchParams, SamplingParams
86 from vllm.tracing import (
@@ -89,11 +91,13 @@ from vllm.tracing import (
91 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
92 from vllm.utils import (
93 AsyncMicrobatchTokenizer,
94 + collect_from_async_generator,
95 is_list_of,
96 make_async,
97 merge_async_iterators,
98 random_uuid,
99 )
100 +from vllm.v1.engine import EngineCoreRequest
101
102 logger = init_logger(__name__)
103
@@ -240,7 +244,6 @@ class OpenAIServing:
244 def __init__(
245 self,
246 engine_client: EngineClient,
243 - model_config: ModelConfig,
247 models: OpenAIServingModels,
248 *,
249 request_logger: Optional[RequestLogger],
@@ -251,8 +254,6 @@ class OpenAIServing:
254 super().__init__()
255
256 self.engine_client = engine_client
254 - self.model_config = model_config
255 - self.max_model_len = model_config.max_model_len
257
258 self.models = models
259
@@ -268,12 +269,194 @@ class OpenAIServing:
269 self._async_tokenizer_pool: dict[AnyTokenizer, AsyncMicrobatchTokenizer] = {}
270 self.log_error_stack = log_error_stack
271
271 - async def _get_processor(self) -> Processor:
272 - if not hasattr(self, "_processor"):
273 - vllm_config = await self.engine_client.get_vllm_config()
274 - self._processor = Processor(vllm_config)
272 + self.processor = self.models.processor
273 + self.io_processor = self.models.io_processor
274 + self.model_config = self.models.model_config
275 + self.max_model_len = self.model_config.max_model_len
276 +
277 + async def beam_search(
278 + self,
279 + prompt: PromptType,
280 + request_id: str,
281 + params: BeamSearchParams,
282 + lora_request: Optional[LoRARequest] = None,
283 + ) -> AsyncGenerator[RequestOutput, None]:
284 + beam_width = params.beam_width
285 + max_tokens = params.max_tokens
286 + ignore_eos = params.ignore_eos
287 + temperature = params.temperature
288 + length_penalty = params.length_penalty
289 + include_stop_str_in_output = params.include_stop_str_in_output
290 +
291 + processor = self.processor
292 + tokenizer = processor.tokenizer
293 + if tokenizer is None:
294 + raise ValueError(
295 + "You cannot use beam search when `skip_tokenizer_init` is True"
296 + )
297 +
298 + eos_token_id: int = tokenizer.eos_token_id # type: ignore
299 +
300 + if is_explicit_encoder_decoder_prompt(prompt):
301 + raise NotImplementedError
302 + else:
303 + processed_inputs = processor.input_preprocessor._prompt_to_llm_inputs(
304 + prompt
305 + )
306
276 - return self._processor
307 + if processed_inputs["type"] == "embeds":
308 + raise NotImplementedError
309 +
310 + # This is a workaround to fix multimodal beam search; this is a
311 + # bandaid fix for 2 small problems:
312 + # 1. Multi_modal_data on the processed_inputs currently resolves to
313 + # `None`.
314 + # 2. preprocessing above expands the multimodal placeholders. However,
315 + # this happens again in generation, so the double expansion causes
316 + # a mismatch.
317 + # TODO - would be ideal to handle this more gracefully.
318 + prompt_text: Optional[str]
319 + prompt_token_ids: list[int]
320 + multi_modal_data: Optional[MultiModalDataDict]
321 + if isinstance(prompt, str):
322 + prompt_text = prompt
323 + prompt_token_ids = []
324 + multi_modal_data = None
325 + else:
326 + prompt_text = prompt.get("prompt") # type: ignore
327 + prompt_token_ids = prompt.get("prompt_token_ids", []) # type: ignore
328 + multi_modal_data = prompt.get("multi_modal_data") # type: ignore
329 +
330 + mm_processor_kwargs: Optional[dict[str, Any]] = processed_inputs.get(
331 + "mm_processor_kwargs"
332 + ) # type: ignore
333 +
334 + tokenized_length = len(prompt_token_ids)
335 +
336 + sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
337 +
338 + beam_search_params = SamplingParams(
339 + logprobs=2 * beam_width,
340 + max_tokens=1,
341 + temperature=temperature,
342 + )
343 + all_beams = [
344 + BeamSearchSequence(
345 + tokens=prompt_token_ids,
346 + cum_logprob=0,
347 + logprobs=[],
348 + multi_modal_data=multi_modal_data,
349 + mm_processor_kwargs=mm_processor_kwargs,
350 + lora_request=lora_request,
351 + )
352 + ]
353 + completed = []
354 +
355 + for _ in range(max_tokens):
356 + prompts_batch, lora_req_batch = zip(
357 + *[
358 + (
359 + EngineTokensPrompt(
360 + prompt_token_ids=beam.tokens,
361 + multi_modal_data=beam.multi_modal_data,
362 + mm_processor_kwargs=beam.mm_processor_kwargs,
363 + ),
364 + beam.lora_request,
365 + )
366 + for beam in all_beams
367 + ]
368 + )
369 +
370 + tasks = []
371 + request_id_batch = f"{request_id}-{random_uuid()}"
372 +
373 + for i, (individual_prompt, lora_req) in enumerate(
374 + zip(prompts_batch, lora_req_batch)
375 + ):
376 + request_id_item = f"{request_id_batch}-beam-{i}"
377 + task = asyncio.create_task(
378 + collect_from_async_generator(
379 + self.engine_client.generate(
380 + individual_prompt,
381 + beam_search_params,
382 + request_id_item,
383 + lora_request=lora_req,
384 + )
385 + )
386 + )
387 + tasks.append(task)
388 +
389 + output = [x[0] for x in await asyncio.gather(*tasks)]
390 +
391 + new_beams = []
392 + for i, current_beam in enumerate(all_beams):
393 + result = output[i]
394 +
395 + if result.outputs[0].logprobs is not None:
396 + logprobs = result.outputs[0].logprobs[0]
397 + for token_id, logprob_obj in logprobs.items():
398 + if token_id == eos_token_id and not ignore_eos:
399 + completed.append(
400 + BeamSearchSequence(
401 + tokens=current_beam.tokens + [token_id]
402 + if include_stop_str_in_output
403 + else current_beam.tokens,
404 + logprobs=current_beam.logprobs + [logprobs],
405 + cum_logprob=current_beam.cum_logprob
406 + + logprob_obj.logprob,
407 + finish_reason="stop",
408 + stop_reason=eos_token_id,
409 + )
410 + )
411 + else:
412 + new_beams.append(
413 + BeamSearchSequence(
414 + tokens=current_beam.tokens + [token_id],
415 + logprobs=current_beam.logprobs + [logprobs],
416 + lora_request=current_beam.lora_request,
417 + cum_logprob=current_beam.cum_logprob
418 + + logprob_obj.logprob,
419 + multi_modal_data=current_beam.multi_modal_data,
420 + mm_processor_kwargs=current_beam.mm_processor_kwargs,
421 + )
422 + )
423 +
424 + sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
425 + all_beams = sorted_beams[:beam_width]
426 +
427 + completed.extend(all_beams)
428 + sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
429 + best_beams = sorted_completed[:beam_width]
430 +
431 + for beam in best_beams:
432 + if beam.tokens[-1] == eos_token_id and not ignore_eos:
433 + # Skip the eos token in the text.
434 + tokens = beam.tokens[tokenized_length:-1]
435 + else:
436 + tokens = beam.tokens[tokenized_length:]
437 + beam.text = tokenizer.decode(tokens)
438 +
439 + yield RequestOutput(
440 + request_id=request_id,
441 + prompt=prompt_text,
442 + outputs=[
443 + CompletionOutput(
444 + text=beam.text, # type: ignore
445 + cumulative_logprob=beam.cum_logprob,
446 + token_ids=beam.tokens[tokenized_length:],
447 + index=i,
448 + logprobs=beam.logprobs,
449 + finish_reason=beam.finish_reason
450 + if beam.finish_reason is not None
451 + else "length",
452 + stop_reason=beam.stop_reason,
453 + )
454 + for (i, beam) in enumerate(best_beams)
455 + ],
456 + finished=True,
457 + prompt_token_ids=prompt_token_ids,
458 + prompt_logprobs=None,
459 + )
460
461 def _get_renderer(self, tokenizer: Optional[AnyTokenizer]) -> BaseRenderer:
462 """
@@ -938,8 +1121,7 @@ class OpenAIServing:
1121 self.max_model_len, params.truncate_prompt_tokens, tokenization_kwargs
1122 )
1123
941 - processor = await self._get_processor()
942 - engine_request = processor.process_inputs(
1124 + engine_request = self.processor.process_inputs(
1125 request_id,
1126 engine_prompt,
1127 params,
vllm/entrypoints/openai/serving_models.py
+6 -6
@@ -7,7 +7,6 @@ from dataclasses import dataclass
7 from http import HTTPStatus
8 from typing import Optional, Union
9
10 -from vllm.config import ModelConfig
10 from vllm.engine.protocol import EngineClient
11 from vllm.entrypoints.openai.protocol import (
12 ErrorInfo,
@@ -51,18 +50,14 @@ class OpenAIServingModels:
50 def __init__(
51 self,
52 engine_client: EngineClient,
54 - model_config: ModelConfig,
53 base_model_paths: list[BaseModelPath],
54 *,
55 lora_modules: Optional[list[LoRAModulePath]] = None,
56 ):
57 super().__init__()
58
61 - self.base_model_paths = base_model_paths
62 -
63 - self.max_model_len = model_config.max_model_len
59 self.engine_client = engine_client
65 - self.model_config = model_config
60 + self.base_model_paths = base_model_paths
61
62 self.static_lora_modules = lora_modules
63 self.lora_requests: dict[str, LoRARequest] = {}
@@ -75,6 +70,11 @@ class OpenAIServingModels:
70 )
71 self.lora_resolver_lock: dict[str, Lock] = defaultdict(Lock)
72
73 + self.processor = self.engine_client.processor
74 + self.io_processor = self.engine_client.io_processor
75 + self.model_config = self.engine_client.model_config
76 + self.max_model_len = self.model_config.max_model_len
77 +
78 async def init_static_loras(self):
79 """Loads all static LoRA modules.
80 Raises if any fail to load"""
vllm/entrypoints/openai/serving_pooling.py
-6
@@ -13,7 +13,6 @@ import torch
13 from fastapi import Request
14 from typing_extensions import assert_never
15
16 -from vllm.config import VllmConfig
16 from vllm.engine.protocol import EngineClient
17 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
18 from vllm.entrypoints.logger import RequestLogger
@@ -34,7 +33,6 @@ from vllm.entrypoints.renderer import RenderConfig
33 from vllm.entrypoints.utils import _validate_truncation_size
34 from vllm.logger import init_logger
35 from vllm.outputs import PoolingOutput, PoolingRequestOutput
37 -from vllm.plugins.io_processors import get_io_processor
36 from vllm.utils import merge_async_iterators
37
38 logger = init_logger(__name__)
@@ -60,7 +58,6 @@ class OpenAIServingPooling(OpenAIServing):
58 def __init__(
59 self,
60 engine_client: EngineClient,
63 - vllm_config: VllmConfig,
61 models: OpenAIServingModels,
62 *,
63 request_logger: Optional[RequestLogger],
@@ -71,7 +68,6 @@ class OpenAIServingPooling(OpenAIServing):
68 ) -> None:
69 super().__init__(
70 engine_client=engine_client,
74 - model_config=vllm_config.model_config,
71 models=models,
72 request_logger=request_logger,
73 log_error_stack=log_error_stack,
@@ -80,8 +76,6 @@ class OpenAIServingPooling(OpenAIServing):
76 self.chat_template = chat_template
77 self.chat_template_content_format: Final = chat_template_content_format
78 self.trust_request_chat_template = trust_request_chat_template
83 - io_processor_plugin = self.model_config.io_processor_plugin
84 - self.io_processor = get_io_processor(vllm_config, io_processor_plugin)
79
80 async def create_pooling(
81 self,
vllm/entrypoints/openai/serving_responses.py
+1 -4
@@ -49,7 +49,6 @@ from openai.types.responses.response_reasoning_item import (
49 from openai_harmony import Message as OpenAIHarmonyMessage
50
51 from vllm import envs
52 -from vllm.config import ModelConfig
52 from vllm.engine.protocol import EngineClient
53 from vllm.entrypoints.chat_utils import (
54 ChatCompletionMessageParam,
@@ -109,7 +108,6 @@ class OpenAIServingResponses(OpenAIServing):
108 def __init__(
109 self,
110 engine_client: EngineClient,
112 - model_config: ModelConfig,
111 models: OpenAIServingModels,
112 *,
113 request_logger: Optional[RequestLogger],
@@ -127,7 +125,6 @@ class OpenAIServingResponses(OpenAIServing):
125 ) -> None:
126 super().__init__(
127 engine_client=engine_client,
130 - model_config=model_config,
128 models=models,
129 request_logger=request_logger,
130 return_tokens_as_token_ids=return_tokens_as_token_ids,
@@ -176,7 +173,7 @@ class OpenAIServingResponses(OpenAIServing):
173 "the store."
174 )
175
179 - self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
176 + self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
177 if self.use_harmony:
178 logger.warning(
179 "For gpt-oss, we ignore --enable-auto-tool-choice "
vllm/entrypoints/openai/serving_score.py
-3
@@ -7,7 +7,6 @@ from typing import Any, Optional, Union
7
8 from fastapi import Request
9
10 -from vllm.config import ModelConfig
10 from vllm.engine.protocol import EngineClient
11 from vllm.entrypoints.logger import RequestLogger
12 from vllm.entrypoints.openai.protocol import (
@@ -47,7 +46,6 @@ class ServingScores(OpenAIServing):
46 def __init__(
47 self,
48 engine_client: EngineClient,
50 - model_config: ModelConfig,
49 models: OpenAIServingModels,
50 *,
51 request_logger: Optional[RequestLogger],
@@ -55,7 +53,6 @@ class ServingScores(OpenAIServing):
53 ) -> None:
54 super().__init__(
55 engine_client=engine_client,
58 - model_config=model_config,
56 models=models,
57 request_logger=request_logger,
58 log_error_stack=log_error_stack,
vllm/entrypoints/openai/serving_tokenization.py
-3
@@ -6,7 +6,6 @@ from typing import Any, Final, Optional, Union
6 import jinja2
7 from fastapi import Request
8
9 -from vllm.config import ModelConfig
9 from vllm.engine.protocol import EngineClient
10 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
11 from vllm.entrypoints.logger import RequestLogger
@@ -32,7 +31,6 @@ class OpenAIServingTokenization(OpenAIServing):
31 def __init__(
32 self,
33 engine_client: EngineClient,
35 - model_config: ModelConfig,
34 models: OpenAIServingModels,
35 *,
36 request_logger: Optional[RequestLogger],
@@ -43,7 +41,6 @@ class OpenAIServingTokenization(OpenAIServing):
41 ) -> None:
42 super().__init__(
43 engine_client=engine_client,
46 - model_config=model_config,
44 models=models,
45 request_logger=request_logger,
46 log_error_stack=log_error_stack,
vllm/entrypoints/openai/serving_transcription.py
-5
@@ -5,7 +5,6 @@ from typing import Optional, Union
5
6 from fastapi import Request
7
8 -from vllm.config import ModelConfig
8 from vllm.engine.protocol import EngineClient
9 from vllm.entrypoints.logger import RequestLogger
10 from vllm.entrypoints.openai.protocol import (
@@ -34,7 +33,6 @@ class OpenAIServingTranscription(OpenAISpeechToText):
33 def __init__(
34 self,
35 engine_client: EngineClient,
37 - model_config: ModelConfig,
36 models: OpenAIServingModels,
37 *,
38 request_logger: Optional[RequestLogger],
@@ -43,7 +41,6 @@ class OpenAIServingTranscription(OpenAISpeechToText):
41 ):
42 super().__init__(
43 engine_client=engine_client,
46 - model_config=model_config,
44 models=models,
45 request_logger=request_logger,
46 return_tokens_as_token_ids=return_tokens_as_token_ids,
@@ -95,7 +92,6 @@ class OpenAIServingTranslation(OpenAISpeechToText):
92 def __init__(
93 self,
94 engine_client: EngineClient,
98 - model_config: ModelConfig,
95 models: OpenAIServingModels,
96 *,
97 request_logger: Optional[RequestLogger],
@@ -104,7 +100,6 @@ class OpenAIServingTranslation(OpenAISpeechToText):
100 ):
101 super().__init__(
102 engine_client=engine_client,
107 - model_config=model_config,
103 models=models,
104 request_logger=request_logger,
105 return_tokens_as_token_ids=return_tokens_as_token_ids,
vllm/entrypoints/openai/speech_to_text.py
+1 -4
@@ -12,7 +12,6 @@ import numpy as np
12 from fastapi import Request
13
14 import vllm.envs as envs
15 -from vllm.config import ModelConfig
15 from vllm.engine.protocol import EngineClient
16 from vllm.entrypoints.logger import RequestLogger
17 from vllm.entrypoints.openai.protocol import (
@@ -53,7 +52,6 @@ class OpenAISpeechToText(OpenAIServing):
52 def __init__(
53 self,
54 engine_client: EngineClient,
56 - model_config: ModelConfig,
55 models: OpenAIServingModels,
56 *,
57 request_logger: Optional[RequestLogger],
@@ -63,7 +61,6 @@ class OpenAISpeechToText(OpenAIServing):
61 ):
62 super().__init__(
63 engine_client=engine_client,
66 - model_config=model_config,
64 models=models,
65 request_logger=request_logger,
66 return_tokens_as_token_ids=return_tokens_as_token_ids,
@@ -74,7 +71,7 @@ class OpenAISpeechToText(OpenAIServing):
71 self.task_type = task_type
72
73 self.asr_config = self.model_cls.get_speech_to_text_config(
77 - model_config, task_type
74 + self.model_config, task_type
75 )
76
77 self.max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
vllm/inputs/data.py
+4 -4
@@ -20,13 +20,13 @@ class TextPrompt(TypedDict):
20 prompt: str
21 """The input text to be tokenized before passing to the model."""
22
23 - multi_modal_data: NotRequired["MultiModalDataDict"]
23 + multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
24 """
25 Optional multi-modal data to pass to the model,
26 if the model supports it.
27 """
28
29 - mm_processor_kwargs: NotRequired[dict[str, Any]]
29 + mm_processor_kwargs: NotRequired[Optional[dict[str, Any]]]
30 """
31 Optional multi-modal processor kwargs to be forwarded to the
32 multimodal input mapper & processor. Note that if multiple modalities
@@ -61,13 +61,13 @@ class TokensPrompt(TypedDict):
61 token_type_ids: NotRequired[list[int]]
62 """A list of token type IDs to pass to the cross encoder model."""
63
64 - multi_modal_data: NotRequired["MultiModalDataDict"]
64 + multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
65 """
66 Optional multi-modal data to pass to the model,
67 if the model supports it.
68 """
69
70 - mm_processor_kwargs: NotRequired[dict[str, Any]]
70 + mm_processor_kwargs: NotRequired[Optional[dict[str, Any]]]
71 """
72 Optional multi-modal processor kwargs to be forwarded to the
73 multimodal input mapper & processor. Note that if multiple modalities
vllm/inputs/preprocess.py
+7 -10
@@ -17,7 +17,7 @@ from vllm.multimodal.inputs import (
17 MultiModalUUIDDict,
18 )
19 from vllm.multimodal.processing import BaseMultiModalProcessor
20 -from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
20 +from vllm.transformers_utils.tokenizer import AnyTokenizer
21 from vllm.utils.jsontree import json_iter_leaves
22
23 from .data import (
@@ -45,20 +45,17 @@ class InputPreprocessor:
45 def __init__(
46 self,
47 model_config: ModelConfig,
48 + tokenizer: Optional[AnyTokenizer],
49 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
50 mm_processor_cache: Optional[BaseMultiModalProcessorCache] = None,
51 ) -> None:
52 super().__init__()
53
54 self.model_config = model_config
55 + self.tokenizer = tokenizer
56 self.mm_registry = mm_registry
57 self.mm_processor_cache = mm_processor_cache
58
57 - if model_config.skip_tokenizer_init:
58 - self.tokenizer = None
59 - else:
60 - self.tokenizer = init_tokenizer_from_configs(model_config)
61 -
59 def get_tokenizer(self) -> AnyTokenizer:
60 if self.tokenizer is None:
61 raise ValueError(
@@ -351,8 +348,8 @@ class InputPreprocessor:
348 if self.model_config.is_multimodal_model:
349 inputs = self._process_multimodal(
350 prompt_token_ids,
354 - parsed_content.get("multi_modal_data", {}),
355 - parsed_content.get("mm_processor_kwargs"),
351 + parsed_content.get("multi_modal_data") or {},
352 + parsed_content.get("mm_processor_kwargs") or {},
353 tokenization_kwargs=tokenization_kwargs,
354 mm_uuids=mm_uuids,
355 )
@@ -380,8 +377,8 @@ class InputPreprocessor:
377 if self.model_config.is_multimodal_model:
378 inputs = self._process_multimodal(
379 prompt_text,
383 - parsed_content.get("multi_modal_data", {}),
384 - parsed_content.get("mm_processor_kwargs"),
380 + parsed_content.get("multi_modal_data") or {},
381 + parsed_content.get("mm_processor_kwargs") or {},
382 tokenization_kwargs=tokenization_kwargs,
383 mm_uuids=mm_uuids,
384 )
vllm/v1/engine/async_llm.py
+19 -16
@@ -12,23 +12,23 @@ import numpy as np
12 import torch
13
14 import vllm.envs as envs
15 -from vllm.config import ModelConfig, VllmConfig
15 +from vllm.config import VllmConfig
16 from vllm.engine.arg_utils import AsyncEngineArgs
17 from vllm.engine.protocol import EngineClient
18 from vllm.entrypoints.utils import _validate_truncation_size
19 from vllm.envs import VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
20 from vllm.inputs import PromptType
21 -from vllm.inputs.preprocess import InputPreprocessor
21 from vllm.logger import init_logger
22 from vllm.lora.request import LoRARequest
23 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
24 from vllm.outputs import PoolingRequestOutput, RequestOutput
25 +from vllm.plugins.io_processors import get_io_processor
26 from vllm.pooling_params import PoolingParams
27 from vllm.sampling_params import SamplingParams
28 from vllm.tasks import SupportedTask
29 from vllm.tracing import init_tracer
30 from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
31 -from vllm.transformers_utils.tokenizer import AnyTokenizer
31 +from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
32 from vllm.usage.usage_lib import UsageContext
33 from vllm.utils import Device, as_list, cancel_task_threadsafe, cdiv, deprecate_kwargs
34 from vllm.v1.engine import EngineCoreRequest
@@ -104,8 +104,16 @@ class AsyncLLM(EngineClient):
104 "logger list; enabling logging without default stat loggers"
105 )
106
107 - # Processor (converts Inputs --> EngineCoreRequests).
108 - self.processor = Processor(vllm_config, mm_registry=mm_registry)
107 + if self.model_config.skip_tokenizer_init:
108 + tokenizer = None
109 + else:
110 + tokenizer = init_tokenizer_from_configs(self.model_config)
111 +
112 + self.processor = Processor(self.vllm_config, tokenizer)
113 + self.io_processor = get_io_processor(
114 + self.vllm_config,
115 + self.model_config.io_processor_plugin,
116 + )
117
118 # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
119 self.output_processor = OutputProcessor(
@@ -245,10 +253,6 @@ class AsyncLLM(EngineClient):
253
254 cancel_task_threadsafe(getattr(self, "output_handler", None))
255
248 - @property
249 - def tokenizer(self) -> Optional[AnyTokenizer]:
250 - return self.processor.tokenizer
251 -
256 async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
257 return await self.engine_core.get_supported_tasks_async()
258
@@ -615,14 +619,13 @@ class AsyncLLM(EngineClient):
619 logger.info("Request %s failed.", request_id)
620 raise EngineGenerateError() from e
621
618 - async def get_vllm_config(self) -> VllmConfig:
619 - return self.vllm_config
620 -
621 - async def get_model_config(self) -> ModelConfig:
622 - return self.model_config
622 + @property
623 + def tokenizer(self) -> Optional[AnyTokenizer]:
624 + return self.processor.tokenizer
625
624 - async def get_input_preprocessor(self) -> InputPreprocessor:
625 - return self.processor.input_preprocessor
626 + @tokenizer.setter
627 + def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
628 + self.processor.tokenizer = tokenizer
629
630 async def get_tokenizer(self) -> AnyTokenizer:
631 if self.tokenizer is None:
vllm/v1/engine/llm_engine.py
+20 -17
@@ -19,11 +19,12 @@ from vllm.logger import init_logger
19 from vllm.lora.request import LoRARequest
20 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
21 from vllm.outputs import PoolingRequestOutput, RequestOutput
22 +from vllm.plugins.io_processors import get_io_processor
23 from vllm.pooling_params import PoolingParams
24 from vllm.sampling_params import SamplingParams
25 from vllm.tasks import SupportedTask
26 from vllm.tracing import init_tracer
26 -from vllm.transformers_utils.tokenizer import AnyTokenizer
27 +from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
28 from vllm.usage.usage_lib import UsageContext
29 from vllm.utils import Device
30 from vllm.v1.engine import EngineCoreRequest
@@ -95,8 +96,16 @@ class LLMEngine:
96 self.dp_group = None
97 self.should_execute_dummy_batch = False
98
98 - # Processor (convert Inputs --> EngineCoreRequests)
99 - self.processor = Processor(vllm_config, mm_registry=mm_registry)
99 + if self.model_config.skip_tokenizer_init:
100 + tokenizer = None
101 + else:
102 + tokenizer = init_tokenizer_from_configs(self.model_config)
103 +
104 + self.processor = Processor(self.vllm_config, tokenizer)
105 + self.io_processor = get_io_processor(
106 + self.vllm_config,
107 + self.model_config.io_processor_plugin,
108 + )
109
110 # OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
111 self.output_processor = OutputProcessor(
@@ -204,14 +213,6 @@ class LLMEngine:
213 def validate_outputs(cls, outputs, output_type):
214 return outputs
215
207 - @property
208 - def tokenizer(self) -> Optional[AnyTokenizer]:
209 - return self.processor.tokenizer
210 -
211 - @tokenizer.setter
212 - def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
213 - self.processor.tokenizer = tokenizer
214 -
216 def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
217 return self.engine_core.get_supported_tasks()
218
@@ -313,12 +314,6 @@ class LLMEngine:
314
315 return processed_outputs.request_outputs
316
316 - def get_vllm_config(self):
317 - return self.vllm_config
318 -
319 - def get_model_config(self):
320 - return self.model_config
321 -
317 def start_profile(self):
318 self.engine_core.profile(True)
319
@@ -345,6 +340,14 @@ class LLMEngine:
340 assert self.log_stats, "Stat logging disabled"
341 return get_metrics_snapshot()
342
343 + @property
344 + def tokenizer(self) -> Optional[AnyTokenizer]:
345 + return self.processor.tokenizer
346 +
347 + @tokenizer.setter
348 + def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
349 + self.processor.tokenizer = tokenizer
350 +
351 def get_tokenizer(self) -> AnyTokenizer:
352 if self.tokenizer is None:
353 raise ValueError(
vllm/v1/engine/processor.py
+2
@@ -37,6 +37,7 @@ class Processor:
37 def __init__(
38 self,
39 vllm_config: VllmConfig,
40 + tokenizer: Optional[AnyTokenizer],
41 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
42 ) -> None:
43 self.vllm_config = vllm_config
@@ -52,6 +53,7 @@ class Processor:
53
54 self.input_preprocessor = InputPreprocessor(
55 self.model_config,
56 + tokenizer,
57 mm_registry,
58 mm_processor_cache=self.mm_processor_cache,
59 )