A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 10:00 UTC
[Refactor] Pass Renderer to Input Processor (#34329)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 12, 2026 at 11:38 UTC
b96f7314b451c01d2c727a93636c023b07adf732
20 files changed
+107
-106
tests/entrypoints/openai/test_serving_responses.py
+2
-3
@@ -125,6 +125,7 @@ class TestInitializeToolSessions:
125
engine_client = MagicMock()
126
127
model_config = MagicMock()
128
+ model_config.max_model_len = 100
129
model_config.hf_config.model_type = "test"
130
model_config.get_diff_sampling_param.return_value = {}
131
engine_client.model_config = model_config
@@ -212,6 +213,7 @@ class TestValidateGeneratorInput:
213
engine_client = MagicMock()
214
215
model_config = MagicMock()
216
+ model_config.max_model_len = 100
217
model_config.hf_config.model_type = "test"
218
model_config.get_diff_sampling_param.return_value = {}
219
engine_client.model_config = model_config
@@ -231,9 +233,6 @@ class TestValidateGeneratorInput:
233
chat_template_content_format="auto",
234
)
235
234
- # Set max_model_len for testing
235
- instance.max_model_len = 100
236
-
236
return instance
237
238
def test_validate_generator_input(self, serving_responses_instance):
tests/models/language/generation/test_hybrid.py
+4
-2
@@ -507,7 +507,8 @@ def test_apc_single_prompt_block_align_alignment(
507
vllm_runner_kwargs["enable_prefix_caching"] = True
508
with vllm_runner(**vllm_runner_kwargs) as vllm_model:
509
# Retrieve the default mamba state block size
510
- mamba_block_size = vllm_model.llm.llm_engine.cache_config.mamba_block_size
510
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
511
+ mamba_block_size = vllm_config.cache_config.mamba_block_size
512
513
# In case the hybrid model does not have the
514
# "mamba_block_size" assume a fixed constant
@@ -660,7 +661,8 @@ def test_apc_multiple_prompts_block_align_alignment(
661
vllm_runner_kwargs["enable_prefix_caching"] = True
662
with vllm_runner(**vllm_runner_kwargs) as vllm_model:
663
# Retrieve the default mamba state block size
663
- mamba_block_size = vllm_model.llm.llm_engine.cache_config.mamba_block_size
664
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
665
+ mamba_block_size = vllm_config.cache_config.mamba_block_size
666
667
# In case the hybrid model does not have the
668
# "mamba_block_size" assume a fixed constant
tests/models/language/pooling/test_auto_prefix_cache_support.py
+6
-3
@@ -25,7 +25,8 @@ def test_classify_models(
25
with vllm_runner(
26
model, max_model_len=512, dtype=dtype, enable_prefix_caching=True
27
) as vllm_model:
28
- cache_config = vllm_model.llm.llm_engine.cache_config
28
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
29
+ cache_config = vllm_config.cache_config
30
assert cache_config.enable_prefix_caching
31
32
# First Run
@@ -74,7 +75,8 @@ def test_embed_models(
75
max_model_len=None,
76
enable_prefix_caching=True,
77
) as vllm_model:
77
- cache_config = vllm_model.llm.llm_engine.cache_config
78
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
79
+ cache_config = vllm_config.cache_config
80
assert cache_config.enable_prefix_caching
81
82
# First Run
@@ -106,5 +108,6 @@ def test_non_causal_models(
108
hf_runner, vllm_runner, example_prompts, model: str, dtype: str
109
) -> None:
110
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
109
- cache_config = vllm_model.llm.llm_engine.cache_config
111
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
112
+ cache_config = vllm_config.cache_config
113
assert not cache_config.enable_prefix_caching
tests/v1/e2e/test_pooling_chunked_prefill.py
+2
-1
@@ -161,7 +161,8 @@ def test_pooling_prefix_cache(vllm_runner, monkeypatch):
161
assert chunks[0] <= prompt1_len
162
assert chunks[0] < prompt2_len
163
164
- cache_config = llm.get_llm().llm_engine.cache_config
164
+ vllm_config = llm.get_llm().llm_engine.vllm_config
165
+ cache_config = vllm_config.cache_config
166
print(f"{cache_config=}")
167
# Prefixes are cached in blocks
168
assert (prompt2_len - chunks[0]) % cache_config.block_size == 0
tests/v1/sample/test_logprobs.py
+2
-1
@@ -311,7 +311,8 @@ def test_get_logprobs_and_prompt_logprobs(
311
temperature: "temperature" sampling parameter
312
example_prompts: example prompt fixture
313
"""
314
- do_apc = vllm_model.llm.llm_engine.cache_config.enable_prefix_caching
314
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
315
+ do_apc = vllm_config.cache_config.enable_prefix_caching
316
if do_apc and (temperature < 2.0 or batch_logprobs_composition != SAMPLE_PROMPT):
317
# Skip some test-cases to save time.
318
pytest.skip()
vllm/config/pooler.py
+1
-1
@@ -54,7 +54,7 @@ class PoolerConfig:
54
Reduce the dimensions of embeddings if model
55
support matryoshka representation. Defaults to None.
56
"""
57
- enable_chunked_processing: bool | None = None
57
+ enable_chunked_processing: bool = False
58
"""
59
Whether to enable chunked processing for long inputs that exceed the model's
60
maximum position embeddings. When enabled, long inputs will be split into
vllm/engine/protocol.py
+2
-5
@@ -31,12 +31,9 @@ class EngineClient(ABC):
31
32
vllm_config: VllmConfig
33
model_config: ModelConfig
34
- input_processor: InputProcessor
34
+ renderer: BaseRenderer
35
io_processor: IOProcessor | None
36
-
37
- @property
38
- @abstractmethod
39
- def renderer(self) -> BaseRenderer: ...
36
+ input_processor: InputProcessor
37
38
@property
39
@abstractmethod
vllm/entrypoints/llm.py
+4
-3
@@ -356,8 +356,9 @@ class LLM:
356
self.supported_tasks = supported_tasks
357
358
self.model_config = self.llm_engine.model_config
359
- self.input_processor = self.llm_engine.input_processor
359
+ self.renderer = self.llm_engine.renderer
360
self.io_processor = self.llm_engine.io_processor
361
+ self.input_processor = self.llm_engine.input_processor
362
363
# Cache for __repr__ to avoid repeated collective_rpc calls
364
self._cached_repr: str | None = None
@@ -816,7 +817,7 @@ class LLM:
817
A list of `TokensPrompts` objects containing the tokenized prompt
818
after chat template interpolation, and the raw multi-modal inputs.
819
"""
819
- renderer = self.llm_engine.renderer
820
+ renderer = self.renderer
821
model_config = self.model_config
822
823
parsed_prompts = [
@@ -858,7 +859,7 @@ class LLM:
859
A list of `TokensPrompts` objects containing the tokenized prompt
860
after chat template interpolation, and the raw multi-modal inputs.
861
"""
861
- renderer = self.llm_engine.renderer
862
+ renderer = self.renderer
863
864
chat_params = ChatParams(
865
chat_template=chat_template,
vllm/entrypoints/openai/chat_completion/serving.py
+3
-3
@@ -239,8 +239,7 @@ class OpenAIServingChat(OpenAIServing):
239
raise self.engine_client.dead_error
240
241
try:
242
- renderer = self.engine_client.renderer
243
- tokenizer = renderer.tokenizer
242
+ tokenizer = self.renderer.tokenizer
243
244
tool_parser = self.tool_parser
245
@@ -375,6 +374,7 @@ class OpenAIServingChat(OpenAIServing):
374
data_parallel_rank = self._get_data_parallel_rank(raw_request)
375
376
# Schedule the request and get the result generator.
377
+ max_model_len = self.model_config.max_model_len
378
generators: list[AsyncGenerator[RequestOutput, None]] = []
379
try:
380
for i, engine_prompt in enumerate(engine_prompts):
@@ -387,7 +387,7 @@ class OpenAIServingChat(OpenAIServing):
387
)
388
389
max_tokens = get_max_tokens(
390
- self.max_model_len,
390
+ max_model_len,
391
request.max_completion_tokens
392
if request.max_completion_tokens is not None
393
else request.max_tokens,
vllm/entrypoints/openai/completion/serving.py
+2
-1
@@ -157,13 +157,14 @@ class OpenAIServingCompletion(OpenAIServing):
157
data_parallel_rank = self._get_data_parallel_rank(raw_request)
158
159
# Schedule the request and get the result generator.
160
+ max_model_len = self.model_config.max_model_len
161
generators: list[AsyncGenerator[RequestOutput, None]] = []
162
try:
163
for i, engine_prompt in enumerate(engine_prompts):
164
prompt_text = self._extract_prompt_text(engine_prompt)
165
166
max_tokens = get_max_tokens(
166
- self.max_model_len,
167
+ max_model_len,
168
request.max_tokens,
169
self._extract_prompt_len(engine_prompt),
170
self.default_sampling_params,
vllm/entrypoints/openai/engine/serving.py
+16
-15
@@ -242,11 +242,10 @@ class OpenAIServing:
242
243
self.log_error_stack = log_error_stack
244
245
- self.input_processor = self.models.input_processor
246
- self.io_processor = self.models.io_processor
247
- self.renderer = self.models.renderer
248
- self.model_config = self.models.model_config
249
- self.max_model_len = self.model_config.max_model_len
245
+ self.model_config = engine_client.model_config
246
+ self.renderer = engine_client.renderer
247
+ self.io_processor = engine_client.io_processor
248
+ self.input_processor = engine_client.input_processor
249
250
async def beam_search(
251
self,
@@ -537,7 +536,7 @@ class OpenAIServing:
536
537
if (
538
truncate_prompt_tokens is not None
540
- and truncate_prompt_tokens > self.max_model_len
539
+ and truncate_prompt_tokens > self.model_config.max_model_len
540
):
541
return self.create_error_response(
542
"truncate_prompt_tokens value is "
@@ -844,6 +843,7 @@ class OpenAIServing:
843
input_text: str,
844
) -> TokensPrompt:
845
token_num = len(input_ids)
846
+ max_model_len = self.model_config.max_model_len
847
848
# Note: EmbeddingRequest, ClassificationRequest,
849
# and ScoreRequest doesn't have max_tokens
@@ -862,7 +862,7 @@ class OpenAIServing:
862
):
863
# Note: input length can be up to the entire model context length
864
# since these requests don't generate tokens.
865
- if token_num > self.max_model_len:
865
+ if token_num > max_model_len:
866
operations: dict[type[AnyRequest], str] = {
867
ScoreDataRequest: "score",
868
ScoreTextRequest: "score",
@@ -873,7 +873,7 @@ class OpenAIServing:
873
operation = operations.get(type(request), "embedding generation")
874
raise VLLMValidationError(
875
f"This model's maximum context length is "
876
- f"{self.max_model_len} tokens. However, you requested "
876
+ f"{max_model_len} tokens. However, you requested "
877
f"{token_num} tokens in the input for {operation}. "
878
f"Please reduce the length of the input.",
879
parameter="input_tokens",
@@ -898,22 +898,22 @@ class OpenAIServing:
898
899
# Note: input length can be up to model context length - 1 for
900
# completion-like requests.
901
- if token_num >= self.max_model_len:
901
+ if token_num >= max_model_len:
902
raise VLLMValidationError(
903
f"This model's maximum context length is "
904
- f"{self.max_model_len} tokens. However, your request has "
904
+ f"{max_model_len} tokens. However, your request has "
905
f"{token_num} input tokens. Please reduce the length of "
906
"the input messages.",
907
parameter="input_tokens",
908
value=token_num,
909
)
910
911
- if max_tokens is not None and token_num + max_tokens > self.max_model_len:
911
+ if max_tokens is not None and token_num + max_tokens > max_model_len:
912
raise VLLMValidationError(
913
"'max_tokens' or 'max_completion_tokens' is too large: "
914
f"{max_tokens}. This model's maximum context length is "
915
- f"{self.max_model_len} tokens and your request has "
916
- f"{token_num} input tokens ({max_tokens} > {self.max_model_len}"
915
+ f"{max_model_len} tokens and your request has "
916
+ f"{token_num} input tokens ({max_tokens} > {max_model_len}"
917
f" - {token_num}).",
918
parameter="max_tokens",
919
value=max_tokens,
@@ -1089,6 +1089,7 @@ class OpenAIServing:
1089
priority: int = 0,
1090
trace_headers: Mapping[str, str] | None = None,
1091
):
1092
+ max_model_len = self.model_config.max_model_len
1093
prompt_text = self._extract_prompt_text(engine_prompt)
1094
1095
orig_priority = priority
@@ -1148,7 +1149,7 @@ class OpenAIServing:
1149
token_ids = context.render_for_completion()
1150
engine_prompt = TokensPrompt(prompt_token_ids=token_ids)
1151
1151
- sampling_params.max_tokens = self.max_model_len - len(token_ids)
1152
+ sampling_params.max_tokens = max_model_len - len(token_ids)
1153
elif isinstance(context, ParsableContext):
1154
engine_prompts = await self._render_next_turn(
1155
context.request,
@@ -1162,7 +1163,7 @@ class OpenAIServing:
1163
prompt_text = self._extract_prompt_text(engine_prompt)
1164
1165
sampling_params.max_tokens = get_max_tokens(
1165
- self.max_model_len,
1166
+ max_model_len,
1167
context.request.max_output_tokens,
1168
self._extract_prompt_len(engine_prompt),
1169
self.default_sampling_params, # type: ignore
vllm/entrypoints/openai/models/serving.py
+7
-7
@@ -59,11 +59,10 @@ class OpenAIServingModels:
59
)
60
self.lora_resolver_lock: dict[str, Lock] = defaultdict(Lock)
61
62
- self.input_processor = self.engine_client.input_processor
63
- self.io_processor = self.engine_client.io_processor
64
- self.renderer = self.engine_client.renderer
62
self.model_config = self.engine_client.model_config
66
- self.max_model_len = self.model_config.max_model_len
63
+ self.renderer = self.engine_client.renderer
64
+ self.io_processor = self.engine_client.io_processor
65
+ self.input_processor = self.engine_client.input_processor
66
67
async def init_static_loras(self):
68
"""Loads all static LoRA modules.
@@ -96,12 +95,13 @@ class OpenAIServingModels:
95
return self.base_model_paths[0].name
96
97
async def show_available_models(self) -> ModelList:
99
- """Show available models. This includes the base model and all
100
- adapters"""
98
+ """Show available models. This includes the base model and all adapters."""
99
+ max_model_len = self.model_config.max_model_len
100
+
101
model_cards = [
102
ModelCard(
103
id=base_model.name,
104
- max_model_len=self.max_model_len,
104
+ max_model_len=max_model_len,
105
root=base_model.model_path,
106
permission=[ModelPermission()],
107
)
vllm/entrypoints/openai/responses/serving.py
+7
-5
@@ -296,10 +296,12 @@ class OpenAIServingResponses(OpenAIServing):
296
) -> ErrorResponse | None:
297
"""Add validations to the input to the generator here."""
298
prompt_len = self._extract_prompt_len(engine_prompt)
299
- if self.max_model_len <= prompt_len:
299
+ max_model_len = self.model_config.max_model_len
300
+
301
+ if prompt_len >= max_model_len:
302
error_message = (
303
f"The engine prompt length {prompt_len} "
302
- f"exceeds the max_model_len {self.max_model_len}. "
304
+ f"exceeds the max_model_len {max_model_len}. "
305
"Please reduce prompt."
306
)
307
return self.create_error_response(
@@ -414,6 +416,7 @@ class OpenAIServingResponses(OpenAIServing):
416
raw_request.state.request_metadata = request_metadata
417
418
# Schedule the request and get the result generator.
419
+ max_model_len = self.model_config.max_model_len
420
generators: list[AsyncGenerator[ConversationContext, None]] = []
421
422
builtin_tool_list: list[str] = []
@@ -431,8 +434,7 @@ class OpenAIServingResponses(OpenAIServing):
434
assert len(builtin_tool_list) == 0
435
available_tools = []
436
try:
434
- renderer = self.engine_client.renderer
435
- tokenizer = renderer.get_tokenizer()
437
+ tokenizer = self.renderer.get_tokenizer()
438
439
for engine_prompt in engine_prompts:
440
maybe_error = self._validate_generator_input(engine_prompt)
@@ -440,7 +442,7 @@ class OpenAIServingResponses(OpenAIServing):
442
return maybe_error
443
444
default_max_tokens = get_max_tokens(
443
- self.max_model_len,
445
+ max_model_len,
446
request.max_output_tokens,
447
self._extract_prompt_len(engine_prompt),
448
self.default_sampling_params,
vllm/entrypoints/pooling/embed/serving.py
+6
-14
@@ -69,16 +69,8 @@ class OpenAIServingEmbedding(OpenAIServing):
69
self.trust_request_chat_template = trust_request_chat_template
70
71
pooler_config = self.model_config.pooler_config
72
-
73
- # Avoid repeated attribute lookups
74
- self.supports_chunked_processing = bool(
75
- pooler_config and pooler_config.enable_chunked_processing
76
- )
77
- self.max_embed_len = (
78
- pooler_config.max_embed_len
79
- if pooler_config and pooler_config.max_embed_len
80
- else None
81
- )
72
+ assert pooler_config is not None
73
+ self.pooler_config = pooler_config
74
75
async def _preprocess(
76
self,
@@ -240,7 +232,7 @@ class OpenAIServingEmbedding(OpenAIServing):
232
"""Check if chunked processing should be used for this request."""
233
return (
234
isinstance(request, (EmbeddingCompletionRequest, EmbeddingChatRequest))
243
- and self.supports_chunked_processing
235
+ and self.pooler_config.enable_chunked_processing
236
)
237
238
async def _process_chunked_request(
@@ -310,14 +302,14 @@ class OpenAIServingEmbedding(OpenAIServing):
302
max_pos_embeddings = self._get_max_position_embeddings()
303
304
# Determine the effective max length for validation
313
- if self.max_embed_len is not None:
305
+ if self.pooler_config.max_embed_len:
306
# Use max_embed_len for validation instead of max_model_len
307
length_type = "maximum embedding input length"
316
- max_length_value = self.max_embed_len
308
+ max_length_value = self.pooler_config.max_embed_len
309
else:
310
# Fall back to max_model_len validation (original behavior)
311
length_type = "maximum context length"
320
- max_length_value = self.max_model_len
312
+ max_length_value = self.model_config.max_model_len
313
314
validation_error_msg = (
315
"This model's {length_type} is {max_length_value} tokens. "
vllm/entrypoints/serve/tokenize/serving.py
+1
-1
@@ -117,7 +117,7 @@ class OpenAIServingTokenization(OpenAIServing):
117
tokens=input_ids,
118
token_strs=token_strs,
119
count=len(input_ids),
120
- max_model_len=self.max_model_len,
120
+ max_model_len=self.model_config.max_model_len,
121
)
122
123
async def create_detokenize(
vllm/inputs/preprocess.py
+3
-2
@@ -16,7 +16,7 @@ from vllm.multimodal.inputs import (
16
MultiModalUUIDDict,
17
)
18
from vllm.multimodal.processing import BaseMultiModalProcessor
19
-from vllm.renderers import renderer_from_config
19
+from vllm.renderers import BaseRenderer, renderer_from_config
20
from vllm.renderers.inputs import (
21
DecoderDictPrompt,
22
DecoderOnlyDictPrompt,
@@ -56,6 +56,7 @@ class InputPreprocessor:
56
self,
57
model_config: ModelConfig,
58
observability_config: ObservabilityConfig | None = None,
59
+ renderer: BaseRenderer | None = None,
60
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
61
mm_processor_cache: BaseMultiModalProcessorCache | None = None,
62
) -> None:
@@ -63,7 +64,7 @@ class InputPreprocessor:
64
65
self.model_config = model_config
66
self.observability_config = observability_config
66
- self.renderer = renderer_from_config(model_config)
67
+ self.renderer = renderer or renderer_from_config(model_config)
68
self.mm_registry = mm_registry
69
self.mm_processor_cache = mm_processor_cache
70
vllm/v1/engine/async_llm.py
+13
-14
@@ -27,7 +27,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
27
from vllm.outputs import STREAM_FINISHED, PoolingRequestOutput, RequestOutput
28
from vllm.plugins.io_processors import get_io_processor
29
from vllm.pooling_params import PoolingParams
30
-from vllm.renderers import BaseRenderer, merge_kwargs
30
+from vllm.renderers import merge_kwargs, renderer_from_config
31
from vllm.renderers.inputs import DictPrompt, TokPrompt
32
from vllm.renderers.inputs.preprocess import extract_prompt_components
33
from vllm.sampling_params import RequestOutputKind, SamplingParams
@@ -110,9 +110,10 @@ class AsyncLLM(EngineClient):
110
# Ensure we can serialize custom transformer configs
111
maybe_register_config_serialize_by_value()
112
113
- self.model_config = vllm_config.model_config
113
self.vllm_config = vllm_config
114
+ self.model_config = vllm_config.model_config
115
self.observability_config = vllm_config.observability_config
116
+
117
tracing_endpoint = self.observability_config.otlp_traces_endpoint
118
if tracing_endpoint is not None:
119
init_tracer("vllm.llm_engine", tracing_endpoint)
@@ -131,20 +132,22 @@ class AsyncLLM(EngineClient):
132
"enabling logging without default stat loggers."
133
)
134
134
- self.input_processor = InputProcessor(self.vllm_config)
135
+ self.renderer = renderer = renderer_from_config(self.model_config)
136
self.io_processor = get_io_processor(
137
self.vllm_config,
138
self.model_config.io_processor_plugin,
139
)
140
140
- # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
141
+ # Convert TokPrompt --> EngineCoreRequest.
142
+ self.input_processor = InputProcessor(self.vllm_config, renderer)
143
+
144
+ # Converts EngineCoreOutputs --> RequestOutput.
145
self.output_processor = OutputProcessor(
142
- self.tokenizer,
146
+ renderer.tokenizer,
147
log_stats=self.log_stats,
148
stream_interval=self.vllm_config.scheduler_config.stream_interval,
149
+ tracing_enabled=tracing_endpoint is not None,
150
)
146
- if tracing_endpoint is not None:
147
- self.output_processor.tracing_enabled = True
151
152
# EngineCore (starts the engine in background process).
153
self.engine_core = EngineCoreClient.make_async_mp_client(
@@ -891,17 +894,13 @@ class AsyncLLM(EngineClient):
894
895
@property
896
def tokenizer(self) -> TokenizerLike | None:
894
- return self.input_processor.tokenizer
897
+ return self.renderer.tokenizer
898
899
def get_tokenizer(self) -> TokenizerLike:
897
- return self.input_processor.get_tokenizer()
898
-
899
- @property
900
- def renderer(self) -> BaseRenderer:
901
- return self.input_processor.renderer
900
+ return self.renderer.get_tokenizer()
901
902
async def is_tracing_enabled(self) -> bool:
904
- return self.observability_config.otlp_traces_endpoint is not None # type: ignore
903
+ return self.observability_config.otlp_traces_endpoint is not None
904
905
async def do_log_stats(self) -> None:
906
if self.logger_manager:
vllm/v1/engine/input_processor.py
+8
-8
@@ -27,7 +27,7 @@ from vllm.multimodal.parse import ModalityDataItems, MultiModalDataItems
27
from vllm.multimodal.processing.context import set_request_id
28
from vllm.multimodal.utils import argsort_mm_positions
29
from vllm.pooling_params import PoolingParams
30
-from vllm.renderers import BaseRenderer
30
+from vllm.renderers import BaseRenderer, renderer_from_config
31
from vllm.renderers.inputs import DictPrompt, TokPrompt
32
from vllm.sampling_params import SamplingParams
33
from vllm.tasks import POOLING_TASKS, SupportedTask
@@ -44,6 +44,8 @@ class InputProcessor:
44
def __init__(
45
self,
46
vllm_config: VllmConfig,
47
+ renderer: BaseRenderer | None = None,
48
+ *,
49
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
50
) -> None:
51
self.vllm_config = vllm_config
@@ -57,6 +59,7 @@ class InputProcessor:
59
60
self.generation_config_fields = model_config.try_get_generation_config()
61
62
+ self.renderer = renderer or renderer_from_config(model_config)
63
self.mm_registry = mm_registry
64
self.mm_processor_cache = mm_registry.processor_cache_from_config(vllm_config)
65
@@ -74,20 +77,17 @@ class InputProcessor:
77
self.input_preprocessor = InputPreprocessor(
78
model_config,
79
self.observability_config,
77
- mm_registry,
80
+ renderer=renderer,
81
+ mm_registry=mm_registry,
82
mm_processor_cache=self.mm_processor_cache,
83
)
84
85
@property
86
def tokenizer(self) -> TokenizerLike | None:
83
- return self.input_preprocessor.tokenizer
87
+ return self.renderer.tokenizer
88
89
def get_tokenizer(self) -> TokenizerLike:
86
- return self.input_preprocessor.get_tokenizer()
87
-
88
- @property
89
- def renderer(self) -> BaseRenderer:
90
- return self.input_preprocessor.renderer
90
+ return self.renderer.get_tokenizer()
91
92
def _validate_params(
93
self,
vllm/v1/engine/llm_engine.py
+15
-16
@@ -21,7 +21,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
21
from vllm.outputs import PoolingRequestOutput, RequestOutput
22
from vllm.plugins.io_processors import get_io_processor
23
from vllm.pooling_params import PoolingParams
24
-from vllm.renderers import BaseRenderer
24
+from vllm.renderers import renderer_from_config
25
from vllm.renderers.inputs import DictPrompt, TokPrompt
26
from vllm.renderers.inputs.preprocess import extract_prompt_components
27
from vllm.sampling_params import SamplingParams
@@ -62,9 +62,12 @@ class LLMEngine:
62
multiprocess_mode: bool = False,
63
) -> None:
64
self.vllm_config = vllm_config
65
- self.observability_config = vllm_config.observability_config
65
self.model_config = vllm_config.model_config
67
- self.cache_config = vllm_config.cache_config
66
+ self.observability_config = vllm_config.observability_config
67
+
68
+ tracing_endpoint = self.observability_config.otlp_traces_endpoint
69
+ if tracing_endpoint is not None:
70
+ init_tracer("vllm.llm_engine", tracing_endpoint)
71
72
self.log_stats = log_stats
73
@@ -87,22 +90,22 @@ class LLMEngine:
90
self.dp_group = None
91
self.should_execute_dummy_batch = False
92
90
- self.input_processor = InputProcessor(self.vllm_config)
93
+ self.renderer = renderer = renderer_from_config(self.model_config)
94
self.io_processor = get_io_processor(
95
self.vllm_config,
96
self.model_config.io_processor_plugin,
97
)
98
96
- # OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
99
+ # Convert TokPrompt --> EngineCoreRequest.
100
+ self.input_processor = InputProcessor(self.vllm_config, renderer)
101
+
102
+ # Converts EngineCoreOutputs --> RequestOutput.
103
self.output_processor = OutputProcessor(
98
- self.tokenizer,
104
+ renderer.tokenizer,
105
log_stats=self.log_stats,
106
stream_interval=self.vllm_config.scheduler_config.stream_interval,
107
+ tracing_enabled=tracing_endpoint is not None,
108
)
102
- endpoint = self.observability_config.otlp_traces_endpoint
103
- if endpoint is not None:
104
- init_tracer("vllm.llm_engine", endpoint)
105
- self.output_processor.tracing_enabled = True
109
110
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
111
self.engine_core = EngineCoreClient.make_client(
@@ -365,14 +368,10 @@ class LLMEngine:
368
369
@property
370
def tokenizer(self) -> TokenizerLike | None:
368
- return self.input_processor.tokenizer
371
+ return self.renderer.tokenizer
372
373
def get_tokenizer(self) -> TokenizerLike:
371
- return self.input_processor.get_tokenizer()
372
-
373
- @property
374
- def renderer(self) -> BaseRenderer:
375
- return self.input_processor.renderer
374
+ return self.renderer.get_tokenizer()
375
376
def do_log_stats(self) -> None:
377
"""Log stats if logging is enabled."""
vllm/v1/engine/output_processor.py
+3
-1
@@ -417,8 +417,10 @@ class OutputProcessor:
417
def __init__(
418
self,
419
tokenizer: TokenizerLike | None,
420
+ *,
421
log_stats: bool,
422
stream_interval: int = 1,
423
+ tracing_enabled: bool = False,
424
):
425
self.log_stats = log_stats
426
self.tokenizer = tokenizer
@@ -427,7 +429,7 @@ class OutputProcessor:
429
self.parent_requests: dict[str, ParentRequest] = {}
430
self.external_req_ids: defaultdict[str, list[str]] = defaultdict(list)
431
self.lora_states = LoRARequestStates(log_stats)
430
- self.tracing_enabled: bool = False
432
+ self.tracing_enabled = tracing_enabled
433
self._requests_drained = asyncio.Event()
434
self._requests_drained.set()
435