A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 12:30 UTC
Convert online APIs to use Renderer (#34084)
Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”> Co-authored-by: Reagan Lee <“reaganjlee@gmail.com”>
Reagan Lee committed
Feb 10, 2026 at 11:44 UTC
fdd6f2ad58b113fe0fdc3fd9998e63d6064b5f16
2 files changed
+33
-9
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+21
-5
@@ -471,15 +471,31 @@ class OpenAISpeechToText(OpenAIServing):
471
lora_request=lora_request,
472
)
473
474
- list_result_generator = [
475
- self.engine_client.generate(
474
+ trace_headers = (
475
+ None
476
+ if raw_request is None
477
+ else await self._get_trace_headers(raw_request.headers)
478
+ )
479
+
480
+ list_result_generator = []
481
+ for i, prompt in enumerate(prompts):
482
+ request_id_item = f"{request_id}_{i}"
483
+ engine_request = self.input_processor.process_inputs(
484
+ request_id_item,
485
prompt,
486
sampling_params,
478
- f"{request_id}_{i}",
487
lora_request=lora_request,
488
+ trace_headers=trace_headers,
489
+ priority=0,
490
+ )
491
+ list_result_generator.append(
492
+ self.engine_client.generate(
493
+ engine_request,
494
+ sampling_params,
495
+ request_id_item,
496
+ lora_request=lora_request,
497
+ )
498
)
481
- for i, prompt in enumerate(prompts)
482
- ]
499
except ValueError as e:
500
return self.create_error_response(e)
501
vllm/entrypoints/serve/disagg/serving.py
+12
-4
@@ -99,8 +99,6 @@ class ServingTokens(OpenAIServing):
99
if raw_request:
100
raw_request.state.request_metadata = request_metadata
101
102
- # TODO(NickLucche): Change to EngineCoreRequest once Renderer work is
103
- # completed
102
engine_prompts = await self._preprocess_completion(
103
request,
104
prompt_input=request.token_ids,
@@ -132,16 +130,26 @@ class ServingTokens(OpenAIServing):
130
tok_params = request.build_tok_params(self.model_config)
131
tokenization_kwargs = tok_params.get_encode_kwargs()
132
135
- result_generator = self.engine_client.generate(
133
+ engine_request = self.input_processor.process_inputs(
134
+ request_id,
135
engine_prompt,
136
sampling_params,
138
- request_id,
137
lora_request=lora_request,
138
tokenization_kwargs=tokenization_kwargs,
139
trace_headers=trace_headers,
140
priority=request.priority,
141
)
142
143
+ result_generator = self.engine_client.generate(
144
+ engine_request,
145
+ sampling_params,
146
+ request_id,
147
+ lora_request=lora_request,
148
+ trace_headers=trace_headers,
149
+ priority=request.priority,
150
+ tokenization_kwargs=tokenization_kwargs,
151
+ )
152
+
153
except ValueError as e:
154
return self.create_error_response(str(e))
155