A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 02:30 UTC
[Model] Use explicit types in `get_generation_prompt` (#33551)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 2, 2026 at 20:38 UTC
b10d05b8a8aec29ae507405f0a08bf7facbb0551
8 files changed
+82
-66
vllm/entrypoints/openai/translations/speech_to_text.py
+26
-14
@@ -37,7 +37,8 @@ from vllm.entrypoints.openai.translations.protocol import (
37
TranslationStreamResponse,
38
)
39
from vllm.exceptions import VLLMValidationError
40
-from vllm.inputs.data import PromptType
40
+from vllm.inputs.data import ExplicitEncoderDecoderPrompt, PromptType
41
+from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
42
from vllm.logger import init_logger
43
from vllm.logprobs import FlatLogprobs, Logprob
44
from vllm.model_executor.models import SupportsTranscription, supports_transcription
@@ -296,26 +297,37 @@ class OpenAISpeechToText(OpenAIServing):
297
to_language=to_language,
298
)
299
if request.response_format == "verbose_json":
299
- if not isinstance(prompt, dict):
300
+ if not is_explicit_encoder_decoder_prompt(prompt):
301
raise VLLMValidationError(
301
- "Expected prompt to be a dict",
302
+ "Expected prompt to be an encoder-decoder prompt",
303
parameter="prompt",
304
value=type(prompt).__name__,
305
)
305
- prompt_dict = cast(dict, prompt)
306
- decoder_prompt = prompt.get("decoder_prompt")
307
- if not isinstance(decoder_prompt, str):
308
- raise VLLMValidationError(
309
- "Expected decoder_prompt to be str",
310
- parameter="decoder_prompt",
311
- value=type(decoder_prompt).__name__,
312
- )
313
- prompt_dict["decoder_prompt"] = decoder_prompt.replace(
314
- "<|notimestamps|>", "<|0.00|>"
315
- )
306
+
307
+ prompt = self._preprocess_verbose_prompt(prompt)
308
+
309
prompts.append(prompt)
310
return prompts, duration
311
312
+ def _repl_verbose_text(self, text: str):
313
+ return text.replace("<|notimestamps|>", "<|0.00|>")
314
+
315
+ def _preprocess_verbose_prompt(self, prompt: ExplicitEncoderDecoderPrompt):
316
+ dec_prompt = prompt["decoder_prompt"]
317
+
318
+ if isinstance(dec_prompt, str):
319
+ prompt["decoder_prompt"] = self._repl_verbose_text(dec_prompt)
320
+ elif isinstance(dec_prompt, dict) and "prompt" in dec_prompt:
321
+ dec_prompt["prompt"] = self._repl_verbose_text(dec_prompt["prompt"])
322
+ else:
323
+ raise VLLMValidationError(
324
+ "Expected decoder_prompt to contain text",
325
+ parameter="decoder_prompt",
326
+ value=type(dec_prompt).__name__,
327
+ )
328
+
329
+ return prompt
330
+
331
def _get_verbose_segments(
332
self,
333
tokens: tuple,
vllm/model_executor/models/gemma3n_mm.py
+6
-5
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from collections.abc import Iterable, Mapping, Sequence
4
-from typing import Annotated, Any, Literal, cast
4
+from typing import Annotated, Any, Literal
5
6
import numpy as np
7
import torch
@@ -19,7 +19,7 @@ from transformers.models.siglip import SiglipImageProcessorFast
19
20
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
21
from vllm.config.multimodal import BaseDummyOptions
22
-from vllm.inputs.data import PromptType
22
+from vllm.inputs.data import PromptType, TextPrompt
23
from vllm.logger import init_logger
24
from vllm.model_executor.layers.layernorm import RMSNorm
25
from vllm.model_executor.layers.linear import RowParallelLinear
@@ -807,9 +807,10 @@ class Gemma3nForConditionalGeneration(
807
808
prompt += ": <audio_soft_token><end_of_turn>\n<start_of_turn>model\n"
809
810
- audio = (audio, stt_config.sample_rate)
811
- prompts_dict = {"multi_modal_data": {"audio": audio}, "prompt": prompt}
812
- return cast(PromptType, prompts_dict)
810
+ return TextPrompt(
811
+ prompt=prompt,
812
+ multi_modal_data={"audio": (audio, stt_config.sample_rate)},
813
+ )
814
815
@classmethod
816
def get_speech_to_text_config(
vllm/model_executor/models/glmasr.py
+7
-7
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from collections.abc import Iterable, Mapping, Sequence
5
-from typing import Annotated, Any, Literal, TypeAlias, cast
5
+from typing import Annotated, Any, Literal, TypeAlias
6
7
import numpy as np
8
import torch
@@ -14,7 +14,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
14
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
15
from vllm.config.multimodal import BaseDummyOptions
16
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
17
-from vllm.inputs.data import PromptType
17
+from vllm.inputs.data import PromptType, TokensPrompt
18
from vllm.model_executor.layers.activation import get_act_fn
19
from vllm.model_executor.layers.attention import MMEncoderAttention
20
from vllm.model_executor.layers.linear import (
@@ -1159,8 +1159,8 @@ class GlmAsrForConditionalGeneration(
1159
)
1160
1161
prompt_token_ids = tokenizer.encode(prompt)
1162
- prompt_dict = {
1163
- "prompt_token_ids": prompt_token_ids,
1164
- "multi_modal_data": {"audio": audio},
1165
- }
1166
- return cast(PromptType, prompt_dict)
1162
+
1163
+ return TokensPrompt(
1164
+ prompt_token_ids=prompt_token_ids,
1165
+ multi_modal_data={"audio": audio},
1166
+ )
vllm/model_executor/models/granite_speech.py
+7
-7
@@ -26,7 +26,7 @@
26
27
import math
28
from collections.abc import Iterable, Mapping
29
-from typing import Annotated, Literal, cast
29
+from typing import Annotated, Literal
30
31
import numpy as np
32
import torch
@@ -36,7 +36,7 @@ from transformers import BatchFeature, PretrainedConfig
36
37
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
38
from vllm.config.multimodal import BaseDummyOptions
39
-from vllm.inputs.data import PromptType
39
+from vllm.inputs.data import PromptType, TokensPrompt
40
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
41
from vllm.model_executor.layers.quantization import QuantizationConfig
42
from vllm.model_executor.models.module_mapping import MultiModelKeys
@@ -879,11 +879,11 @@ class GraniteSpeechForConditionalGeneration(
879
)
880
881
prompt_token_ids = tokenizer.encode(prompt)
882
- prompt = {
883
- "prompt_token_ids": prompt_token_ids,
884
- "multi_modal_data": {"audio": audio},
885
- }
886
- return cast(PromptType, prompt)
882
+
883
+ return TokensPrompt(
884
+ prompt_token_ids=prompt_token_ids,
885
+ multi_modal_data={"audio": audio},
886
+ )
887
888
# Adapted from https://github.com/huggingface/transformers/blob/v4.56.0/src/transformers/models/granite_speech/feature_extraction_granite_speech.py#L122 # noqa: E501
889
@classmethod
vllm/model_executor/models/qwen3_asr.py
+7
-7
@@ -23,7 +23,7 @@
23
"""Inference-only Qwen3-ASR model."""
24
25
from collections.abc import Iterable, Mapping, Sequence
26
-from typing import Any, Literal, cast
26
+from typing import Any, Literal
27
28
import numpy as np
29
import torch
@@ -33,7 +33,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
33
34
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
35
from vllm.config.multimodal import BaseDummyOptions
36
-from vllm.inputs.data import PromptType
36
+from vllm.inputs.data import PromptType, TokensPrompt
37
from vllm.logger import init_logger
38
from vllm.model_executor.models.interfaces import (
39
MultiModalEmbeddings,
@@ -561,11 +561,11 @@ class Qwen3ASRForConditionalGeneration(
561
)
562
563
prompt_token_ids = tokenizer.encode(prompt)
564
- prompt_dict = {
565
- "prompt_token_ids": prompt_token_ids,
566
- "multi_modal_data": {"audio": audio},
567
- }
568
- return cast(PromptType, prompt_dict)
564
+
565
+ return TokensPrompt(
566
+ prompt_token_ids=prompt_token_ids,
567
+ multi_modal_data={"audio": audio},
568
+ )
569
570
@classmethod
571
def post_process_output(cls, text: str) -> str:
vllm/model_executor/models/voxtral.py
+8
-5
@@ -25,7 +25,7 @@ from transformers.tokenization_utils_base import TextInput
25
26
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
27
from vllm.config.multimodal import BaseDummyOptions
28
-from vllm.inputs.data import PromptType
28
+from vllm.inputs.data import PromptType, TokensPrompt
29
from vllm.logger import init_logger
30
from vllm.model_executor.layers.quantization import QuantizationConfig
31
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -488,10 +488,13 @@ class VoxtralForConditionalGeneration(
488
)
489
490
tokenized = tokenizer.instruct.encode_transcription(req)
491
- audio = (tokenized.audios[0].audio_array, stt_config.sample_rate)
492
- prompts_dict = {"multi_modal_data": {"audio": audio}}
493
- prompts_dict["prompt_token_ids"] = tokenized.tokens
494
- return cast(PromptType, prompts_dict)
491
+
492
+ return TokensPrompt(
493
+ prompt_token_ids=tokenized.tokens,
494
+ multi_modal_data={
495
+ "audio": (tokenized.audios[0].audio_array, stt_config.sample_rate)
496
+ },
497
+ )
498
499
@classmethod
500
def get_num_audio_tokens(
vllm/model_executor/models/voxtral_realtime.py
+8
-5
@@ -4,7 +4,7 @@
4
import asyncio
5
import math
6
from collections.abc import AsyncGenerator, Mapping
7
-from typing import Literal, cast
7
+from typing import Literal
8
9
import numpy as np
10
import torch
@@ -453,7 +453,10 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
453
)
454
455
tokenized = tokenizer.instruct.encode_transcription(req)
456
- audio = (tokenized.audios[0].audio_array, stt_config.sample_rate)
457
- prompts_dict = {"multi_modal_data": {"audio": audio}}
458
- prompts_dict["prompt_token_ids"] = tokenized.tokens
459
- return cast(PromptType, prompts_dict)
456
+
457
+ return TokensPrompt(
458
+ prompt_token_ids=tokenized.tokens,
459
+ multi_modal_data={
460
+ "audio": (tokenized.audios[0].audio_array, stt_config.sample_rate)
461
+ },
462
+ )
vllm/model_executor/models/whisper.py
+13
-16
@@ -5,7 +5,7 @@ import enum
5
import math
6
from collections.abc import Iterable, Mapping, Sequence
7
from contextlib import nullcontext
8
-from typing import Annotated, Literal, cast
8
+from typing import Annotated, Literal
9
10
import numpy as np
11
import torch
@@ -21,7 +21,7 @@ from vllm.compilation.decorators import support_torch_compile
21
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
22
from vllm.config.multimodal import BaseDummyOptions
23
from vllm.distributed import get_tensor_model_parallel_world_size
24
-from vllm.inputs.data import PromptType
24
+from vllm.inputs.data import ExplicitEncoderDecoderPrompt, PromptType, TextPrompt
25
from vllm.logger import init_logger
26
from vllm.model_executor.layers.activation import get_act_fn
27
from vllm.model_executor.layers.attention import (
@@ -815,21 +815,18 @@ class WhisperForConditionalGeneration(
815
raise ValueError(
816
"Language must be specified when creating the Whisper prompt"
817
)
818
- prompt = {
819
- "encoder_prompt": {
820
- # Whisper does not support encoder prompt.
821
- "prompt": "",
822
- "multi_modal_data": {
823
- "audio": (audio, stt_config.sample_rate),
824
- },
825
- },
826
- "decoder_prompt": (
827
- (f"<|prev|>{request_prompt}" if request_prompt else "")
828
- + f"<|startoftranscript|><|{language}|>"
829
- + f"<|{task_type}|><|notimestamps|>"
818
+
819
+ decoder_text = (
820
+ f"<|prev|>{request_prompt}" if request_prompt else ""
821
+ ) + f"<|startoftranscript|><|{language}|><|{task_type}|><|notimestamps|>"
822
+
823
+ return ExplicitEncoderDecoderPrompt(
824
+ encoder_prompt=TextPrompt(
825
+ prompt="", # Whisper does not support encoder prompt.
826
+ multi_modal_data={"audio": (audio, stt_config.sample_rate)},
827
),
831
- }
832
- return cast(PromptType, prompt)
828
+ decoder_prompt=TextPrompt(prompt=decoder_text),
829
+ )
830
831
@classmethod
832
def get_placeholder_str(cls, modality: str, i: int) -> str | None: