A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 02:00 UTC
[Model] Add Granite 4.0 1B speech to supported models (#38019)
Signed-off-by: Nick Cao <ncao@redhat.com>
Nick Cao committed
Mar 24, 2026 at 14:23 UTC
935c46dd9bad76b11c4f7392ed8140109093e7ca
4 files changed
+20
-9
docs/models/supported_models.md
+1
-1
@@ -658,7 +658,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
658
| `FunASRForConditionalGeneration` | FunASR | `allendou/Fun-ASR-Nano-2512-vllm`, etc. | | |
659
| `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
660
| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
661
-| `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-speech-3.3-2b`, `ibm-granite/granite-speech-3.3-8b`, etc. | ✅︎ | ✅︎ |
661
+| `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-4.0-1b-speech`, `ibm-granite/granite-speech-3.3-2b`, etc. | ✅︎ | ✅︎ |
662
| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | | ✅︎ |
663
| `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, etc. | | ✅︎ |
664
| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
tests/models/multimodal/generation/test_granite_speech.py
+15
-7
@@ -29,10 +29,13 @@ def vllm_to_hf_output(
29
30
31
MODEL_NAME = "ibm-granite/granite-speech-3.3-2b"
32
-# Audio lora co-exists directly in the model directory, but
33
-# currently still needs to be passed directly to vLLM.
34
-audio_lora_path = MODEL_NAME
35
-models = [MODEL_NAME]
32
+MODEL_NAME_4_0 = "ibm-granite/granite-4.0-1b-speech"
33
+# Audio lora co-exists directly in the 3.3 model directory,
34
+# the 4.0 model has adapters merged into the weights.
35
+models: dict[str, str | None] = {
36
+ MODEL_NAME: MODEL_NAME,
37
+ MODEL_NAME_4_0: None,
38
+}
39
40
41
@pytest.fixture
@@ -60,6 +63,7 @@ def run_test(
63
tensor_parallel_size: int,
64
distributed_executor_backend: str | None = None,
65
attention_config: dict | None = None,
66
+ audio_lora_path: str | None = None,
67
):
68
"""Inference result should be the same between hf and vllm.
69
@@ -84,12 +88,14 @@ def run_test(
88
limit_mm_per_prompt={"audio": 1},
89
tensor_parallel_size=tensor_parallel_size,
90
distributed_executor_backend=distributed_executor_backend,
87
- enable_lora=True,
91
+ enable_lora=audio_lora_path is not None,
92
max_lora_rank=64,
93
enforce_eager=True,
94
attention_config=attention_config,
95
) as vllm_model:
92
- lora_request = LoRARequest("audio", 1, audio_lora_path)
96
+ lora_request = (
97
+ LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
98
+ )
99
vllm_outputs_per_case = [
100
vllm_model.generate_greedy_logprobs(
101
prompts,
@@ -125,7 +131,7 @@ def run_test(
131
)
132
133
128
-@pytest.mark.parametrize("model", models)
134
+@pytest.mark.parametrize("model,audio_lora_path", models.items())
135
@pytest.mark.parametrize(
136
"dtype", ["float16"] if current_platform.is_rocm() else ["bfloat16"]
137
)
@@ -138,6 +144,7 @@ def test_models(
144
hf_runner,
145
vllm_runner,
146
model: str,
147
+ audio_lora_path: str | None,
148
audio_assets: AudioTestAssets,
149
granite_speech_attention_config,
150
dtype: str,
@@ -167,4 +174,5 @@ def test_models(
174
num_logprobs=num_logprobs,
175
tensor_parallel_size=1,
176
attention_config=granite_speech_attention_config,
177
+ audio_lora_path=audio_lora_path,
178
)
tests/models/registry.py
+2
-1
@@ -810,7 +810,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
810
),
811
"GraniteVision": _HfExamplesInfo("ibm-granite/granite-vision-3.3-2b"),
812
"GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
813
- "ibm-granite/granite-speech-3.3-2b"
813
+ "ibm-granite/granite-speech-3.3-2b",
814
+ extras={"4.0-1b": "ibm-granite/granite-4.0-1b-speech"},
815
),
816
"GLM4VForCausalLM": _HfExamplesInfo(
817
"zai-org/glm-4v-9b",
vllm/model_executor/models/granite_speech.py
+2
@@ -75,12 +75,14 @@ from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
75
76
# NOTE lang support is based on what is written here:
77
# https://huggingface.co/ibm-granite/granite-speech-3.3-2b
78
+# https://huggingface.co/ibm-granite/granite-4.0-1b-speech
79
# Though this may vary from model to model, and also many langs
80
# work pretty well with zero shot.
81
ISO639_1_SUPPORTED_LANGS = {
82
"en": "English",
83
"fr": "French",
84
"de": "German",
85
+ "ja": "Japanese",
86
"pt": "Portuguese",
87
"es": "Spanish",
88
}