A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 22:30 UTC
Add model support for granite speech plus (#43519)
Signed-off-by: Zvi Kons[WSL] <zvi@il.ibm.com> Signed-off-by: Zvi Kons (BlueVela) <zvi@il.ibm.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Zvi Kons committed
Jun 4, 2026 at 17:47 UTC
b21443e23c11b0bb756d31006ca74e9959ee624b
6 files changed
+106
-3
docs/models/supported_models.md
+2
@@ -569,6 +569,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
569
| `GlmOcrForConditionalGeneration` | GLM-OCR | T + I<sup>E+</sup> | `zai-org/GLM-OCR`, etc. | ✅︎ | ✅︎ |
570
| `Granite4VisionForConditionalGeneration` | Granite 4 Vision | T + I<sup>E+</sup> | `ibm-granite/granite-4.1-3b-vision`, etc. | ✅︎ | ✅︎ |
571
| `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ |
572
+| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | T + A | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ |
573
| `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | |
574
| `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | |
575
| `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | ✅︎ | ✅︎ |
@@ -709,6 +710,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
710
| `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
711
| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
712
| `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-4.0-1b-speech`, `ibm-granite/granite-speech-3.3-2b`, etc. | ✅︎ | ✅︎ |
713
+| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ |
714
| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | ✅︎ | ✅︎ |
715
| `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, etc. | | ✅︎ |
716
| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
tests/models/multimodal/generation/test_granite_speech.py
+4
-1
@@ -30,11 +30,14 @@ def vllm_to_hf_output(
30
31
MODEL_NAME = "ibm-granite/granite-speech-3.3-2b"
32
MODEL_NAME_4_0 = "ibm-granite/granite-4.0-1b-speech"
33
+# "plus" variant of granite speech (uses GraniteSpeechPlusForConditionalGeneration).
34
+MODEL_NAME_4_1_PLUS = "ibm-granite/granite-speech-4.1-2b-plus"
35
# Audio lora co-exists directly in the 3.3 model directory,
34
-# the 4.0 model has adapters merged into the weights.
36
+# the 4.0 and 4.1-plus models have adapters merged into the weights.
37
models: dict[str, str | None] = {
38
MODEL_NAME: MODEL_NAME,
39
MODEL_NAME_4_0: None,
40
+ MODEL_NAME_4_1_PLUS: None,
41
}
42
43
tests/models/registry.py
+4
@@ -938,6 +938,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
938
"ibm-granite/granite-speech-3.3-2b",
939
extras={"4.0-1b": "ibm-granite/granite-4.0-1b-speech"},
940
),
941
+ "GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo(
942
+ "ibm-granite/granite-speech-4.1-2b-plus",
943
+ min_transformers_version="5.8.0",
944
+ ),
945
"GLM4VForCausalLM": _HfExamplesInfo(
946
"zai-org/glm-4v-9b",
947
trust_remote_code=True,
vllm/model_executor/models/granite_speech.py
+13
-2
@@ -614,8 +614,7 @@ class GraniteSpeechForConditionalGeneration(
614
)
615
616
with self._mark_tower_model(vllm_config, "audio"):
617
- # Conformer encoder
618
- self.encoder = GraniteSpeechCTCEncoder(
617
+ self.encoder = self._build_encoder(
618
config=config.encoder_config,
619
quant_config=quant_config,
620
prefix=maybe_prefix(prefix, "encoder"),
@@ -633,6 +632,18 @@ class GraniteSpeechForConditionalGeneration(
632
self.language_model.make_empty_intermediate_tensors
633
)
634
635
+ def _build_encoder(
636
+ self,
637
+ config: PretrainedConfig,
638
+ quant_config: QuantizationConfig | None,
639
+ prefix: str,
640
+ ) -> "GraniteSpeechCTCEncoder":
641
+ return GraniteSpeechCTCEncoder(
642
+ config=config,
643
+ quant_config=quant_config,
644
+ prefix=prefix,
645
+ )
646
+
647
def _parse_and_validate_audio_input(
648
self,
649
**kwargs: object,
vllm/model_executor/models/granite_speech_plus.py
new
+79
@@ -0,0 +1,79 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Inference-only IBM Granite Speech Plus model."""
4
+
5
+import torch
6
+from transformers import PretrainedConfig
7
+
8
+from vllm.model_executor.layers.quantization import QuantizationConfig
9
+from vllm.multimodal import MULTIMODAL_REGISTRY
10
+
11
+from .granite_speech import (
12
+ GraniteSpeechCTCEncoder,
13
+ GraniteSpeechDummyInputsBuilder,
14
+ GraniteSpeechForConditionalGeneration,
15
+ GraniteSpeechMultiModalProcessingInfo,
16
+ GraniteSpeechMultiModalProcessor,
17
+)
18
+
19
+ISO639_1_SUPPORTED_LANGS = {
20
+ "en": "English",
21
+ "fr": "French",
22
+ "de": "German",
23
+ "pt": "Portuguese",
24
+ "es": "Spanish",
25
+}
26
+
27
+
28
+class GraniteSpeechPlusCTCEncoder(GraniteSpeechCTCEncoder):
29
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
30
+ hidden_states = self.input_linear(hidden_states)
31
+ # cat_hidden_layers selects non-negative layer indices (0 = encoder
32
+ # input, N = output of layer N) whose hidden states are concatenated
33
+ # along the feature dim *in addition to* the final hidden states,
34
+ # which are always appended last.
35
+ cat_layers = set(self.config.cat_hidden_layers or [])
36
+ exported_hidden_states = []
37
+
38
+ if 0 in cat_layers:
39
+ exported_hidden_states.append(hidden_states)
40
+
41
+ for idx, layer in enumerate(self.layers, start=1):
42
+ hidden_states = layer(hidden_states, attention_dists=self.attention_dists)
43
+
44
+ # Skip the final layer here since its output is always appended
45
+ # below; capturing it twice would double-append.
46
+ if idx in cat_layers and idx != self.num_layers:
47
+ exported_hidden_states.append(hidden_states)
48
+
49
+ if idx == self.num_layers // 2:
50
+ hidden_states_mid = hidden_states.clone()
51
+ hidden_states_mid, _ = self.out(hidden_states_mid)
52
+ hidden_states_mid = self.softmax(hidden_states_mid)
53
+ hidden_states_mid, _ = self.out_mid(hidden_states_mid)
54
+ hidden_states += hidden_states_mid
55
+
56
+ if exported_hidden_states:
57
+ hidden_states = torch.cat([*exported_hidden_states, hidden_states], dim=-1)
58
+ return hidden_states
59
+
60
+
61
+@MULTIMODAL_REGISTRY.register_processor(
62
+ GraniteSpeechMultiModalProcessor,
63
+ info=GraniteSpeechMultiModalProcessingInfo,
64
+ dummy_inputs=GraniteSpeechDummyInputsBuilder,
65
+)
66
+class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGeneration):
67
+ supported_languages = ISO639_1_SUPPORTED_LANGS
68
+
69
+ def _build_encoder(
70
+ self,
71
+ config: PretrainedConfig,
72
+ quant_config: QuantizationConfig | None,
73
+ prefix: str,
74
+ ) -> GraniteSpeechCTCEncoder:
75
+ return GraniteSpeechPlusCTCEncoder(
76
+ config=config,
77
+ quant_config=quant_config,
78
+ prefix=prefix,
79
+ )
vllm/model_executor/models/registry.py
+4
@@ -419,6 +419,10 @@ _MULTIMODAL_MODELS = {
419
"granite_speech",
420
"GraniteSpeechForConditionalGeneration",
421
),
422
+ "GraniteSpeechPlusForConditionalGeneration": (
423
+ "granite_speech_plus",
424
+ "GraniteSpeechPlusForConditionalGeneration",
425
+ ),
426
"Granite4VisionForConditionalGeneration": (
427
"granite4_vision",
428
"Granite4VisionForConditionalGeneration",