A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 07:15 UTC
[Model] Implement LoRA support for Qwen3ASRForConditionalGeneration (#37247)
Signed-off-by: Peter Nguyen <petern0408@gmail.com>
Peter Nguyen committed
Apr 10, 2026 at 07:34 UTC
8d0f908b98cd692204753421dd39695f3094c8f0
4 files changed
+63
-5
docs/models/supported_models.md
+1
-1
@@ -666,7 +666,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
666
| `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
667
| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
668
| `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-4.0-1b-speech`, `ibm-granite/granite-speech-3.3-2b`, etc. | ✅︎ | ✅︎ |
669
-| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | | ✅︎ |
669
+| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | ✅︎ | ✅︎ |
670
| `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, etc. | | ✅︎ |
671
| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
672
| `WhisperForConditionalGeneration` | Whisper | `openai/whisper-small`, `openai/whisper-large-v3-turbo`, etc. | | |
vllm/model_executor/models/qwen3_asr.py
+26
@@ -37,6 +37,7 @@ from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensProm
37
from vllm.logger import init_logger
38
from vllm.model_executor.models.interfaces import (
39
MultiModalEmbeddings,
40
+ SupportsLoRA,
41
SupportsMRoPE,
42
SupportsMultiModal,
43
SupportsPP,
@@ -266,7 +267,21 @@ class Qwen3ASRForConditionalGeneration(
267
SupportsPP,
268
SupportsMRoPE,
269
SupportsTranscription,
270
+ SupportsLoRA,
271
):
272
+ # LoRA support
273
+ packed_modules_mapping = {
274
+ "qkv_proj": [
275
+ "q_proj",
276
+ "k_proj",
277
+ "v_proj",
278
+ ],
279
+ "gate_up_proj": [
280
+ "gate_proj",
281
+ "up_proj",
282
+ ],
283
+ }
284
+
285
supported_languages = ISO639_1_SUPPORTED_LANGS
286
287
hf_to_vllm_mapper = WeightsMapper(
@@ -513,6 +528,17 @@ class Qwen3ASRForConditionalGeneration(
528
tower_model=["audio_tower."],
529
)
530
531
+ def get_num_mm_encoder_tokens(self, num_audio_tokens: int) -> int:
532
+ """Return the number of tokens processed by the audio tower encoder.
533
+
534
+ Required for LoRA support on the tower module.
535
+ """
536
+ # For Qwen3-ASR, the audio tower produces one embedding per audio
537
+ # placeholder token inserted into the prompt (no additional
538
+ # merge/downsample step like vision towers). Therefore, the encoder
539
+ # token budget is identity.
540
+ return num_audio_tokens
541
+
542
@classmethod
543
def get_speech_to_text_config(
544
cls, model_config: ModelConfig, task_type: str
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+22
-3
@@ -57,6 +57,7 @@ from vllm.model_executor.layers.conv import Conv3dLayer
57
from vllm.model_executor.layers.linear import (
58
ColumnParallelLinear,
59
QKVParallelLinear,
60
+ ReplicatedLinear,
61
RowParallelLinear,
62
)
63
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -357,7 +358,13 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
358
conv_out_dim = config.downsample_hidden_size * (
359
(((config.num_mel_bins + 1) // 2 + 1) // 2 + 1) // 2
360
)
360
- self.conv_out = nn.Linear(conv_out_dim, config.d_model, bias=False)
361
+ self.conv_out = ReplicatedLinear(
362
+ conv_out_dim,
363
+ config.d_model,
364
+ bias=False,
365
+ return_bias=False,
366
+ prefix=f"{prefix}.conv_out",
367
+ )
368
369
# Transformer encoder layers
370
self.layers = nn.ModuleList(
@@ -372,9 +379,21 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
379
380
# Output layers
381
self.ln_post = nn.LayerNorm(config.d_model)
375
- self.proj1 = nn.Linear(config.d_model, config.d_model)
382
+ self.proj1 = ReplicatedLinear(
383
+ config.d_model,
384
+ config.d_model,
385
+ bias=True,
386
+ return_bias=False,
387
+ prefix=f"{prefix}.proj1",
388
+ )
389
self.act = _ACTIVATION_REGISTRY[config.activation_function]
377
- self.proj2 = nn.Linear(config.d_model, config.output_dim)
390
+ self.proj2 = ReplicatedLinear(
391
+ config.d_model,
392
+ config.output_dim,
393
+ bias=True,
394
+ return_bias=False,
395
+ prefix=f"{prefix}.proj2",
396
+ )
397
398
# Get attention backend
399
self.attn_backend = get_vit_attn_backend(
vllm/v1/worker/gpu_model_runner.py
+14
-1
@@ -2783,7 +2783,20 @@ class GPUModelRunner(
2783
)
2784
self.lora_manager.set_active_adapters(lora_requests, tower_mapping)
2785
2786
- if hasattr(self.model, "get_num_mm_connector_tokens"):
2786
+ # Only set connector mapping if the model actually has a connector.
2787
+ # Some multimodal models inherit a stub `get_num_mm_connector_tokens`
2788
+ # from `SupportsMultiModal`, which returns None and should not be
2789
+ # treated as a signal that connector LoRA is supported.
2790
+ mm_mapping = (
2791
+ self.model.get_mm_mapping() # type: ignore[attr-defined]
2792
+ if hasattr(self.model, "get_mm_mapping")
2793
+ else None
2794
+ )
2795
+ if (
2796
+ mm_mapping is not None
2797
+ and mm_mapping.connector
2798
+ and hasattr(self.model, "get_num_mm_connector_tokens")
2799
+ ):
2800
post_op_counts = [
2801
self.model.get_num_mm_connector_tokens(num_tokens) # type: ignore[attr-defined]
2802
for num_tokens in encoder_token_counts