@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d722e9e614
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 20:15 UTC

Add GLM-ASR multimodal support (#31436)

Signed-off-by: baonudesifeizhai <baonudesifeizhai@gmail.com> Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

baonudesifeizhai committed Dec 31, 2025 at 10:12 UTC d722e9e614f667ed9b6f951fdc284646bf3a572c
8 files changed +764 -2
docs/models/supported_models.md
+3 -2
@@ -774,10 +774,11 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
774
775 | Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
776 |--------------|--------|-------------------|----------------------|---------------------------|
777 -| `WhisperForConditionalGeneration` | Whisper | `openai/whisper-small`, `openai/whisper-large-v3-turbo`, etc. | | |
778 -| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
777 | `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
778 +| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
779 | `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-speech-3.3-2b`, `ibm-granite/granite-speech-3.3-8b`, etc. | ✅︎ | ✅︎ |
780 +| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
781 +| `WhisperForConditionalGeneration` | Whisper | `openai/whisper-small`, `openai/whisper-large-v3-turbo`, etc. | | |
782
783 !!! note
784 `VoxtralForConditionalGeneration` requires `mistral-common[audio]` to be installed.
examples/offline_inference/audio_language.py
+29
@@ -358,6 +358,34 @@ def run_voxtral(question: str, audio_count: int) -> ModelRequestData:
358 )
359
360
361 +# GLM-ASR
362 +def run_glmasr(question: str, audio_count: int) -> ModelRequestData:
363 + model_name = "zai-org/GLM-ASR-Nano-2512"
364 +
365 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
366 +
367 + # GLM-ASR uses <|pad|> token for audio
368 + audio_placeholder = "<|pad|>" * audio_count
369 +
370 + messages = [{"role": "user", "content": f"{audio_placeholder}{question}"}]
371 + prompt = tokenizer.apply_chat_template(
372 + messages, tokenize=False, add_generation_prompt=True
373 + )
374 +
375 + engine_args = EngineArgs(
376 + model=model_name,
377 + trust_remote_code=True,
378 + max_model_len=4096,
379 + max_num_seqs=2,
380 + limit_mm_per_prompt={"audio": audio_count},
381 + )
382 +
383 + return ModelRequestData(
384 + engine_args=engine_args,
385 + prompt=prompt,
386 + )
387 +
388 +
389 # Whisper
390 def run_whisper(question: str, audio_count: int) -> ModelRequestData:
391 assert audio_count == 1, "Whisper only support single audio input per prompt"
@@ -381,6 +409,7 @@ def run_whisper(question: str, audio_count: int) -> ModelRequestData:
409 model_example_map = {
410 "audioflamingo3": run_audioflamingo3,
411 "gemma3n": run_gemma3n,
412 + "glmasr": run_glmasr,
413 "granite_speech": run_granite_speech,
414 "midashenglm": run_midashenglm,
415 "minicpmo": run_minicpmo,
tests/models/multimodal/processing/test_common.py
+14
@@ -84,6 +84,19 @@ def qwen3_vl_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
84 return mm_data
85
86
87 +def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
88 + """
89 + Patch the multimodal data for GLM-ASR model.
90 + GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
91 + """
92 + if "audio" in mm_data:
93 + audio = mm_data["audio"]
94 + if isinstance(audio, list) and len(audio) > 1:
95 + # Limit to single audio to match text requirement
96 + mm_data["audio"] = [audio[0]]
97 + return mm_data
98 +
99 +
100 # For some multimodal models, tokenizer will always add bos_token
101 # at the beginning of prompt by default, causing hf_processor outputs
102 # incorrect token ids. So we need use `add_special_tokens=False` here
@@ -108,6 +121,7 @@ MM_DATA_PATCHES = {
121 "ernie4_5_moe_vl": qwen3_vl_patch_mm_data,
122 "glm4v": glm4_1v_patch_mm_data,
123 "glm4v_moe": glm4_1v_patch_mm_data,
124 + "glmasr": glmasr_patch_mm_data,
125 "qwen3_vl": qwen3_vl_patch_mm_data,
126 "qwen3_vl_moe": qwen3_vl_patch_mm_data,
127 }
tests/models/registry.py
+5
@@ -655,6 +655,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
655 "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
656 "Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
657 "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
658 + "GlmAsrForConditionalGeneration": _HfExamplesInfo(
659 + "zai-org/GLM-ASR-Nano-2512",
660 + trust_remote_code=True,
661 + min_transformers_version="5.0",
662 + ),
663 "GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
664 "ibm-granite/granite-speech-3.3-2b"
665 ),
vllm/model_executor/models/glmasr.py new
+545
@@ -0,0 +1,545 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Iterable, Mapping, Sequence
5 +from typing import Annotated, Any, Literal, TypeAlias, cast
6 +
7 +import numpy as np
8 +import torch
9 +import torch.nn as nn
10 +from transformers import BatchFeature
11 +from transformers.models.glmasr import GlmAsrConfig, GlmAsrEncoder, GlmAsrProcessor
12 +from transformers.models.whisper import WhisperFeatureExtractor
13 +
14 +from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
15 +from vllm.config.multimodal import BaseDummyOptions
16 +from vllm.inputs.data import PromptType
17 +from vllm.model_executor.layers.activation import get_act_fn
18 +from vllm.model_executor.layers.linear import (
19 + ColumnParallelLinear,
20 + RowParallelLinear,
21 +)
22 +from vllm.model_executor.layers.quantization import QuantizationConfig
23 +from vllm.model_executor.models.module_mapping import MultiModelKeys
24 +from vllm.multimodal import MULTIMODAL_REGISTRY
25 +from vllm.multimodal.inputs import (
26 + MultiModalDataDict,
27 + MultiModalFieldConfig,
28 + MultiModalKwargsItems,
29 +)
30 +from vllm.multimodal.parse import (
31 + DictEmbeddingItems,
32 + ModalityData,
33 + ModalityDataItems,
34 + MultiModalDataItems,
35 + MultiModalDataParser,
36 +)
37 +from vllm.multimodal.processing import (
38 + PromptReplacement,
39 + PromptUpdate,
40 + PromptUpdateDetails,
41 +)
42 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
43 +from vllm.sequence import IntermediateTensors
44 +from vllm.tokenizers import cached_tokenizer_from_config
45 +from vllm.transformers_utils.processor import cached_processor_from_config
46 +from vllm.utils.tensor_schema import TensorSchema, TensorShape
47 +
48 +from .audioflamingo3 import (
49 + AudioFlamingo3MultiModalDataParser,
50 + AudioFlamingo3MultiModalProcessor,
51 + AudioFlamingo3ProcessingInfo,
52 +)
53 +from .audioflamingo3 import (
54 + _audioflamingo3_field_config as _glmasr_field_config,
55 +)
56 +from .glmasr_utils import (
57 + DEFAULT_CONV_PARAMS,
58 + DEFAULT_MAX_AUDIO_LEN_S,
59 + DEFAULT_MERGE_FACTOR,
60 + _flatten_audio_features_by_length,
61 + _get_audio_output_lengths_for_tower,
62 + _get_num_features_for_item,
63 + _group_audio_embeddings,
64 + _normalize_chunk_counts,
65 +)
66 +from .interfaces import (
67 + MultiModalEmbeddings,
68 + SupportsLoRA,
69 + SupportsMultiModal,
70 + SupportsPP,
71 + SupportsTranscription,
72 +)
73 +from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
74 +from .whisper import ISO639_1_SUPPORTED_LANGS
75 +
76 +
77 +class GlmAsrFeatureInputs(TensorSchema):
78 + """
79 + Dimensions:
80 + - num_chunks: Number of audio chunks (flattened)
81 + - nmb: Number of mel bins
82 + - num_audios: Number of original audio files
83 + """
84 +
85 + type: Literal["audio_features"]
86 + input_features: Annotated[
87 + torch.Tensor | list[torch.Tensor],
88 + TensorShape("num_chunks", "nmb", "chunk_length", dynamic_dims={"chunk_length"}),
89 + ]
90 + feature_attention_mask: Annotated[
91 + torch.Tensor | list[torch.Tensor],
92 + TensorShape("num_chunks", "chunk_length", dynamic_dims={"chunk_length"}),
93 + ]
94 + chunk_counts: Annotated[
95 + torch.Tensor | list[torch.Tensor],
96 + TensorShape("num_audios"),
97 + ]
98 +
99 +
100 +class GlmAsrEmbeddingInputs(TensorSchema):
101 + """
102 + Dimensions:
103 + - bn: Batch size
104 + - naf: Number of audio features
105 + - hs: Hidden size (must match the hidden size of language model
106 + backbone)
107 + """
108 +
109 + type: Literal["audio_embeds"] = "audio_embeds"
110 + audio_embeds: Annotated[
111 + list[torch.Tensor],
112 + TensorShape("bn", "naf", "hs", dynamic_dims={"naf"}),
113 + ]
114 +
115 +
116 +GlmAsrInputs: TypeAlias = GlmAsrFeatureInputs | GlmAsrEmbeddingInputs
117 +
118 +
119 +class GlmAsrMultiModalProjector(nn.Module):
120 + def __init__(
121 + self,
122 + config: GlmAsrConfig,
123 + quant_config: QuantizationConfig | None = None,
124 + prefix: str = "",
125 + ):
126 + super().__init__()
127 + self.linear_1 = ColumnParallelLinear(
128 + input_size=config.audio_config.intermediate_size,
129 + output_size=config.text_config.hidden_size * 2,
130 + quant_config=quant_config,
131 + prefix=f"{prefix}.linear_1",
132 + )
133 + self.act = get_act_fn(config.projector_hidden_act)
134 + self.linear_2 = RowParallelLinear(
135 + input_size=config.text_config.hidden_size * 2,
136 + output_size=config.text_config.hidden_size,
137 + quant_config=quant_config,
138 + prefix=f"{prefix}.linear_2",
139 + )
140 +
141 + def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
142 + hidden_states, _ = self.linear_1(audio_features)
143 + hidden_states = self.act(hidden_states)
144 + hidden_states, _ = self.linear_2(hidden_states)
145 + return hidden_states
146 +
147 +
148 +class GlmAsrProcessingInfo(AudioFlamingo3ProcessingInfo):
149 + def get_hf_config(self) -> GlmAsrConfig:
150 + return self.ctx.get_hf_config(GlmAsrConfig)
151 +
152 + def get_hf_processor(self, **kwargs: object) -> GlmAsrProcessor:
153 + return self.ctx.get_hf_processor(GlmAsrProcessor, **kwargs)
154 +
155 + def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
156 + # Reuse parent implementation, but add type annotation and assertion
157 + feature_extractor = super().get_feature_extractor(**kwargs)
158 + assert isinstance(feature_extractor, WhisperFeatureExtractor)
159 + return feature_extractor
160 +
161 +
162 +class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
163 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
164 + num_audios = mm_counts.get("audio", 0)
165 + hf_processor = self.info.get_hf_processor()
166 + return hf_processor.audio_token * num_audios
167 +
168 + def get_dummy_mm_data(
169 + self,
170 + seq_len: int,
171 + mm_counts: Mapping[str, int],
172 + mm_options: Mapping[str, BaseDummyOptions] | None = None,
173 + ) -> MultiModalDataDict:
174 + feature_extractor = self.info.get_feature_extractor()
175 + sampling_rate = feature_extractor.sampling_rate
176 + num_audios = mm_counts.get("audio", 0)
177 + audio_overrides = mm_options.get("audio") if mm_options else None
178 +
179 + max_audio_len = getattr(
180 + self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S
181 + )
182 + audio_len = int(max_audio_len * sampling_rate)
183 +
184 + return {
185 + "audio": self._get_dummy_audios(
186 + length=audio_len, num_audios=num_audios, overrides=audio_overrides
187 + )
188 + }
189 +
190 +
191 +class GlmAsrMultiModalDataParser(AudioFlamingo3MultiModalDataParser):
192 + def _parse_audio_data(
193 + self,
194 + data: dict[str, torch.Tensor] | ModalityData[Any],
195 + ) -> ModalityDataItems[Any, Any] | None:
196 + if isinstance(data, dict):
197 + return DictEmbeddingItems(
198 + data,
199 + modality="audio",
200 + required_fields={"audio_embeds"},
201 + fields_factory=_glmasr_field_config,
202 + )
203 + return super()._parse_audio_data(data)
204 +
205 +
206 +class GlmAsrMultiModalProcessor(AudioFlamingo3MultiModalProcessor):
207 + def _get_data_parser(self) -> MultiModalDataParser:
208 + feature_extractor = self.info.get_feature_extractor()
209 + return GlmAsrMultiModalDataParser(target_sr=feature_extractor.sampling_rate)
210 +
211 + def _calculate_chunk_counts(
212 + self,
213 + audio_list: list[Any],
214 + feature_extractor: WhisperFeatureExtractor,
215 + processor: GlmAsrProcessor,
216 + ) -> list[int]:
217 + """Calculate chunk counts for each audio."""
218 + sampling_rate = feature_extractor.sampling_rate
219 + chunk_length = feature_extractor.chunk_length
220 + max_audio_len = getattr(processor, "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S)
221 + window_size = int(sampling_rate * chunk_length)
222 + max_windows = int(max_audio_len // chunk_length)
223 +
224 + chunk_counts = []
225 + for audio in audio_list:
226 + n_samples = len(audio) if isinstance(audio, list) else audio.shape[0]
227 + n_chunks = max(1, (n_samples + window_size - 1) // window_size)
228 + chunk_counts.append(min(n_chunks, max_windows))
229 + return chunk_counts
230 +
231 + def _call_hf_processor(
232 + self,
233 + prompt: str,
234 + mm_data: dict[str, object],
235 + mm_kwargs: Mapping[str, Any],
236 + tok_kwargs: Mapping[str, object],
237 + ) -> BatchFeature:
238 + # Normalize input: handle deprecated key and list conversion.
239 + if "audios" in mm_data:
240 + mm_data["audio"] = mm_data.pop("audios")
241 +
242 + audio = mm_data.get("audio", [])
243 + audio_list = [audio] if audio and not isinstance(audio, list) else audio
244 +
245 + # Early return for text-only.
246 + if not audio_list:
247 + prompt_ids = self.info.get_tokenizer().encode(prompt)
248 + prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
249 + return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
250 +
251 + # Get processor for chunk counts calculation
252 + processor = self.info.get_hf_processor(**mm_kwargs)
253 +
254 + # Call parent method (it will handle sampling_rate)
255 + outputs = super()._call_hf_processor(
256 + prompt=prompt,
257 + mm_data=mm_data,
258 + mm_kwargs=mm_kwargs,
259 + tok_kwargs=tok_kwargs,
260 + )
261 +
262 + # Postprocess: rename mask and add chunk counts.
263 + if "input_features_mask" in outputs:
264 + outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
265 +
266 + # Override chunk counts calculation with GLM-ASR specific logic
267 + chunk_counts = self._calculate_chunk_counts(
268 + audio_list, processor.feature_extractor, processor
269 + )
270 + outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
271 +
272 + return outputs
273 +
274 + def _get_mm_fields_config(
275 + self,
276 + hf_inputs: BatchFeature,
277 + hf_processor_mm_kwargs: Mapping[str, object],
278 + ) -> Mapping[str, MultiModalFieldConfig]:
279 + return _glmasr_field_config(hf_inputs)
280 +
281 + def _get_prompt_updates(
282 + self,
283 + mm_items: MultiModalDataItems,
284 + hf_processor_mm_kwargs: Mapping[str, object],
285 + out_mm_kwargs: MultiModalKwargsItems,
286 + ) -> Sequence[PromptUpdate]:
287 + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
288 + tokenizer = self.info.get_tokenizer()
289 + vocab = tokenizer.get_vocab()
290 + config = self.info.get_hf_config()
291 +
292 + audio_token = getattr(processor, "audio_token", "<|pad|>")
293 + audio_token_id = vocab.get(audio_token)
294 + if audio_token_id is None:
295 + audio_token_id = processor.audio_token_id
296 +
297 + merge_factor = getattr(config, "merge_factor", DEFAULT_MERGE_FACTOR)
298 + out_mm_data = out_mm_kwargs.get_data()
299 + feature_attention_mask = out_mm_data.get("feature_attention_mask")
300 + chunk_counts = out_mm_data.get("chunk_counts")
301 +
302 + def get_replacement_glmasr(item_idx: int):
303 + conv_params = getattr(config, "conv_params", DEFAULT_CONV_PARAMS)
304 + audio_embeds = out_mm_data.get("audio_embeds")
305 + num_features = _get_num_features_for_item(
306 + feature_attention_mask,
307 + chunk_counts,
308 + item_idx,
309 + audio_embeds,
310 + merge_factor,
311 + conv_params,
312 + )
313 +
314 + if num_features == 0:
315 + raise ValueError("Audio is too short")
316 +
317 + audio_tokens = [audio_token_id] * int(num_features)
318 + return PromptUpdateDetails.select_token_id(
319 + audio_tokens,
320 + embed_token_id=audio_token_id,
321 + )
322 +
323 + return [
324 + PromptReplacement(
325 + modality="audio",
326 + target=audio_token,
327 + replacement=get_replacement_glmasr,
328 + )
329 + ]
330 +
331 +
332 +@MULTIMODAL_REGISTRY.register_processor(
333 + GlmAsrMultiModalProcessor,
334 + info=GlmAsrProcessingInfo,
335 + dummy_inputs=GlmAsrDummyInputsBuilder,
336 +)
337 +class GlmAsrForConditionalGeneration(
338 + nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, SupportsTranscription
339 +):
340 + supported_languages = ISO639_1_SUPPORTED_LANGS
341 +
342 + packed_modules_mapping = {
343 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
344 + "gate_up_proj": ["gate_proj", "up_proj"],
345 + }
346 +
347 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
348 + super().__init__()
349 + config = vllm_config.model_config.hf_config
350 + quant_config = vllm_config.quant_config
351 + multimodal_config = vllm_config.model_config.multimodal_config
352 + self.config = config
353 + self.multimodal_config = multimodal_config
354 +
355 + self.audio_tower = GlmAsrEncoder(config.audio_config)
356 + self.multi_modal_projector = GlmAsrMultiModalProjector(
357 + config,
358 + quant_config=quant_config,
359 + prefix=maybe_prefix(prefix, "multi_modal_projector"),
360 + )
361 + self.quant_config = quant_config
362 +
363 + self.language_model = init_vllm_registered_model(
364 + vllm_config=vllm_config,
365 + hf_config=config.text_config,
366 + prefix=maybe_prefix(prefix, "language_model"),
367 + architectures=["LlamaForCausalLM"],
368 + )
369 +
370 + self.make_empty_intermediate_tensors = (
371 + self.language_model.make_empty_intermediate_tensors
372 + )
373 +
374 + @classmethod
375 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
376 + if modality.startswith("audio"):
377 + return "<|begin_of_audio|><|pad|><|end_of_audio|>"
378 +
379 + raise ValueError("Only audio modality is supported")
380 +
381 + def get_mm_mapping(self) -> MultiModelKeys:
382 + return MultiModelKeys.from_string_field(
383 + language_model="language_model.",
384 + connector="multi_modal_projector.",
385 + tower_model="audio_tower.",
386 + )
387 +
388 + def _parse_and_validate_audio_input(self, **kwargs: object) -> GlmAsrInputs | None:
389 + audio_embeds = kwargs.pop("audio_embeds", None)
390 + if audio_embeds is not None:
391 + return GlmAsrEmbeddingInputs(type="audio_embeds", audio_embeds=audio_embeds)
392 +
393 + input_features = kwargs.pop("input_features", None)
394 + if input_features is None:
395 + return None
396 +
397 + return GlmAsrFeatureInputs(
398 + type="audio_features",
399 + input_features=input_features,
400 + feature_attention_mask=kwargs.pop("feature_attention_mask", None),
401 + chunk_counts=kwargs.pop("chunk_counts", None),
402 + )
403 +
404 + def _process_audio_input(
405 + self, audio_input: GlmAsrInputs
406 + ) -> torch.Tensor | tuple[torch.Tensor, ...]:
407 + if audio_input["type"] == "audio_embeds":
408 + return tuple(audio_input["audio_embeds"])
409 +
410 + input_features = audio_input["input_features"]
411 + feature_attention_mask = audio_input["feature_attention_mask"]
412 +
413 + if isinstance(input_features, list):
414 + input_features = torch.cat(input_features, dim=0)
415 + feature_attention_mask = torch.cat(feature_attention_mask, dim=0)
416 +
417 + num_chunks = input_features.shape[0]
418 + chunk_counts = _normalize_chunk_counts(
419 + audio_input.get("chunk_counts"), num_chunks=num_chunks
420 + )
421 +
422 + audio_hidden_states = self.audio_tower(input_features).last_hidden_state
423 + audio_hidden_states = audio_hidden_states.reshape(
424 + num_chunks,
425 + -1,
426 + self.config.audio_config.intermediate_size,
427 + )
428 + audio_features = self.multi_modal_projector(audio_hidden_states)
429 +
430 + merge_factor = getattr(self.config, "merge_factor", DEFAULT_MERGE_FACTOR)
431 + conv_params = getattr(self.config, "conv_params", DEFAULT_CONV_PARAMS)
432 +
433 + audio_output_lengths = _get_audio_output_lengths_for_tower(
434 + self.audio_tower,
435 + feature_attention_mask.sum(-1),
436 + merge_factor,
437 + conv_params,
438 + )
439 +
440 + masked_audio_features = _flatten_audio_features_by_length(
441 + audio_features, audio_output_lengths
442 + )
443 +
444 + chunk_embeddings = torch.split(
445 + masked_audio_features, audio_output_lengths.flatten().tolist()
446 + )
447 + return _group_audio_embeddings(chunk_embeddings, chunk_counts)
448 +
449 + def get_language_model(self) -> torch.nn.Module:
450 + return self.language_model
451 +
452 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
453 + audio_input = self._parse_and_validate_audio_input(**kwargs)
454 + if audio_input is None:
455 + return []
456 + masked_audio_features = self._process_audio_input(audio_input)
457 + return masked_audio_features
458 +
459 + def forward(
460 + self,
461 + input_ids: torch.Tensor,
462 + positions: torch.Tensor,
463 + intermediate_tensors: IntermediateTensors | None = None,
464 + inputs_embeds: torch.Tensor | None = None,
465 + **kwargs: object,
466 + ) -> torch.Tensor | IntermediateTensors:
467 + if intermediate_tensors is not None:
468 + inputs_embeds = None
469 +
470 + hidden_states = self.language_model.model(
471 + input_ids,
472 + positions,
473 + intermediate_tensors,
474 + inputs_embeds=inputs_embeds,
475 + )
476 + return hidden_states
477 +
478 + def compute_logits(
479 + self,
480 + hidden_states: torch.Tensor,
481 + ) -> torch.Tensor | None:
482 + return self.language_model.compute_logits(hidden_states)
483 +
484 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
485 + skip_prefixes = ["audio_tower.embed_positions"]
486 + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
487 + return loader.load_weights(weights)
488 +
489 + @classmethod
490 + def _get_audio_token(cls, model_config: ModelConfig) -> str:
491 + """Get the audio token from processor.
492 +
493 + Similar to get_placeholder_str but returns single token.
494 + """
495 + processor = cached_processor_from_config(model_config)
496 + return getattr(processor, "audio_token", "<|pad|>")
497 +
498 + @classmethod
499 + def get_speech_to_text_config(
500 + cls, model_config: ModelConfig, task_type: str
501 + ) -> SpeechToTextConfig:
502 + processor = cached_processor_from_config(model_config)
503 + feature_extractor = processor.feature_extractor
504 + max_audio_clip_s = getattr(processor, "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S)
505 + return SpeechToTextConfig(
506 + max_audio_clip_s=max_audio_clip_s,
507 + sample_rate=feature_extractor.sampling_rate,
508 + )
509 +
510 + @classmethod
511 + def get_generation_prompt(
512 + cls,
513 + audio: np.ndarray,
514 + model_config: ModelConfig,
515 + stt_config: SpeechToTextConfig,
516 + language: str | None,
517 + task_type: Literal["transcribe", "translate"],
518 + request_prompt: str,
519 + to_language: str | None,
520 + ) -> PromptType:
521 + """Get the generation prompt to be used for transcription requests."""
522 + tokenizer = cached_tokenizer_from_config(model_config)
523 + audio_token = cls._get_audio_token(model_config)
524 +
525 + if task_type == "translate":
526 + full_lang_name_to = cls.supported_languages.get(to_language, to_language)
527 + user_content = f"{audio_token}translate the speech to {full_lang_name_to}"
528 + elif task_type == "transcribe":
529 + user_content = (
530 + f"{audio_token}can you transcribe the speech into a written format?"
531 + )
532 + else:
533 + raise ValueError(f"Unsupported task type {task_type}")
534 +
535 + messages = [{"role": "user", "content": user_content}]
536 + prompt = tokenizer.apply_chat_template(
537 + messages, tokenize=False, add_generation_prompt=True
538 + )
539 +
540 + prompt_token_ids = tokenizer.encode(prompt)
541 + prompt_dict = {
542 + "prompt_token_ids": prompt_token_ids,
543 + "multi_modal_data": {"audio": audio},
544 + }
545 + return cast(PromptType, prompt_dict)
vllm/model_executor/models/glmasr_utils.py new
+165
@@ -0,0 +1,165 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Sequence
5 +from typing import cast
6 +
7 +import torch
8 +import torch.nn as nn
9 +
10 +DEFAULT_MAX_AUDIO_LEN_S = 655
11 +DEFAULT_MERGE_FACTOR = 4
12 +# Default convolution parameters: (padding, kernel_size, stride)
13 +# These correspond to the two conv layers in GlmAsrEncoder
14 +DEFAULT_CONV_PARAMS = [(1, 3, 1), (1, 3, 2)]
15 +
16 +
17 +def _calculate_conv_output_length(
18 + input_length: torch.Tensor, padding: int, kernel_size: int, stride: int
19 +) -> torch.Tensor:
20 + """Calculate Conv1d output length using standard formula."""
21 + # Standard formula: floor((input + 2*padding - kernel_size) / stride) + 1
22 + return (input_length + 2 * padding - kernel_size) // stride + 1
23 +
24 +
25 +def _as_list_chunk_counts(
26 + chunk_counts: torch.Tensor | list[int] | list[torch.Tensor],
27 +) -> list[int]:
28 + if isinstance(chunk_counts, torch.Tensor):
29 + return chunk_counts.tolist()
30 + if chunk_counts and isinstance(chunk_counts[0], torch.Tensor):
31 + tensor_counts = cast(list[torch.Tensor], chunk_counts)
32 + return [int(c.item()) for c in tensor_counts]
33 + return [int(c) for c in chunk_counts]
34 +
35 +
36 +def _normalize_chunk_counts(
37 + chunk_counts: torch.Tensor | list[int] | list[torch.Tensor] | None,
38 + num_chunks: int,
39 +) -> list[int]:
40 + if chunk_counts is None:
41 + return [1] * num_chunks
42 + return _as_list_chunk_counts(chunk_counts)
43 +
44 +
45 +def _get_audio_output_lengths_from_lengths(
46 + audio_lengths: torch.Tensor,
47 + merge_factor: int,
48 + conv_params: list[tuple[int, int, int]],
49 +) -> torch.Tensor:
50 + for padding, kernel_size, stride in conv_params:
51 + audio_lengths = _calculate_conv_output_length(
52 + audio_lengths, padding, kernel_size, stride
53 + )
54 + return (audio_lengths - merge_factor) // merge_factor + 1
55 +
56 +
57 +def _get_audio_output_lengths_from_mask(
58 + mask: torch.Tensor,
59 + merge_factor: int,
60 + conv_params: list[tuple[int, int, int]],
61 +) -> torch.Tensor:
62 + audio_lengths = mask.sum(-1)
63 + return _get_audio_output_lengths_from_lengths(
64 + audio_lengths, merge_factor, conv_params
65 + )
66 +
67 +
68 +def _get_audio_output_lengths_for_tower(
69 + audio_tower: nn.Module,
70 + audio_lengths: torch.Tensor,
71 + merge_factor: int,
72 + conv_params: list[tuple[int, int, int]],
73 +) -> torch.Tensor:
74 + if hasattr(audio_tower, "_get_feat_extract_output_lengths"):
75 + _, audio_output_lengths = audio_tower._get_feat_extract_output_lengths(
76 + audio_lengths
77 + )
78 + return audio_output_lengths
79 + return _get_audio_output_lengths_from_lengths(
80 + audio_lengths, merge_factor, conv_params
81 + )
82 +
83 +
84 +def _flatten_audio_features_by_length(
85 + audio_features: torch.Tensor,
86 + audio_output_lengths: torch.Tensor,
87 +) -> torch.Tensor:
88 + num_chunks, max_audio_tokens, embed_dim = audio_features.shape
89 + audio_output_lengths = audio_output_lengths.unsqueeze(1)
90 + audio_features_mask = (
91 + torch.arange(max_audio_tokens)
92 + .expand(num_chunks, max_audio_tokens)
93 + .to(audio_output_lengths.device)
94 + < audio_output_lengths
95 + )
96 + return audio_features[audio_features_mask].view(-1, embed_dim)
97 +
98 +
99 +def _group_audio_embeddings(
100 + chunk_embeddings: Sequence[torch.Tensor],
101 + chunk_counts: Sequence[int],
102 +) -> tuple[torch.Tensor, ...]:
103 + grouped_embeddings = []
104 + current_idx = 0
105 + for count in chunk_counts:
106 + audio_chunks = chunk_embeddings[current_idx : current_idx + count]
107 + grouped_embeddings.append(torch.cat(audio_chunks, dim=0))
108 + current_idx += count
109 + return tuple(grouped_embeddings)
110 +
111 +
112 +def _normalize_to_tensor(mask: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
113 + """Convert mask to tensor, handling both list and tensor formats."""
114 + if isinstance(mask, list):
115 + return (
116 + torch.stack(mask)
117 + if mask and isinstance(mask[0], torch.Tensor)
118 + else torch.tensor(mask)
119 + )
120 + return mask
121 +
122 +
123 +def _extract_mask_for_item(
124 + feature_attention_mask: torch.Tensor | list[torch.Tensor],
125 + chunk_counts: torch.Tensor | list[int] | None,
126 + item_idx: int,
127 +) -> torch.Tensor:
128 + """Extract attention mask for a specific audio item."""
129 + if chunk_counts is None:
130 + # Single item per audio
131 + mask = feature_attention_mask[item_idx]
132 + if isinstance(feature_attention_mask, torch.Tensor):
133 + return mask.unsqueeze(0)
134 + return _normalize_to_tensor(mask)
135 +
136 + # Multiple chunks per audio: calculate slice indices
137 + counts = _as_list_chunk_counts(chunk_counts)
138 + start_idx = sum(counts[:item_idx])
139 + end_idx = start_idx + counts[item_idx]
140 +
141 + # Extract slice
142 + if isinstance(feature_attention_mask, torch.Tensor):
143 + return feature_attention_mask[start_idx:end_idx]
144 + mask_slice = feature_attention_mask[start_idx:end_idx]
145 + return _normalize_to_tensor(mask_slice)
146 +
147 +
148 +def _get_num_features_for_item(
149 + feature_attention_mask: torch.Tensor | None,
150 + chunk_counts: torch.Tensor | list[int] | None,
151 + item_idx: int,
152 + audio_embeds: list[torch.Tensor] | None,
153 + merge_factor: int,
154 + conv_params: list[tuple[int, int, int]],
155 +) -> int:
156 + """Get number of features for a specific audio item."""
157 + if feature_attention_mask is not None:
158 + mask = _extract_mask_for_item(feature_attention_mask, chunk_counts, item_idx)
159 + audio_output_lengths = _get_audio_output_lengths_from_mask(
160 + mask, merge_factor, conv_params
161 + )
162 + return audio_output_lengths.sum().item()
163 + if audio_embeds is not None:
164 + return audio_embeds[item_idx].shape[0]
165 + raise ValueError("Either feature_attention_mask or audio_embeds must be provided")
vllm/model_executor/models/registry.py
+1
@@ -304,6 +304,7 @@ _MULTIMODAL_MODELS = {
304 "gemma3n_mm",
305 "Gemma3nForConditionalGeneration",
306 ),
307 + "GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
308 "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
309 "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"), # noqa: E501
310 "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"), # noqa: E501
vllm/transformers_utils/config.py
+2
@@ -136,6 +136,7 @@ class HFConfigParser(ConfigParserBase):
136 model,
137 revision=revision,
138 code_revision=code_revision,
139 + trust_remote_code=trust_remote_code,
140 token=_get_hf_token(),
141 **kwargs,
142 )
@@ -157,6 +158,7 @@ class HFConfigParser(ConfigParserBase):
158 model,
159 revision=revision,
160 code_revision=code_revision,
161 + trust_remote_code=trust_remote_code,
162 token=_get_hf_token(),
163 **kwargs,
164 )