A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 01:30 UTC
[Bugfix] Fix GLM-ASR audio encoder RoPE dim (#32540)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jan 18, 2026 at 19:17 UTC
38bf2ffb21d54eb67a68d4ba82125922f4a4c7ad
2 files changed
+40
-30
examples/offline_inference/audio_language.py
+28
-28
@@ -89,6 +89,34 @@ def run_gemma3n(question: str, audio_count: int) -> ModelRequestData:
89
)
90
91
92
+# GLM-ASR
93
+def run_glmasr(question: str, audio_count: int) -> ModelRequestData:
94
+ model_name = "zai-org/GLM-ASR-Nano-2512"
95
+
96
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
97
+
98
+ # GLM-ASR uses <|pad|> token for audio
99
+ audio_placeholder = "<|pad|>" * audio_count
100
+
101
+ messages = [{"role": "user", "content": f"{audio_placeholder}{question}"}]
102
+ prompt = tokenizer.apply_chat_template(
103
+ messages, tokenize=False, add_generation_prompt=True
104
+ )
105
+
106
+ engine_args = EngineArgs(
107
+ model=model_name,
108
+ trust_remote_code=True,
109
+ max_model_len=4096,
110
+ max_num_seqs=2,
111
+ limit_mm_per_prompt={"audio": audio_count},
112
+ )
113
+
114
+ return ModelRequestData(
115
+ engine_args=engine_args,
116
+ prompt=prompt,
117
+ )
118
+
119
+
120
# Granite Speech
121
def run_granite_speech(question: str, audio_count: int) -> ModelRequestData:
122
# NOTE - the setting in this example are somewhat different from what is
@@ -358,34 +386,6 @@ def run_voxtral(question: str, audio_count: int) -> ModelRequestData:
386
)
387
388
361
-# GLM-ASR
362
-def run_glmasr(question: str, audio_count: int) -> ModelRequestData:
363
- model_name = "zai-org/GLM-ASR-Nano-2512"
364
-
365
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
366
-
367
- # GLM-ASR uses <|pad|> token for audio
368
- audio_placeholder = "<|pad|>" * audio_count
369
-
370
- messages = [{"role": "user", "content": f"{audio_placeholder}{question}"}]
371
- prompt = tokenizer.apply_chat_template(
372
- messages, tokenize=False, add_generation_prompt=True
373
- )
374
-
375
- engine_args = EngineArgs(
376
- model=model_name,
377
- trust_remote_code=True,
378
- max_model_len=4096,
379
- max_num_seqs=2,
380
- limit_mm_per_prompt={"audio": audio_count},
381
- )
382
-
383
- return ModelRequestData(
384
- engine_args=engine_args,
385
- prompt=prompt,
386
- )
387
-
388
-
389
# Whisper
390
def run_whisper(question: str, audio_count: int) -> ModelRequestData:
391
assert audio_count == 1, "Whisper only support single audio input per prompt"
vllm/model_executor/models/glmasr.py
+12
-2
@@ -181,6 +181,12 @@ class GlmAsrEncoderAttention(nn.Module):
181
182
# Use vLLM's ApplyRotaryEmb CustomOp
183
# enforce_enable=True ensures the op is always enabled (important for ViT)
184
+ rope_params = getattr(config, "rope_parameters", None)
185
+ if rope_params:
186
+ partial_rotary_factor = rope_params.get("partial_rotary_factor", 0.5)
187
+ else:
188
+ partial_rotary_factor = getattr(config, "partial_rotary_factor", 0.5)
189
+ self.rotary_dim = int(self.head_dim * partial_rotary_factor)
190
self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)
191
192
# Use vLLM's MMEncoderAttention for hardware-optimized attention
@@ -226,8 +232,12 @@ class GlmAsrEncoderAttention(nn.Module):
232
# Apply rotary position embeddings using vLLM's ApplyRotaryEmb
233
# ApplyRotaryEmb expects x: [batch, seq, heads, head_dim]
234
# cos/sin: [seq_len, rotary_dim/2]
229
- q = self.apply_rotary_emb(q, rotary_pos_emb_cos, rotary_pos_emb_sin)
230
- k = self.apply_rotary_emb(k, rotary_pos_emb_cos, rotary_pos_emb_sin)
235
+ q[..., : self.rotary_dim] = self.apply_rotary_emb(
236
+ q[..., : self.rotary_dim], rotary_pos_emb_cos, rotary_pos_emb_sin
237
+ )
238
+ k[..., : self.rotary_dim] = self.apply_rotary_emb(
239
+ k[..., : self.rotary_dim], rotary_pos_emb_cos, rotary_pos_emb_sin
240
+ )
241
242
# MMEncoderAttention expects [batch, seq, num_heads, head_dim]
243
# It handles GQA internally via repeat_interleave