A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 01:15 UTC
[Voxtral Realtime] Introduce global log mel max (#33574)
Signed-off-by: Patrick von Platen <patrick.v.platen@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Patrick von Platen committed
Feb 2, 2026 at 23:01 UTC
5019c59dd2d34feb2bc6e52699a36059eacf64a9
4 files changed
+19
-11
tests/entrypoints/openai/test_realtime_validation.py
+2
-2
@@ -126,8 +126,8 @@ async def test_multi_chunk_streaming(
126
assert event["type"] == "transcription.done"
127
assert event["text"] == full_text
128
assert full_text == (
129
- " He has first words I spoke in the original phonograph."
129
+ " First words I spoke in the original phonograph."
130
" A little piece of practical poetry. Mary had a little lamb,"
131
- " it squeaked with quite a flow, and everywhere that Mary went,"
131
+ " it sleeps with quite a flow, and everywhere that Mary went,"
132
" the lamb was sure to go"
133
)
tests/models/multimodal/generation/test_voxtral_realtime.py
+1
-8
@@ -37,7 +37,7 @@ EXPECTED_TEXT = [
37
(
38
" First words I spoke in the original phonograph. "
39
"A little piece of practical poetry. Mary had a little lamb,"
40
- " it sleeps with quite a snow, and everywhere that Mary went, "
40
+ " its fleece was quite a slow, and everywhere that Mary went, "
41
"the lamb was sure to go."
42
),
43
(
@@ -246,13 +246,6 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
246
247
texts = [tokenizer.decode(output_tokens) for output_tokens in output_tokens_list]
248
249
- # 'true' streaming and 'offline' streaming differ a bit because log-mels are
250
- # differently noramalized
251
- texts[0] = (
252
- texts[0]
253
- .replace("He has f", "F")
254
- .replace("its fleece was quite a slow", "it sleeps with quite a snow")
255
- )
249
texts[1] = texts[1].replace("a base hit", "OBS").replace("oh my", "oh, my")
250
251
assert texts == EXPECTED_TEXT
vllm/model_executor/models/voxtral.py
+13
-1
@@ -782,7 +782,19 @@ class VoxtralEncoderModel(nn.Module):
782
magnitudes = stft[..., :-1].abs() ** 2
783
mel_spec = self.mel_filters.T @ magnitudes
784
log_spec = torch.clamp(mel_spec, min=1e-10).log10()
785
- log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
785
+
786
+ if global_log_mel_max := self.config.global_log_mel_max:
787
+ if not isinstance(global_log_mel_max, float):
788
+ raise TypeError(f"{global_log_mel_max=} needs to be of type float.")
789
+ log_spec_max = torch.tensor(
790
+ global_log_mel_max,
791
+ device=log_spec.device,
792
+ dtype=log_spec.dtype,
793
+ )
794
+ else:
795
+ log_spec_max = log_spec.max()
796
+
797
+ log_spec = torch.maximum(log_spec, log_spec_max - 8.0)
798
log_spec = (log_spec + 4.0) / 4.0
799
return log_spec.to(input_dtype)
800
vllm/transformers_utils/configs/mistral.py
+3
@@ -248,6 +248,9 @@ def _remap_mistral_audio_args(config: dict) -> dict:
248
sliding_window=encoder_args.get("sliding_window", None),
249
block_pool_size=block_pool_size,
250
pos_embed=encoder_args.get("pos_embed", "sinusoidal"),
251
+ global_log_mel_max=encoder_args["audio_encoding_args"].get(
252
+ "global_log_mel_max"
253
+ ),
254
# only needed for RoPE
255
max_position_embeddings=block_pool_size * config["max_position_embeddings"],
256
),