A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 17:00 UTC
[ROCm][CI] Fix engine teardown and text normalization to stabilize voxtral test (#37138)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Mar 15, 2026 at 23:49 UTC
d4c57863f76efd1276d5424b3f7ef8049b10802c
2 files changed
+68
-24
tests/models/multimodal/generation/test_voxtral_realtime.py
+62
-24
@@ -1,8 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import contextlib
4
from dataclasses import asdict
5
6
import pytest
7
+import pytest_asyncio
8
from mistral_common.audio import Audio
9
from mistral_common.protocol.instruct.chunk import RawAudio
10
from mistral_common.protocol.transcription.request import (
@@ -17,18 +19,21 @@ from vllm.assets.audio import AudioAsset
19
from vllm.engine.arg_utils import AsyncEngineArgs
20
from vllm.v1.engine.async_llm import AsyncLLM
21
22
+from ....utils import ROCM_ENGINE_KWARGS
23
+
24
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
21
-ENGINE_CONFIG = dict(
22
- model=MODEL_NAME,
23
- max_model_len=8192,
24
- max_num_seqs=4,
25
- limit_mm_per_prompt={"audio": 1},
26
- config_format="mistral",
27
- load_format="mistral",
28
- tokenizer_mode="mistral",
29
- enforce_eager=True,
30
- gpu_memory_utilization=0.9,
31
-)
25
+ENGINE_CONFIG = {
26
+ "model": MODEL_NAME,
27
+ "max_model_len": 8192,
28
+ "max_num_seqs": 4,
29
+ "limit_mm_per_prompt": {"audio": 1},
30
+ "config_format": "mistral",
31
+ "load_format": "mistral",
32
+ "tokenizer_mode": "mistral",
33
+ "enforce_eager": True,
34
+ "gpu_memory_utilization": 0.9,
35
+ **ROCM_ENGINE_KWARGS,
36
+}
37
38
39
EXPECTED_TEXT = [
@@ -49,6 +54,14 @@ EXPECTED_TEXT = [
54
]
55
56
57
+def _normalize(texts: list[str]) -> list[str]:
58
+ # The model occasionally transcribes "OBS" as "a base hit" and
59
+ # "oh, my" as "oh my", but both are acoustically valid. Normalise so
60
+ # the assertion is stable across runs and hardware.
61
+ texts[1] = texts[1].replace("a base hit", "OBS").replace("oh my", "oh, my")
62
+ return texts
63
+
64
+
65
@pytest.fixture
66
def audio_assets() -> list[AudioAsset]:
67
return [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
@@ -60,15 +73,27 @@ def tokenizer() -> MistralTokenizer:
73
74
75
@pytest.fixture
63
-def engine() -> LLM:
76
+def engine():
77
engine_args = EngineArgs(**ENGINE_CONFIG)
65
- return LLM(**asdict(engine_args))
78
+ llm = LLM(**asdict(engine_args))
79
+ try:
80
+ yield llm
81
+ finally:
82
+ with contextlib.suppress(Exception):
83
+ llm.llm_engine.engine_core.shutdown()
84
+ import torch
85
86
+ torch.accelerator.empty_cache()
87
68
-@pytest.fixture
69
-def async_engine() -> AsyncLLM:
88
+
89
+@pytest_asyncio.fixture
90
+async def async_engine():
91
engine_args = AsyncEngineArgs(**ENGINE_CONFIG)
71
- return AsyncLLM.from_engine_args(engine_args)
92
+ llm = AsyncLLM.from_engine_args(engine_args)
93
+ try:
94
+ yield llm
95
+ finally:
96
+ llm.shutdown()
97
98
99
def test_voxtral_realtime_forward(audio_assets, tokenizer, engine):
@@ -108,8 +133,13 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, engine):
133
sampling_params=sampling_params,
134
)
135
111
- texts = [out.outputs[0].text for out in outputs]
112
- assert texts == EXPECTED_TEXT
136
+ texts = _normalize([out.outputs[0].text for out in outputs])
137
+ for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
138
+ assert got == expected, (
139
+ f"Output mismatch at index {i}:\n"
140
+ f" got: {got!r}\n"
141
+ f" expected: {expected!r}"
142
+ )
143
144
145
@pytest.mark.asyncio
@@ -149,9 +179,17 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
179
180
output_tokens_list.append(output_tokens)
181
152
- texts = [
153
- tokenizer.decode(output_tokens, special_token_policy=SpecialTokenPolicy.IGNORE)
154
- for output_tokens in output_tokens_list
155
- ]
156
- texts[1] = texts[1].replace("a base hit", "OBS").replace("oh my", "oh, my")
157
- assert texts == EXPECTED_TEXT
182
+ texts = _normalize(
183
+ [
184
+ tokenizer.decode(
185
+ output_tokens, special_token_policy=SpecialTokenPolicy.IGNORE
186
+ )
187
+ for output_tokens in output_tokens_list
188
+ ]
189
+ )
190
+ for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
191
+ assert got == expected, (
192
+ f"Output mismatch at index {i}:\n"
193
+ f" got: {got!r}\n"
194
+ f" expected: {expected!r}"
195
+ )
tests/utils.py
+6
@@ -122,6 +122,12 @@ ROCM_EXTRA_ARGS = (
122
if current_platform.is_rocm()
123
else []
124
)
125
+# Python-API equivalent of ROCM_EXTRA_ARGS for use with EngineArgs kwargs.
126
+ROCM_ENGINE_KWARGS: dict = (
127
+ {"enable_prefix_caching": False, "max_num_seqs": 1}
128
+ if current_platform.is_rocm()
129
+ else {}
130
+)
131
132
133
class RemoteVLLMServer: