A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 21:30 UTC
[CI] Consolidate Speech to Text tests (#42274)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
May 12, 2026 at 03:50 UTC
a0dc7a0f368a3f212d37e39ba4dd7b682e378fe2
18 files changed
+115
-68
.buildkite/test-amd.yaml
+30
-4
@@ -1322,7 +1322,6 @@ steps:
1322
commands:
1323
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
1324
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
1325
- - pytest -v -s entrypoints/openai/speech_to_text/
1325
- pytest -v -s entrypoints/test_chat_utils.py
1326
1327
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
@@ -1338,7 +1337,21 @@ steps:
1337
- tests/entrypoints/test_chat_utils
1338
commands:
1339
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
1341
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
1340
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
1341
+
1342
+- label: Entrypoints Integration (Speech to Text) # TBD
1343
+ timeout_in_minutes: 180
1344
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1345
+ agent_pool: mi300_1
1346
+ fast_check: true
1347
+ torch_nightly: true
1348
+ working_dir: "/vllm-workspace/tests"
1349
+ source_file_dependencies:
1350
+ - vllm/
1351
+ - tests/entrypoints/speech_to_text
1352
+ commands:
1353
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1354
+ - pytest -v -s entrypoints/speech_to_text
1355
1356
- label: Entrypoints Integration (LLM) # TBD
1357
timeout_in_minutes: 180
@@ -2768,7 +2781,6 @@ steps:
2781
commands:
2782
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
2783
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
2771
- - pytest -v -s entrypoints/openai/speech_to_text/
2784
- pytest -v -s entrypoints/test_chat_utils.py
2785
2786
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
@@ -2784,7 +2796,21 @@ steps:
2796
- tests/entrypoints/test_chat_utils
2797
commands:
2798
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
2787
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
2799
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
2800
+
2801
+- label: Entrypoints Integration (Speech to Text) # TBD
2802
+ timeout_in_minutes: 180
2803
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355]
2804
+ agent_pool: mi355_1
2805
+ fast_check: true
2806
+ torch_nightly: true
2807
+ working_dir: "/vllm-workspace/tests"
2808
+ source_file_dependencies:
2809
+ - vllm/
2810
+ - tests/entrypoints/speech_to_text
2811
+ commands:
2812
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2813
+ - pytest -v -s entrypoints/speech_to_text
2814
2815
- label: Entrypoints Integration (Pooling) # TBD
2816
timeout_in_minutes: 180
.buildkite/test_areas/entrypoints.yaml
+14
-5
@@ -11,7 +11,7 @@ steps:
11
- tests/entrypoints/
12
commands:
13
- pytest -v -s entrypoints/openai/tool_parsers
14
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
14
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text
15
16
- label: Entrypoints Integration (LLM)
17
key: entrypoints-integration-llm
@@ -44,7 +44,6 @@ steps:
44
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
45
- pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
46
47
-
47
- label: Entrypoints Integration (API Server openai - Part 2)
48
key: entrypoints-integration-api-server-openai-part-2
49
timeout_in_minutes: 50
@@ -55,7 +54,6 @@ steps:
54
- tests/entrypoints/test_chat_utils
55
commands:
56
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
58
- - pytest -v -s entrypoints/openai/speech_to_text/
57
- pytest -v -s entrypoints/test_chat_utils.py
58
59
- label: Entrypoints Integration (API Server openai - Part 3)
@@ -69,7 +67,7 @@ steps:
67
- tests/entrypoints/test_chat_utils
68
commands:
69
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
72
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
70
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
71
72
- label: Entrypoints Integration (API Server 2)
73
key: entrypoints-integration-api-server-2
@@ -86,6 +84,17 @@ steps:
84
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
85
- pytest -v -s tool_use
86
87
+- label: Entrypoints Integration (Speech to Text)
88
+ key: entrypoints-integration-speech_to_text
89
+ timeout_in_minutes: 50
90
+ working_dir: "/vllm-workspace/tests"
91
+ source_file_dependencies:
92
+ - vllm/
93
+ - tests/entrypoints/speech_to_text
94
+ commands:
95
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
96
+ - pytest -v -s entrypoints/speech_to_text
97
+
98
- label: Entrypoints Integration (Pooling)
99
key: entrypoints-integration-pooling
100
timeout_in_minutes: 50
@@ -115,5 +124,5 @@ steps:
124
- csrc/
125
- vllm/entrypoints/openai/
126
- vllm/model_executor/models/whisper.py
118
- commands: # LMEval+Transcription WER check
127
+ commands: # LMEval
128
- pytest -s entrypoints/openai/correctness/
pyproject.toml
+2
-1
@@ -123,7 +123,8 @@ python = "./.venv"
123
extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizers_/*",
124
"benchmarks/sonnet.txt", "tests/lora/data/*", "build/*",
125
"examples/pooling/token_embed/*", "tests/models/language/pooling/*",
126
- "vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", "tests/entrypoints/openai/speech_to_text/test_transcription_validation.py",
126
+ "vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*",
127
+ "tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py",
128
"docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html",
129
"tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*"]
130
ignore-hidden = false
tests/entrypoints/openai/chat_completion/test_enable_force_include_usage.py
-55
@@ -67,58 +67,3 @@ async def test_chat_with_enable_force_include_usage(
67
chunk.usage.prompt_tokens + chunk.usage.completion_tokens
68
)
69
last_completion_tokens = chunk.usage.completion_tokens
70
-
71
-
72
-@pytest.fixture(scope="module")
73
-def transcription_server_with_force_include_usage():
74
- args = [
75
- # use half precision for speed and memory savings in CI environment
76
- "--dtype",
77
- "bfloat16",
78
- "--max-num-seqs",
79
- "4",
80
- "--enforce-eager",
81
- "--enable-force-include-usage",
82
- "--gpu-memory-utilization",
83
- "0.2",
84
- ]
85
-
86
- with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
87
- yield remote_server
88
-
89
-
90
-@pytest_asyncio.fixture
91
-async def transcription_client_with_force_include_usage(
92
- transcription_server_with_force_include_usage,
93
-):
94
- async with (
95
- transcription_server_with_force_include_usage.get_async_client() as async_client
96
- ):
97
- yield async_client
98
-
99
-
100
-@pytest.mark.asyncio
101
-async def test_transcription_with_enable_force_include_usage(
102
- transcription_client_with_force_include_usage, winning_call
103
-):
104
- res = (
105
- await transcription_client_with_force_include_usage.audio.transcriptions.create(
106
- model="openai/whisper-large-v3-turbo",
107
- file=winning_call,
108
- language="en",
109
- temperature=0.0,
110
- stream=True,
111
- timeout=30,
112
- )
113
- )
114
-
115
- async for chunk in res:
116
- if not len(chunk.choices):
117
- # final usage sent
118
- usage = chunk.usage
119
- assert isinstance(usage, dict)
120
- assert usage["prompt_tokens"] > 0
121
- assert usage["completion_tokens"] > 0
122
- assert usage["total_tokens"] > 0
123
- else:
124
- assert not hasattr(chunk, "usage")
tests/entrypoints/speech_to_text/__init__.py
renamed
tests/entrypoints/speech_to_text/conftest.py
renamed
tests/entrypoints/speech_to_text/correctness/__init__.py
renamed
tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py
renamed
tests/entrypoints/speech_to_text/realtime/__init__.py
tests/entrypoints/speech_to_text/realtime/test_realtime_validation.py
renamed
+6
-1
@@ -10,7 +10,7 @@ import pybase64 as base64
10
import pytest
11
import websockets
12
13
-from tests.entrypoints.openai.conftest import add_attention_backend
13
+from tests.entrypoints.speech_to_text.conftest import add_attention_backend
14
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
15
from vllm.assets.audio import AudioAsset
16
from vllm.multimodal.media.audio import load_audio
@@ -163,6 +163,11 @@ async def test_multi_chunk_streaming(
163
" A little piece of practical poetry. Mary had a little lamb,"
164
" it sleeps with quite a flow, and everywhere that Mary went,"
165
" the lamb was sure to go."
166
+ ) or full_text == (
167
+ " First words I spoke in the original phonograph."
168
+ " A little piece of practical poetry. Mary had a little lamb,"
169
+ " it squeaked with quite a flow, and everywhere that Mary went,"
170
+ " the lamb was sure to go."
171
)
172
173
tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py
renamed
tests/entrypoints/speech_to_text/transcription/__init__.py
tests/entrypoints/speech_to_text/transcription/test_enable_force_include_usage.py
new
+61
@@ -0,0 +1,61 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import pytest
4
+import pytest_asyncio
5
+
6
+from tests.utils import RemoteOpenAIServer
7
+
8
+
9
+@pytest.fixture(scope="module")
10
+def transcription_server_with_force_include_usage():
11
+ args = [
12
+ # use half precision for speed and memory savings in CI environment
13
+ "--dtype",
14
+ "bfloat16",
15
+ "--max-num-seqs",
16
+ "4",
17
+ "--enforce-eager",
18
+ "--enable-force-include-usage",
19
+ "--gpu-memory-utilization",
20
+ "0.2",
21
+ ]
22
+
23
+ with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
24
+ yield remote_server
25
+
26
+
27
+@pytest_asyncio.fixture
28
+async def transcription_client_with_force_include_usage(
29
+ transcription_server_with_force_include_usage,
30
+):
31
+ async with (
32
+ transcription_server_with_force_include_usage.get_async_client() as async_client
33
+ ):
34
+ yield async_client
35
+
36
+
37
+@pytest.mark.asyncio
38
+async def test_transcription_with_enable_force_include_usage(
39
+ transcription_client_with_force_include_usage, winning_call
40
+):
41
+ res = (
42
+ await transcription_client_with_force_include_usage.audio.transcriptions.create(
43
+ model="openai/whisper-large-v3-turbo",
44
+ file=winning_call,
45
+ language="en",
46
+ temperature=0.0,
47
+ stream=True,
48
+ timeout=30,
49
+ )
50
+ )
51
+
52
+ async for chunk in res:
53
+ if not len(chunk.choices):
54
+ # final usage sent
55
+ usage = chunk.usage
56
+ assert isinstance(usage, dict)
57
+ assert usage["prompt_tokens"] > 0
58
+ assert usage["completion_tokens"] > 0
59
+ assert usage["total_tokens"] > 0
60
+ else:
61
+ assert not hasattr(chunk, "usage")
tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py
renamed
tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py
renamed
+1
-1
@@ -6,7 +6,7 @@ import json
6
7
import pytest
8
9
-from tests.entrypoints.openai.conftest import add_attention_backend
9
+from tests.entrypoints.speech_to_text.conftest import add_attention_backend
10
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
11
12
MISTRAL_FORMAT_ARGS = [
tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py
renamed
tests/entrypoints/speech_to_text/translation/__init__.py
tests/entrypoints/speech_to_text/translation/test_translation_validation.py
renamed
+1
-1
@@ -13,7 +13,7 @@ import pytest
13
import pytest_asyncio
14
import soundfile as sf
15
16
-from tests.entrypoints.openai.conftest import add_attention_backend
16
+from tests.entrypoints.speech_to_text.conftest import add_attention_backend
17
from tests.utils import RemoteOpenAIServer
18
from vllm.logger import init_logger
19
from vllm.multimodal.media.audio import load_audio