@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 57e9bf1864
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 13:45 UTC

[CI] Whisper logprobs tests (#30504)

Signed-off-by: NickLucche <nlucches@redhat.com>

Nicolò Lucchesi committed Dec 13, 2025 at 03:49 UTC 57e9bf18642a391e918400a5afc7c01221635698
3 files changed +134 -113
tests/conftest.py
+7 -1
@@ -702,10 +702,16 @@ class HfRunner:
702 **kwargs,
703 )
704
705 + # Encoder-decoder models return decoder_hidden_states instead of
706 + # hidden_states
707 + hidden_states = (
708 + getattr(output, "hidden_states", None) or output.decoder_hidden_states
709 + )
710 +
711 (
712 seq_logprobs_lst,
713 output_len,
708 - ) = self._hidden_states_to_logprobs(output.hidden_states, num_logprobs)
714 + ) = self._hidden_states_to_logprobs(hidden_states, num_logprobs)
715
716 all_logprobs.append(seq_logprobs_lst)
717 seq_ids = output.sequences[0]
tests/models/multimodal/generation/test_whisper.py
+123 -111
@@ -1,150 +1,146 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from collections.abc import Sequence
5 +from typing import Any
6 +
7 +import librosa
8 import pytest
9 +from transformers import AutoModelForSpeechSeq2Seq
10
6 -from vllm import SamplingParams
11 from vllm.assets.audio import AudioAsset
12 +from vllm.platforms import current_platform
13
9 -from ....conftest import VllmRunner
14 +from ....conftest import HfRunner, PromptAudioInput, VllmRunner
15 from ....utils import create_new_process_for_each_test, multi_gpu_test
16 +from ...registry import HF_EXAMPLE_MODELS
17 +from ...utils import check_logprobs_close
18 +
19 +VLLM_PROMPT = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
20 +HF_PROMPT = ""
21 +# Whisper expects 16kHz audio
22 +WHISPER_SAMPLE_RATE = 16000
23
12 -PROMPTS = [
13 - {
14 - "prompt": "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
15 - "multi_modal_data": {
16 - "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
17 - },
18 - },
19 - { # Test explicit encoder/decoder prompt
20 - "encoder_prompt": {
21 - "prompt": "",
22 - "multi_modal_data": {
23 - "audio": AudioAsset("winning_call").audio_and_sample_rate,
24 - },
25 - },
26 - "decoder_prompt": "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
27 - },
28 -]
29 -
30 -EXPECTED = {
31 - "openai/whisper-tiny": [
32 - " He has birth words I spoke in the original corner of that. And a"
33 - " little piece of black coat poetry. Mary had a little sandwich,"
34 - " sweet, with white and snow. And everyone had it very went the last"
35 - " would sure to go.",
36 - " >> And the old one, fit John the way to Edgar Martinez. >> One more"
37 - " to line down the field line for our base camp. Here comes joy. Here"
38 - " is June and the third base. They're going to wave him in. The throw"
39 - " to the plate will be late. The Mariners are going to play for the"
40 - " American League Championship. I don't believe it. It just continues"
41 - " by all five.",
42 - ],
43 - "openai/whisper-small": [
44 - " The first words I spoke in the original pornograph. A little piece"
45 - " of practical poetry. Mary had a little lamb, its fleece was quite a"
46 - " slow, and everywhere that Mary went the lamb was sure to go.",
47 - " And the old one pitch on the way to Edgar Martinez one month. Here"
48 - " comes joy. Here is Junior to third base. They're gonna wave him"
49 - " in. The throw to the plate will be late. The Mariners are going to"
50 - " play for the American League Championship. I don't believe it. It"
51 - " just continues. My, oh my.",
52 - ],
53 - "openai/whisper-medium": [
54 - " The first words I spoke in the original phonograph, a little piece"
55 - " of practical poetry. Mary had a little lamb, its fleece was quite as"
56 - " slow, and everywhere that Mary went the lamb was sure to go.",
57 - " And the 0-1 pitch on the way to Edgar Martinez swung on the line"
58 - " down the left field line for Obeyshev. Here comes Joy. Here is"
59 - " Jorgen at third base. They're going to wave him in. The throw to the"
60 - " plate will be late. The Mariners are going to play for the American"
61 - " League Championship. I don't believe it. It just continues. My, oh"
62 - " my.",
63 - ],
64 - "openai/whisper-large-v3": [
65 - " The first words I spoke in the original phonograph, a little piece"
66 - " of practical poetry. Mary had a little lamb, its feet were quite as"
67 - " slow, and everywhere that Mary went, the lamb was sure to go.",
68 - " And the 0-1 pitch on the way to Edgar Martinez. Swung on the line."
69 - " Now the left field line for a base hit. Here comes Joy. Here is"
70 - " Junior to third base. They're going to wave him in. The throw to the"
71 - " plate will be late. The Mariners are going to play for the American"
72 - " League Championship. I don't believe it. It just continues. My, oh,"
73 - " my.",
74 - ],
75 - "openai/whisper-large-v3-turbo": [
76 - " The first words I spoke in the original phonograph, a little piece"
77 - " of practical poetry. Mary had a little lamb, its streets were quite"
78 - " as slow, and everywhere that Mary went the lamb was sure to go.",
79 - " And the 0-1 pitch on the way to Edgar Martinez. Swung on the line"
80 - " down the left field line for a base hit. Here comes Joy. Here is"
81 - " Junior to third base. They're going to wave him in. The throw to the"
82 - " plate will be late. The Mariners are going to play for the American"
83 - " League Championship. I don't believe it. It just continues. My, oh,"
84 - " my.",
85 - ],
86 -}
24 +
25 +@pytest.fixture(autouse=True)
26 +def use_spawn_for_whisper(monkeypatch):
27 + """Whisper has issues with forked workers, use spawn instead."""
28 + monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
29
30
31 def run_test(
32 + hf_runner: type[HfRunner],
33 vllm_runner: type[VllmRunner],
34 + inputs: Sequence[tuple[list[str], list[str], PromptAudioInput]],
35 model: str,
36 *,
37 + max_model_len: int,
38 + dtype: str,
39 + max_tokens: int,
40 + num_logprobs: int,
41 tensor_parallel_size: int,
42 distributed_executor_backend: str | None = None,
95 - dtype: str = "half",
43 + enforce_eager: bool = True,
44 ) -> None:
97 - prompt_list = PROMPTS * 10
98 - expected_list = EXPECTED[model] * 10
45 + """Inference result should be the same between hf and vllm.
46
47 + All the audio fixtures for the test are from AudioAsset.
48 + For huggingface runner, we provide the audio as input.
49 + For vllm runner, we provide MultiModalDataDict objects
50 + and corresponding MultiModalConfig as input.
51 + """
52 with vllm_runner(
53 model,
54 dtype=dtype,
103 - max_model_len=448,
55 + max_model_len=max_model_len,
56 tensor_parallel_size=tensor_parallel_size,
57 distributed_executor_backend=distributed_executor_backend,
106 - # TODO (NickLucche) figure out output differences with non-eager and re-enable
107 - enforce_eager=True,
58 + limit_mm_per_prompt={"audio": 2},
59 + enforce_eager=enforce_eager,
60 + disable_custom_all_reduce=True,
61 ) as vllm_model:
109 - llm = vllm_model.llm
110 -
111 - sampling_params = SamplingParams(
112 - temperature=0,
113 - top_p=1.0,
114 - max_tokens=200,
62 + vllm_outputs_per_case = [
63 + vllm_model.generate_greedy_logprobs(
64 + vllm_prompts,
65 + max_tokens,
66 + num_logprobs=num_logprobs,
67 + audios=audios,
68 + )
69 + for vllm_prompts, _, audios in inputs
70 + ]
71 +
72 + with hf_runner(model, dtype=dtype, auto_cls=AutoModelForSpeechSeq2Seq) as hf_model:
73 + hf_outputs_per_case = [
74 + hf_model.generate_greedy_logprobs_limit(
75 + hf_prompts,
76 + max_tokens,
77 + num_logprobs=num_logprobs,
78 + audios=audios,
79 + )
80 + for _, hf_prompts, audios in inputs
81 + ]
82 +
83 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
84 + check_logprobs_close(
85 + outputs_0_lst=hf_outputs,
86 + outputs_1_lst=vllm_outputs,
87 + name_0="hf",
88 + name_1="vllm",
89 )
90
117 - outputs = llm.generate(prompt_list, sampling_params)
91
119 - for output, expected in zip(outputs, expected_list):
120 - print(output.outputs[0].text)
121 - assert output.outputs[0].text == expected
92 +@pytest.fixture
93 +def input_audios() -> list[tuple[list[str], list[str], list[tuple[Any, int]]]]:
94 + audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
95 + inputs = []
96 + for asset in audio_assets:
97 + audio, orig_sr = asset.audio_and_sample_rate
98 + # Resample to Whisper's expected sample rate (16kHz)
99 + if orig_sr != WHISPER_SAMPLE_RATE:
100 + audio = librosa.resample(
101 + audio, orig_sr=orig_sr, target_sr=WHISPER_SAMPLE_RATE
102 + )
103 + # vLLM prompts, HF prompts, audio inputs
104 + inputs.append(([VLLM_PROMPT], [HF_PROMPT], [(audio, WHISPER_SAMPLE_RATE)]))
105 + return inputs
106
107
124 -@pytest.mark.core_model
125 -@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
126 -@pytest.mark.parametrize("dtype", ["half"])
127 -@create_new_process_for_each_test()
128 -def test_models(vllm_runner, model, dtype) -> None:
129 - run_test(
130 - vllm_runner,
131 - model,
132 - tensor_parallel_size=1,
133 - dtype=dtype,
134 - )
108 +def check_model_available(model: str) -> None:
109 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
110 + model_info.check_available_online(on_fail="skip")
111 + model_info.check_transformers_version(on_fail="skip")
112
113
114 +@pytest.mark.core_model
115 @pytest.mark.cpu_model
116 @pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
117 @pytest.mark.parametrize("dtype", ["half"])
140 -def test_models_cpu(vllm_runner, model, dtype) -> None:
141 - # @create_new_process_for_each_test() does not work for some runners
142 - # TODO: to fix cpu privilege issues in run-cpu-test-arm.sh
118 +@pytest.mark.parametrize("num_logprobs", [5])
119 +@pytest.mark.parametrize("enforce_eager", [True, False])
120 +@create_new_process_for_each_test("spawn")
121 +def test_models(
122 + hf_runner,
123 + vllm_runner,
124 + model: str,
125 + dtype: str,
126 + num_logprobs: int,
127 + input_audios,
128 + enforce_eager: bool,
129 +) -> None:
130 + check_model_available(model)
131 + if current_platform.is_cpu() and not enforce_eager:
132 + pytest.skip("Skipping test for CPU with non-eager mode")
133 run_test(
134 + hf_runner,
135 vllm_runner,
136 + input_audios,
137 model,
146 - tensor_parallel_size=1,
138 dtype=dtype,
139 + max_model_len=448,
140 + max_tokens=200,
141 + num_logprobs=num_logprobs,
142 + tensor_parallel_size=1,
143 + enforce_eager=enforce_eager,
144 )
145
146
@@ -152,15 +148,31 @@ def test_models_cpu(vllm_runner, model, dtype) -> None:
148 @pytest.mark.core_model
149 @pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
150 @pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
155 -@create_new_process_for_each_test()
151 +@pytest.mark.parametrize("dtype", ["half"])
152 +@pytest.mark.parametrize("max_tokens", [200])
153 +@pytest.mark.parametrize("num_logprobs", [5])
154 +@create_new_process_for_each_test("spawn")
155 def test_models_distributed(
156 + hf_runner,
157 vllm_runner,
158 - model,
159 - distributed_executor_backend,
158 + model: str,
159 + distributed_executor_backend: str,
160 + dtype: str,
161 + max_tokens: int,
162 + num_logprobs: int,
163 + input_audios,
164 ) -> None:
165 + check_model_available(model)
166 run_test(
167 + hf_runner,
168 vllm_runner,
169 + input_audios,
170 model,
171 + dtype=dtype,
172 + max_model_len=448,
173 + max_tokens=max_tokens,
174 + num_logprobs=num_logprobs,
175 tensor_parallel_size=2,
176 distributed_executor_backend=distributed_executor_backend,
177 + enforce_eager=False,
178 )
tests/models/registry.py
+4 -1
@@ -840,7 +840,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
840 is_available_online=False,
841 ),
842 # [Encoder-decoder]
843 - "WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"),
843 + "WhisperForConditionalGeneration": _HfExamplesInfo(
844 + "openai/whisper-large-v3-turbo",
845 + extras={"v3": "openai/whisper-large-v3"},
846 + ),
847 # [Cross-encoder]
848 "JinaVLForRanking": _HfExamplesInfo("jinaai/jina-reranker-m0"),
849 }