| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from typing import Any |
| 5 | |
| 6 | import pytest |
| 7 | from transformers import AutoModelForSeq2SeqLM |
| 8 | |
| 9 | from vllm.assets.audio import AudioAsset |
| 10 | from vllm.envs import disable_envs_cache |
| 11 | from vllm.lora.request import LoRARequest |
| 12 | from vllm.multimodal.audio import AudioResampler |
| 13 | |
| 14 | from ....conftest import HfRunner, VllmRunner |
| 15 | from ...utils import check_logprobs_close |
| 16 | |
| 17 | AUDIO_ASSET = AudioAsset("mary_had_lamb") |
| 18 | |
| 19 | AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = { |
| 20 | "ibm-granite/granite-speech-3.3-2b": { |
| 21 | "prompt": ( |
| 22 | "<|start_of_role|>system<|end_of_role|>" |
| 23 | "You are a helpful AI assistant<|end_of_text|>\n" |
| 24 | "<|start_of_role|>user<|end_of_role|>" |
| 25 | "<|audio|>can you transcribe the speech into a written format?" |
| 26 | "<|end_of_text|>\n" |
| 27 | "<|start_of_role|>assistant<|end_of_role|>" |
| 28 | ), |
| 29 | "audio_lora_path": "ibm-granite/granite-speech-3.3-2b", |
| 30 | }, |
| 31 | "nvidia/audio-flamingo-3-hf": { |
| 32 | "prompt": ( |
| 33 | "<|im_start|>system\n" |
| 34 | "You are a helpful assistant.<|im_end|>\n" |
| 35 | "<|im_start|>user\n" |
| 36 | "<sound>Transcribe the input speech.<|im_end|>\n" |
| 37 | "<|im_start|>assistant\n" |
| 38 | ), |
| 39 | "vllm_runner_kwargs": { |
| 40 | "gpu_memory_utilization": 0.85, |
| 41 | }, |
| 42 | }, |
| 43 | "microsoft/VibeVoice-ASR-HF": { |
| 44 | "prompt": ( |
| 45 | "<|im_start|>system\n" |
| 46 | "You are a helpful assistant that transcribes audio input " |
| 47 | "into text output in JSON format.<|im_end|>\n" |
| 48 | "<|im_start|>user\n" |
| 49 | "<|object_ref_start|><|box_start|><|object_ref_end|>\n" |
| 50 | "This is a 1.0 seconds audio, please transcribe it with " |
| 51 | "these keys: Start time, End time, Speaker ID, Content" |
| 52 | "<|im_end|>\n" |
| 53 | "<|im_start|>assistant\n" |
| 54 | ), |
| 55 | "sampling_rate": 24000, |
| 56 | "vllm_runner_kwargs": { |
| 57 | "max_num_batched_tokens": 2048, |
| 58 | "gpu_memory_utilization": 0.85, |
| 59 | }, |
| 60 | }, |
| 61 | "zai-org/GLM-ASR-Nano-2512": { |
| 62 | "prompt": ( |
| 63 | "<|user|>\n" |
| 64 | "<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n" |
| 65 | "Please transcribe this audio into text" |
| 66 | "<|assistant|>\n" |
| 67 | ), |
| 68 | }, |
| 69 | } |
| 70 | |
| 71 | |
| 72 | @pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS)) |
| 73 | def test_transformers_audio_generation( |
| 74 | hf_runner: type[HfRunner], |
| 75 | vllm_runner: type[VllmRunner], |
| 76 | monkeypatch, |
| 77 | model_id: str, |
| 78 | ): |
| 79 | """Single-process workaround for V1 fork safety deadlock issue |
| 80 | (vllm-project/vllm/issues/17676). Running multiple audio models together |
| 81 | under pytest can cause (possibly flaky) hangs, so they are grouped under |
| 82 | the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the |
| 83 | deadlock and allows worker processes to terminate cleanly, and release |
| 84 | GPU memory between test runs until the issue is fixed.""" |
| 85 | # TODO: Remove monkeypatch once |
| 86 | # https://github.com/vllm-project/vllm/issues/17676 is fixed. |
| 87 | disable_envs_cache() |
| 88 | monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") |
| 89 | |
| 90 | settings = AUDIO_MODEL_SETTINGS[model_id] |
| 91 | audio_lora_path = settings.get("audio_lora_path") |
| 92 | |
| 93 | audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate |
| 94 | target_sr = settings.get("sampling_rate", orig_sr) |
| 95 | if orig_sr != target_sr: |
| 96 | audio = AudioResampler(target_sr=target_sr).resample(audio, orig_sr=orig_sr) |
| 97 | audio = (audio, target_sr) |
| 98 | |
| 99 | with vllm_runner( |
| 100 | model_id, |
| 101 | model_impl="transformers", |
| 102 | dtype="bfloat16", |
| 103 | max_model_len=2048, |
| 104 | enforce_eager=True, |
| 105 | limit_mm_per_prompt={"audio": 1}, |
| 106 | enable_lora=audio_lora_path is not None, |
| 107 | max_lora_rank=64, |
| 108 | **settings.get("vllm_runner_kwargs", {}), |
| 109 | ) as vllm_model: |
| 110 | lora_request = ( |
| 111 | LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None |
| 112 | ) |
| 113 | vllm_outputs = vllm_model.generate_greedy_logprobs( |
| 114 | [settings["prompt"]], |
| 115 | 128, |
| 116 | num_logprobs=10, |
| 117 | audios=[audio], |
| 118 | lora_request=lora_request, |
| 119 | ) |
| 120 | |
| 121 | with hf_runner( |
| 122 | model_id, dtype="bfloat16", auto_cls=AutoModelForSeq2SeqLM |
| 123 | ) as hf_model: |
| 124 | hf_outputs = hf_model.generate_greedy_logprobs_limit( |
| 125 | [settings["prompt"]], 128, num_logprobs=10, audios=[audio] |
| 126 | ) |
| 127 | |
| 128 | check_logprobs_close( |
| 129 | outputs_0_lst=hf_outputs, |
| 130 | outputs_1_lst=vllm_outputs, |
| 131 | name_0="hf", |
| 132 | name_1="vllm", |
| 133 | ) |