| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import asyncio |
| 5 | from types import SimpleNamespace |
| 6 | from unittest.mock import AsyncMock, Mock |
| 7 | |
| 8 | import pytest |
| 9 | |
| 10 | from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing |
| 11 | from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionResponse |
| 12 | |
| 13 | |
| 14 | async def _never_finishes(): |
| 15 | await asyncio.Event().wait() |
| 16 | yield |
| 17 | |
| 18 | |
| 19 | async def _records_start_then_never_finishes(started_request_ids, request_id): |
| 20 | started_request_ids.append(request_id) |
| 21 | await asyncio.Event().wait() |
| 22 | yield |
| 23 | |
| 24 | |
| 25 | @pytest.mark.asyncio |
| 26 | @pytest.mark.parametrize( |
| 27 | ("engine_inputs", "expected_request_ids"), |
| 28 | [ |
| 29 | ([{"prompt": "chunk"}], ["transcribe-outer-request"]), |
| 30 | ( |
| 31 | [{"prompt": "chunk-0"}, {"prompt": "chunk-1"}], |
| 32 | ["transcribe-outer-request-0", "transcribe-outer-request-1"], |
| 33 | ), |
| 34 | ], |
| 35 | ) |
| 36 | async def test_non_streaming_cancel_aborts_engine_requests( |
| 37 | engine_inputs, expected_request_ids |
| 38 | ): |
| 39 | engine_client = SimpleNamespace( |
| 40 | errored=False, |
| 41 | check_admission=Mock(), |
| 42 | generate=Mock(side_effect=lambda *_args, **_kwargs: _never_finishes()), |
| 43 | abort=AsyncMock(), |
| 44 | is_tracing_enabled=AsyncMock(return_value=False), |
| 45 | ) |
| 46 | |
| 47 | server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing) |
| 48 | server.engine_client = engine_client |
| 49 | server.task_type = "transcribe" |
| 50 | server.models = SimpleNamespace(model_name=lambda: "audio") |
| 51 | server.model_config = SimpleNamespace(max_model_len=1024) |
| 52 | server.model_cls = SimpleNamespace(no_space_languages=set()) |
| 53 | server.default_sampling_params = {} |
| 54 | server.asr_config = SimpleNamespace(max_audio_clip_s=30) |
| 55 | server._check_model = AsyncMock(return_value=None) |
| 56 | server._maybe_get_adapters = Mock(return_value=None) |
| 57 | server._preprocess_speech_to_text = AsyncMock( |
| 58 | return_value=( |
| 59 | engine_inputs, |
| 60 | 40.0, |
| 61 | [30.0 * i for i in range(len(engine_inputs))], |
| 62 | ) |
| 63 | ) |
| 64 | server._log_inputs = Mock() |
| 65 | |
| 66 | request = SimpleNamespace( |
| 67 | model="audio", |
| 68 | response_format="json", |
| 69 | stream=False, |
| 70 | use_beam_search=False, |
| 71 | max_completion_tokens=None, |
| 72 | language="en", |
| 73 | prompt="", |
| 74 | to_sampling_params=Mock(return_value=object()), |
| 75 | ) |
| 76 | raw_request = SimpleNamespace( |
| 77 | headers={"X-Request-Id": "outer-request"}, |
| 78 | state=SimpleNamespace(), |
| 79 | ) |
| 80 | |
| 81 | task = asyncio.create_task( |
| 82 | server._create_speech_to_text( |
| 83 | audio_data=b"audio", |
| 84 | request=request, |
| 85 | raw_request=raw_request, |
| 86 | response_class=TranscriptionResponse, |
| 87 | stream_generator_method=Mock(), |
| 88 | ) |
| 89 | ) |
| 90 | await asyncio.sleep(0) |
| 91 | |
| 92 | task.cancel() |
| 93 | with pytest.raises(asyncio.CancelledError): |
| 94 | await task |
| 95 | |
| 96 | generated_request_ids = [ |
| 97 | call.args[2] for call in engine_client.generate.call_args_list |
| 98 | ] |
| 99 | assert generated_request_ids == expected_request_ids |
| 100 | engine_client.abort.assert_awaited_once_with(expected_request_ids) |
| 101 | |
| 102 | |
| 103 | @pytest.mark.asyncio |
| 104 | async def test_non_streaming_cancel_advances_all_chunk_generators(): |
| 105 | started_request_ids: list[str] = [] |
| 106 | engine_client = SimpleNamespace( |
| 107 | errored=False, |
| 108 | check_admission=Mock(), |
| 109 | generate=Mock( |
| 110 | side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes( |
| 111 | started_request_ids, _args[2] |
| 112 | ) |
| 113 | ), |
| 114 | abort=AsyncMock(), |
| 115 | is_tracing_enabled=AsyncMock(return_value=False), |
| 116 | ) |
| 117 | |
| 118 | engine_inputs = [ |
| 119 | {"prompt": "chunk-0"}, |
| 120 | {"prompt": "chunk-1"}, |
| 121 | {"prompt": "chunk-2"}, |
| 122 | ] |
| 123 | server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing) |
| 124 | server.engine_client = engine_client |
| 125 | server.task_type = "transcribe" |
| 126 | server.models = SimpleNamespace(model_name=lambda: "audio") |
| 127 | server.model_config = SimpleNamespace(max_model_len=1024) |
| 128 | server.model_cls = SimpleNamespace(no_space_languages=set()) |
| 129 | server.default_sampling_params = {} |
| 130 | server.asr_config = SimpleNamespace(max_audio_clip_s=30) |
| 131 | server._check_model = AsyncMock(return_value=None) |
| 132 | server._maybe_get_adapters = Mock(return_value=None) |
| 133 | server._preprocess_speech_to_text = AsyncMock( |
| 134 | return_value=(engine_inputs, 90.0, [0.0, 29.5, 29.5 + 29.7]) |
| 135 | ) |
| 136 | server._log_inputs = Mock() |
| 137 | |
| 138 | request = SimpleNamespace( |
| 139 | model="audio", |
| 140 | response_format="json", |
| 141 | stream=False, |
| 142 | use_beam_search=False, |
| 143 | max_completion_tokens=None, |
| 144 | language="en", |
| 145 | prompt="", |
| 146 | to_sampling_params=Mock(return_value=object()), |
| 147 | ) |
| 148 | raw_request = SimpleNamespace( |
| 149 | headers={"X-Request-Id": "outer-request"}, |
| 150 | state=SimpleNamespace(), |
| 151 | ) |
| 152 | |
| 153 | task = asyncio.create_task( |
| 154 | server._create_speech_to_text( |
| 155 | audio_data=b"audio", |
| 156 | request=request, |
| 157 | raw_request=raw_request, |
| 158 | response_class=TranscriptionResponse, |
| 159 | stream_generator_method=Mock(), |
| 160 | ) |
| 161 | ) |
| 162 | await asyncio.sleep(0.01) |
| 163 | |
| 164 | expected_request_ids = [ |
| 165 | "transcribe-outer-request-0", |
| 166 | "transcribe-outer-request-1", |
| 167 | "transcribe-outer-request-2", |
| 168 | ] |
| 169 | assert set(started_request_ids) == set(expected_request_ids) |
| 170 | |
| 171 | task.cancel() |
| 172 | with pytest.raises(asyncio.CancelledError): |
| 173 | await task |
| 174 | |
| 175 | |
| 176 | @pytest.mark.asyncio |
| 177 | async def test_language_detection_cancel_aborts_engine_request(): |
| 178 | engine_client = SimpleNamespace( |
| 179 | generate=Mock(return_value=_never_finishes()), |
| 180 | abort=AsyncMock(), |
| 181 | ) |
| 182 | |
| 183 | server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing) |
| 184 | server.engine_client = engine_client |
| 185 | server.asr_config = SimpleNamespace() |
| 186 | server.tokenizer = Mock() |
| 187 | server.model_cls = SimpleNamespace( |
| 188 | get_language_detection_prompt=Mock(return_value={"prompt": "detect"}), |
| 189 | get_language_token_ids=Mock(return_value=[1]), |
| 190 | parse_language_detection_output=Mock(), |
| 191 | ) |
| 192 | |
| 193 | request_id = "transcribe-outer-request-lang_detect" |
| 194 | task = asyncio.create_task(server._detect_language(Mock(), request_id)) |
| 195 | await asyncio.sleep(0) |
| 196 | |
| 197 | task.cancel() |
| 198 | with pytest.raises(asyncio.CancelledError): |
| 199 | await task |
| 200 | |
| 201 | engine_client.abort.assert_awaited_once_with(request_id) |