[Bench][BugFix] Fix empty decoder prompt for Cohere ASR in throughput benchmark (#47135)
Signed-off-by: Michal Ganczarenko <michal.ganczarenko@intel.com>
Michał Ganczarenko committed
Jul 1, 2026 at 18:42 UTC
c8bdcc011623de64dbab64813a184ab083730a81
2 files changed
+85
-1
tests/benchmarks/test_audio_dataset.py
+71
@@ -34,6 +34,16 @@ class _Tokenizer:
34
return _TokenizedPrompt(prompt)
35
36
37
+class CohereAsrTokenizer(_Tokenizer):
38
+ def __init__(self, name_or_path: str = "/models/cohere-transcribe") -> None:
39
+ super().__init__(name_or_path)
40
+
41
+
42
+class _CohereNameOnlyTokenizer(_Tokenizer):
43
+ def __init__(self) -> None:
44
+ super().__init__("cohere/some-local-checkpoint")
45
+
46
+
47
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
48
num_samples = int(duration_s * sample_rate)
49
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
@@ -198,3 +208,64 @@ def test_async_request_openai_audio_handles_decoded_audio_arrays(
208
assert session.uploaded_bytes is not None
209
assert output.success is True
210
assert output.generated_text == "hello"
211
+
212
+
213
+_COHERE_ASR_PROMPT = (
214
+ "<|startofcontext|><|startoftranscript|>"
215
+ "<|emo:undefined|><|en|><|en|><|pnc|><|noitn|>"
216
+ "<|notimestamp|><|nodiarize|>"
217
+)
218
+
219
+
220
+def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset:
221
+ audio_path = tmp_path / "sample.wav"
222
+ _write_wav(audio_path, duration_s=0.1)
223
+ dataset = object.__new__(datasets_module.ASRDataset)
224
+ dataset.data = [
225
+ {
226
+ "audio": {"path": str(audio_path), "bytes": None},
227
+ "text": "hello world",
228
+ }
229
+ ]
230
+ return dataset
231
+
232
+
233
+def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None:
234
+ dataset = _make_asr_dataset(tmp_path)
235
+ samples = dataset.sample(
236
+ tokenizer=CohereAsrTokenizer(),
237
+ num_requests=1,
238
+ output_len=32,
239
+ asr_min_audio_len_sec=0.0,
240
+ asr_max_audio_len_sec=1.0,
241
+ )
242
+ assert len(samples) == 1
243
+ assert samples[0].prompt == _COHERE_ASR_PROMPT
244
+
245
+
246
+def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
247
+ tmp_path: Path,
248
+) -> None:
249
+ dataset = _make_asr_dataset(tmp_path)
250
+ samples = dataset.sample(
251
+ tokenizer=_CohereNameOnlyTokenizer(),
252
+ num_requests=1,
253
+ output_len=32,
254
+ asr_min_audio_len_sec=0.0,
255
+ asr_max_audio_len_sec=1.0,
256
+ )
257
+ assert len(samples) == 1
258
+ assert samples[0].prompt == _COHERE_ASR_PROMPT
259
+
260
+
261
+def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None:
262
+ dataset = _make_asr_dataset(tmp_path)
263
+ samples = dataset.sample(
264
+ tokenizer=_Tokenizer(name_or_path="some-other/asr-model"),
265
+ num_requests=1,
266
+ output_len=32,
267
+ asr_min_audio_len_sec=0.0,
268
+ asr_max_audio_len_sec=1.0,
269
+ )
270
+ assert len(samples) == 1
271
+ assert samples[0].prompt == ""
vllm/benchmarks/datasets/datasets.py
+14
-1
@@ -4156,8 +4156,21 @@ class ASRDataset(HuggingFaceDataset):
4156
**kwargs,
4157
) -> list[SampleRequest]:
4158
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
4159
- if "openai" in getattr(tokenizer, "name_or_path", ""):
4159
+ name_or_path = getattr(tokenizer, "name_or_path", "")
4160
+ tok_class = type(tokenizer).__name__
4161
+ if "openai" in name_or_path:
4162
prompt = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
4163
+ elif tok_class == "CohereAsrTokenizer" or "cohere" in name_or_path.lower():
4164
+ # CohereAsrTokenizer does not inject a decoder start token, so the
4165
+ # decoder prompt must supply the full control-token sequence.
4166
+ # Token order: context boundary, transcript start, emotion (default
4167
+ # undefined), language (en), transcription directive (en), punctuation
4168
+ # enabled, no ITN, no timestamp, no diarization.
4169
+ prompt = (
4170
+ "<|startofcontext|><|startoftranscript|>"
4171
+ "<|emo:undefined|><|en|><|en|><|pnc|><|noitn|>"
4172
+ "<|notimestamp|><|nodiarize|>"
4173
+ )
4174
else:
4175
prompt = ""
4176
prompt_len = len(tokenizer(prompt).input_ids)