A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 00:45 UTC
[Cohere] Enable Cohere-Transcribe (#38120)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Ekagra Ranjan committed
Mar 25, 2026 at 19:13 UTC
7b54f60db0f55d74dac8aa3040c02363b7a9f6ec
6 files changed
+23
-11
docs/models/supported_models.md
+1
@@ -654,6 +654,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
654
655
| Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
656
| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
657
+| `CohereAsrForConditionalGeneration` | Cohere-Transcribe | `CohereLabs/cohere-transcribe-03-2026` | | |
658
| `FireRedASR2ForConditionalGeneration` | FireRedASR2 | `allendou/FireRedASR2-LLM-vllm`, etc. | | |
659
| `FunASRForConditionalGeneration` | FunASR | `allendou/Fun-ASR-Nano-2512-vllm`, etc. | | |
660
| `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
examples/offline_inference/audio_language.py
+1
-2
@@ -72,8 +72,7 @@ def run_audioflamingo3(question: str, audio_count: int) -> ModelRequestData:
72
# CohereASR
73
def run_cohere_asr(question: str, audio_count: int) -> ModelRequestData:
74
assert audio_count == 1, "CohereASR only support single audio input per prompt"
75
- # TODO (ekagra): add HF ckpt after asr release
76
- model_name = "/host/engines/vllm/audio/2b-release"
75
+ model_name = "CohereLabs/cohere-transcribe-03-2026"
76
77
prompt = (
78
"<|startofcontext|><|startoftranscript|>"
tests/entrypoints/openai/correctness/test_transcription_api_correctness.py
+16
-4
@@ -19,6 +19,7 @@ import soundfile
19
import torch
20
from datasets import load_dataset
21
from evaluate import load
22
+from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
23
24
from vllm.tokenizers import get_tokenizer
25
@@ -33,6 +34,16 @@ def to_bytes(y, sr):
34
return buffer
35
36
37
+# not all models have a normalizer so use the one from whisper as a standard option
38
+normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")
39
+normalizer_tokenizer = get_tokenizer(
40
+ "openai/whisper-large-v3",
41
+ tokenizer_mode=normalizer_model_info.tokenizer_mode,
42
+ trust_remote_code=normalizer_model_info.trust_remote_code,
43
+)
44
+normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)
45
+
46
+
47
async def transcribe_audio(client, tokenizer, y, sr):
48
# Send loaded audio directly instead of loading from disk,
49
# don't account for that time though
@@ -58,8 +69,8 @@ async def bound_transcribe(sem, client, tokenizer, audio, reference):
69
async with sem:
70
result = await transcribe_audio(client, tokenizer, *audio)
71
# Normalize *english* output/reference for evaluation.
61
- out = tokenizer.normalize(result[2])
62
- ref = tokenizer.normalize(reference)
72
+ out = normalizer(result[2])
73
+ ref = normalizer(reference)
74
return result[:2] + (out, ref)
75
76
@@ -156,8 +167,9 @@ def run_evaluation(
167
"model_config",
168
[
169
("openai/whisper-large-v3", 12.744980),
159
- # TODO (ekagra): add HF ckpt after asr release
160
- # ("/host/engines/vllm/audio/2b-release", 11.73),
170
+ # TODO (ekagra): turn on after asr release
171
+ # CohereASR is used to test the variable encoder length code paths
172
+ # ("CohereLabs/cohere-transcribe-03-2026", 11.92),
173
],
174
)
175
# Original dataset is 20GB+ in size, hence we use a pre-filtered slice.
tests/models/registry.py
+2
-2
@@ -1128,8 +1128,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1128
tokenizer_mode="mistral",
1129
),
1130
# [Encoder-decoder]
1131
- "CohereASRForConditionalGeneration": _HfExamplesInfo(
1132
- "/host/engines/vllm/audio/2b-release",
1131
+ "CohereAsrForConditionalGeneration": _HfExamplesInfo(
1132
+ "CohereLabs/cohere-transcribe-03-2026",
1133
trust_remote_code=True,
1134
is_available_online=False, # TODO (ekagra): revert after asr release
1135
),
vllm/model_executor/models/cohere_asr.py
+1
-1
@@ -1988,7 +1988,7 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
1988
info=CohereASRProcessingInfo,
1989
dummy_inputs=CohereASRDummyInputsBuilder,
1990
)
1991
-class CohereASRForConditionalGeneration(
1991
+class CohereAsrForConditionalGeneration(
1992
nn.Module, SupportsTranscription, SupportsMultiModal
1993
):
1994
packed_modules_mapping = {
vllm/model_executor/models/registry.py
+2
-2
@@ -525,9 +525,9 @@ _MULTIMODAL_MODELS = {
525
"VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
526
"VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
527
# [Encoder-decoder]
528
- "CohereASRForConditionalGeneration": (
528
+ "CohereAsrForConditionalGeneration": (
529
"cohere_asr",
530
- "CohereASRForConditionalGeneration",
530
+ "CohereAsrForConditionalGeneration",
531
),
532
"NemotronParseForConditionalGeneration": (
533
"nemotron_parse",