A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[tests] Improve speed and reliability of test_transcription_api_correctness (#23854)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Aug 29, 2025 at 00:25 UTC
c8b3b299c9f3142546e0a41f835e561af1aaffb7
1 file changed
+7
-4
tests/entrypoints/openai/correctness/test_transcription_api_correctness.py
+7
-4
@@ -49,8 +49,7 @@ async def transcribe_audio(client, tokenizer, y, sr):
49
return latency, num_output_tokens, transcription.text
50
51
52
-async def bound_transcribe(model_name, sem, client, audio, reference):
53
- tokenizer = AutoTokenizer.from_pretrained(model_name)
52
+async def bound_transcribe(sem, client, tokenizer, audio, reference):
53
# Use semaphore to limit concurrent requests.
54
async with sem:
55
result = await transcribe_audio(client, tokenizer, *audio)
@@ -63,15 +62,19 @@ async def bound_transcribe(model_name, sem, client, audio, reference):
62
async def process_dataset(model, client, data, concurrent_request):
63
sem = asyncio.Semaphore(concurrent_request)
64
65
+ # Load tokenizer once outside the loop
66
+ tokenizer = AutoTokenizer.from_pretrained(model)
67
+
68
# Warmup call as the first `librosa.load` server-side is quite slow.
69
audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]
68
- _ = await bound_transcribe(model, sem, client, (audio, sr), "")
70
+ _ = await bound_transcribe(sem, client, tokenizer, (audio, sr), "")
71
72
tasks: list[asyncio.Task] = []
73
for sample in data:
74
audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
75
task = asyncio.create_task(
74
- bound_transcribe(model, sem, client, (audio, sr), sample["text"]))
76
+ bound_transcribe(sem, client, tokenizer, (audio, sr),
77
+ sample["text"]))
78
tasks.append(task)
79
return await asyncio.gather(*tasks)
80