A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 08:45 UTC
Limit gpu utils and lower max BS on test_transcription_api_correctness.py (#41649)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Ekagra Ranjan committed
May 4, 2026 at 13:30 UTC
321fa2d6d1644629ac39d173f6393f37e14bf7b4
1 file changed
+3
-1
tests/entrypoints/openai/correctness/test_transcription_api_correctness.py
+3
-1
@@ -27,7 +27,8 @@ from ....models.registry import HF_EXAMPLE_MODELS
27
from ....utils import RemoteOpenAIServer
28
29
# Tuned to prevent OOM on 18GB GPUs in transcription correctness tests.
30
-MAX_SEQS_FOR_TRANSCRIPTION_TEST = 32
30
+MAX_SEQS_FOR_TRANSCRIPTION_TEST = 8
31
+GPU_UTIL_FOR_TRANSCRIPTION_TEST = 0.5
32
33
34
def to_bytes(y, sr):
@@ -188,6 +189,7 @@ def test_wer_correctness(
189
"--enforce-eager",
190
f"--tokenizer_mode={model_info.tokenizer_mode}",
191
f"--max_num_seqs={MAX_SEQS_FOR_TRANSCRIPTION_TEST}",
192
+ f"--gpu_memory_utilization={GPU_UTIL_FOR_TRANSCRIPTION_TEST}",
193
]
194
if model_info.trust_remote_code:
195
server_args.append("--trust-remote-code")