A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[TPU][Bugfix] fix OOM issue in CI test (#21550)
Signed-off-by: Chengji Yao <chengjiyao@google.com>
Chengji Yao committed
Jul 24, 2025 at 23:01 UTC
40d86ee412eeeca93e0c37432db6b96829cb64e2
1 file changed
+1
-1
tests/v1/tpu/test_basic.py
+1
-1
@@ -59,7 +59,7 @@ def test_basic(
59
# actually test chunked prompt
60
max_num_batched_tokens=1024,
61
max_model_len=8192,
62
- gpu_memory_utilization=0.7,
62
+ gpu_memory_utilization=0.95,
63
max_num_seqs=max_num_seqs,
64
tensor_parallel_size=tensor_parallel_size) as vllm_model:
65
vllm_outputs = vllm_model.generate_greedy(example_prompts,