A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 06:15 UTC
[ROCm][CI] Force max_num_seqs=1 on ROCm In test_sharded_state_loader to reduce flakiness (#33277)
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Micah Williamson committed
Jan 30, 2026 at 22:28 UTC
6c64c41b4abfe10f725088afb1fe143d412dc118
1 file changed
+8
tests/model_executor/model_loader/test_sharded_state_loader.py
+8
@@ -13,6 +13,7 @@ from huggingface_hub import snapshot_download
13
14
from vllm import LLM, SamplingParams
15
from vllm.model_executor.model_loader import ShardedStateLoader
16
+from vllm.platforms import current_platform
17
18
prompts = [
19
"Hello, my name is",
@@ -95,6 +96,10 @@ def test_sharded_state_loader(
96
input_dir = llama_3p2_1b_files
97
ctx = mp.get_context("spawn")
98
99
+ platform_args = {}
100
+ if current_platform.is_rocm():
101
+ platform_args["max_num_seqs"] = 1
102
+
103
# Run in separate processes for memory & CUDA isolation
104
with TemporaryDirectory() as output_dir:
105
p = ctx.Process(
@@ -104,6 +109,7 @@ def test_sharded_state_loader(
109
tensor_parallel_size=tp_size,
110
gpu_memory_utilization=gpu_memory_utilization,
111
enforce_eager=True,
112
+ **platform_args,
113
),
114
)
115
p.start()
@@ -118,6 +124,7 @@ def test_sharded_state_loader(
124
enable_lora=enable_lora,
125
gpu_memory_utilization=gpu_memory_utilization,
126
tensor_parallel_size=tp_size,
127
+ **platform_args,
128
),
129
)
130
p.start()
@@ -141,6 +148,7 @@ def test_sharded_state_loader(
148
gpu_memory_utilization=gpu_memory_utilization,
149
tensor_parallel_size=tp_size,
150
load_format="sharded_state",
151
+ **platform_args,
152
),
153
)
154
p.start()