A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Tests] Harden DP tests (#21508)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Jul 25, 2025 at 10:27 UTC
e38e96a3c044dfa72d98f23b7d105ae20af78f5f
3 files changed
+82
-55
tests/v1/test_external_lb_dp.py
+5
-4
@@ -11,7 +11,7 @@ import pytest
11
import pytest_asyncio
12
13
from tests.utils import RemoteOpenAIServer
14
-from vllm.platforms import Platform
14
+from vllm.platforms import current_platform
15
16
MODEL_NAME = "ibm-research/PowerMoE-3b"
17
@@ -70,10 +70,11 @@ class ExternalLBServerManager:
70
sargs,
71
auto_port=False,
72
env_dict={
73
- "CUDA_VISIBLE_DEVICES":
73
+ current_platform.device_control_env_var:
74
",".join(
75
- str(Platform.device_id_to_physical_device_id(
76
- i))
75
+ str(
76
+ current_platform.
77
+ device_id_to_physical_device_id(i))
78
for i in range(r * TP_SIZE, (r + 1) * TP_SIZE))
79
})
80
server.__enter__()
tests/v1/test_hybrid_lb_dp.py
+29
-31
@@ -12,7 +12,7 @@ import pytest_asyncio
12
13
from tests.utils import RemoteOpenAIServer
14
from tests.v1.test_utils import check_request_balancing
15
-from vllm.platforms import Platform
15
+from vllm.platforms import current_platform
16
17
MODEL_NAME = "ibm-research/PowerMoE-3b"
18
@@ -92,10 +92,12 @@ class HybridLBServerManager:
92
sargs,
93
auto_port=False,
94
env_dict={
95
- "CUDA_VISIBLE_DEVICES":
95
+ current_platform.device_control_env_var:
96
",".join(
97
- str(Platform.device_id_to_physical_device_id(
98
- i)) for i in range(gpu_start, gpu_end))
97
+ str(
98
+ current_platform.
99
+ device_id_to_physical_device_id(i))
100
+ for i in range(gpu_start, gpu_end))
101
})
102
server.__enter__()
103
print(f"Hybrid LB node {node} started successfully with "
@@ -180,7 +182,7 @@ async def test_hybrid_lb_completion(clients: list[openai.AsyncOpenAI],
182
completion = await client.completions.create(
183
model=model_name,
184
prompt="Hello, my name is",
183
- max_tokens=10,
185
+ max_tokens=5,
186
temperature=1.0)
187
188
assert completion.id is not None
@@ -212,27 +214,28 @@ async def test_hybrid_lb_completion(clients: list[openai.AsyncOpenAI],
214
await asyncio.sleep(0.5)
215
216
# Send requests to all nodes - each should balance within its local DP ranks
215
- num_requests_per_node = 25 # Total 50 requests across 2 nodes
217
+ num_requests = 200 # Total 200 requests across 2 nodes
218
all_tasks = []
217
-
218
- for i, client in enumerate(clients):
219
- tasks = [make_request(client) for _ in range(num_requests_per_node)]
220
- all_tasks.extend(tasks)
219
+ for i in range(num_requests):
220
+ client = clients[i % len(clients)]
221
+ all_tasks.append(asyncio.create_task(make_request(client)))
222
+ await asyncio.sleep(0.01)
223
224
results = await asyncio.gather(*all_tasks)
223
- assert len(results) == num_requests_per_node * len(clients)
225
+ assert len(results) == num_requests
226
assert all(completion is not None for completion in results)
227
228
await asyncio.sleep(0.5)
229
230
# Second burst of requests
231
all_tasks = []
230
- for i, client in enumerate(clients):
231
- tasks = [make_request(client) for _ in range(num_requests_per_node)]
232
- all_tasks.extend(tasks)
232
+ for i in range(num_requests):
233
+ client = clients[i % len(clients)]
234
+ all_tasks.append(asyncio.create_task(make_request(client)))
235
+ await asyncio.sleep(0.01)
236
237
results = await asyncio.gather(*all_tasks)
235
- assert len(results) == num_requests_per_node * len(clients)
238
+ assert len(results) == num_requests
239
assert all(completion is not None for completion in results)
240
241
_, server_args = servers[0]
@@ -309,33 +312,28 @@ async def test_hybrid_lb_completion_streaming(clients: list[
312
await asyncio.sleep(0.5)
313
314
# Send streaming requests to all nodes
312
- num_requests_per_node = 25 # Total 50 requests across 2 nodes
315
+ num_requests = 200 # Total 200 requests across 2 nodes
316
all_tasks = []
314
-
315
- for i, client in enumerate(clients):
316
- tasks = [
317
- make_streaming_request(client)
318
- for _ in range(num_requests_per_node)
319
- ]
320
- all_tasks.extend(tasks)
317
+ for i in range(num_requests):
318
+ client = clients[i % len(clients)]
319
+ all_tasks.append(asyncio.create_task(make_streaming_request(client)))
320
+ await asyncio.sleep(0.01)
321
322
results = await asyncio.gather(*all_tasks)
323
- assert len(results) == num_requests_per_node * len(clients)
323
+ assert len(results) == num_requests
324
assert all(results), "Not all streaming requests completed successfully."
325
326
await asyncio.sleep(0.5)
327
328
# Second burst of streaming requests
329
all_tasks = []
330
- for i, client in enumerate(clients):
331
- tasks = [
332
- make_streaming_request(client)
333
- for _ in range(num_requests_per_node)
334
- ]
335
- all_tasks.extend(tasks)
330
+ for i in range(num_requests):
331
+ client = clients[i % len(clients)]
332
+ all_tasks.append(asyncio.create_task(make_streaming_request(client)))
333
+ await asyncio.sleep(0.01)
334
335
results = await asyncio.gather(*all_tasks)
338
- assert len(results) == num_requests_per_node * len(clients)
336
+ assert len(results) == num_requests
337
assert all(results), "Not all streaming requests completed successfully."
338
339
_, server_args = servers[0]
tests/v1/test_internal_lb_dp.py
+48
-20
@@ -11,7 +11,7 @@ import pytest_asyncio
11
12
from tests.utils import RemoteOpenAIServer
13
from tests.v1.test_utils import check_request_balancing
14
-from vllm.platforms import Platform
14
+from vllm.platforms import current_platform
15
16
MODEL_NAME = "ibm-research/PowerMoE-3b"
17
@@ -96,10 +96,12 @@ class MultinodeInternalLBServerManager:
96
sargs,
97
auto_port=False,
98
env_dict={
99
- "CUDA_VISIBLE_DEVICES":
99
+ current_platform.device_control_env_var:
100
",".join(
101
- str(Platform.device_id_to_physical_device_id(
102
- i)) for i in range(r, r + gpus_per_node))
101
+ str(
102
+ current_platform.
103
+ device_id_to_physical_device_id(i))
104
+ for i in range(r, r + gpus_per_node))
105
})
106
server.__enter__()
107
if r == 0:
@@ -219,9 +221,11 @@ class APIOnlyServerManager:
221
engines_server_args,
222
auto_port=False,
223
env_dict={
222
- "CUDA_VISIBLE_DEVICES":
224
+ current_platform.device_control_env_var:
225
",".join(
224
- str(Platform.device_id_to_physical_device_id(i))
226
+ str(
227
+ current_platform.
228
+ device_id_to_physical_device_id(i))
229
for i in range(self.dp_size * self.tp_size))
230
})
231
server.__enter__()
@@ -330,7 +334,7 @@ async def test_multinode_dp_completion(client: openai.AsyncOpenAI,
334
completion = await client.completions.create(
335
model=model_name,
336
prompt="Hello, my name is",
333
- max_tokens=10,
337
+ max_tokens=5,
338
temperature=1.0)
339
340
assert completion.id is not None
@@ -361,8 +365,11 @@ async def test_multinode_dp_completion(client: openai.AsyncOpenAI,
365
await asyncio.sleep(0.5)
366
367
# Send multiple requests - internal LB should distribute across DP ranks
364
- num_requests = 50
365
- all_tasks = [make_request() for _ in range(num_requests)]
368
+ num_requests = 200
369
+ all_tasks = []
370
+ for _ in range(num_requests):
371
+ all_tasks.append(asyncio.create_task(make_request()))
372
+ await asyncio.sleep(0.01)
373
374
results = await asyncio.gather(*all_tasks)
375
assert len(results) == num_requests
@@ -371,7 +378,10 @@ async def test_multinode_dp_completion(client: openai.AsyncOpenAI,
378
await asyncio.sleep(0.5)
379
380
# Second burst of requests
374
- all_tasks = [make_request() for _ in range(num_requests)]
381
+ all_tasks = []
382
+ for _ in range(num_requests):
383
+ all_tasks.append(asyncio.create_task(make_request()))
384
+ await asyncio.sleep(0.01)
385
386
results = await asyncio.gather(*all_tasks)
387
assert len(results) == num_requests
@@ -449,8 +459,11 @@ async def test_multinode_dp_completion_streaming(client: openai.AsyncOpenAI,
459
460
# Send multiple streaming requests - internal LB should distribute across
461
# DP ranks
452
- num_requests = 50
453
- all_tasks = [make_streaming_request() for _ in range(num_requests)]
462
+ num_requests = 200
463
+ all_tasks = []
464
+ for _ in range(num_requests):
465
+ all_tasks.append(asyncio.create_task(make_streaming_request()))
466
+ await asyncio.sleep(0.01)
467
468
results = await asyncio.gather(*all_tasks)
469
assert len(results) == num_requests
@@ -459,7 +472,10 @@ async def test_multinode_dp_completion_streaming(client: openai.AsyncOpenAI,
472
await asyncio.sleep(0.5)
473
474
# Second burst of streaming requests
462
- all_tasks = [make_streaming_request() for _ in range(num_requests)]
475
+ all_tasks = []
476
+ for _ in range(num_requests):
477
+ all_tasks.append(asyncio.create_task(make_streaming_request()))
478
+ await asyncio.sleep(0.01)
479
480
results = await asyncio.gather(*all_tasks)
481
assert len(results) == num_requests
@@ -492,7 +508,7 @@ async def test_api_only_multinode_dp_completion(
508
completion = await api_only_client.completions.create(
509
model=model_name,
510
prompt="Hello, my name is",
495
- max_tokens=10,
511
+ max_tokens=5,
512
temperature=1.0)
513
514
assert completion.id is not None
@@ -522,8 +538,11 @@ async def test_api_only_multinode_dp_completion(
538
539
# Send multiple requests - should be distributed across engines on
540
# headless server
525
- num_requests = 50
526
- all_tasks = [make_request() for _ in range(num_requests)]
541
+ num_requests = 200
542
+ all_tasks = []
543
+ for _ in range(num_requests):
544
+ all_tasks.append(asyncio.create_task(make_request()))
545
+ await asyncio.sleep(0.01)
546
547
results = await asyncio.gather(*all_tasks)
548
assert len(results) == num_requests
@@ -532,7 +551,10 @@ async def test_api_only_multinode_dp_completion(
551
await asyncio.sleep(0.5)
552
553
# Second burst of requests
535
- all_tasks = [make_request() for _ in range(num_requests)]
554
+ all_tasks = []
555
+ for _ in range(num_requests):
556
+ all_tasks.append(asyncio.create_task(make_request()))
557
+ await asyncio.sleep(0.01)
558
559
results = await asyncio.gather(*all_tasks)
560
assert len(results) == num_requests
@@ -610,8 +632,11 @@ async def test_api_only_multinode_dp_completion_streaming(
632
await asyncio.sleep(0.5)
633
634
# Send multiple streaming requests - should be distributed across engines
613
- num_requests = 50
614
- all_tasks = [make_streaming_request() for _ in range(num_requests)]
635
+ num_requests = 200
636
+ all_tasks = []
637
+ for _ in range(num_requests):
638
+ all_tasks.append(asyncio.create_task(make_streaming_request()))
639
+ await asyncio.sleep(0.01)
640
641
results = await asyncio.gather(*all_tasks)
642
assert len(results) == num_requests
@@ -620,7 +645,10 @@ async def test_api_only_multinode_dp_completion_streaming(
645
await asyncio.sleep(0.5)
646
647
# Second burst of streaming requests
623
- all_tasks = [make_streaming_request() for _ in range(num_requests)]
648
+ all_tasks = []
649
+ for _ in range(num_requests):
650
+ all_tasks.append(asyncio.create_task(make_streaming_request()))
651
+ await asyncio.sleep(0.01)
652
653
results = await asyncio.gather(*all_tasks)
654
assert len(results) == num_requests