A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI/Build] Reorganize root-level V1 tests (#25767)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Sep 27, 2025 at 13:51 UTC
cd87bfbf37f2300b7076b496366cd69048819777
12 files changed
+75
-81
.buildkite/scripts/hardware_ci/run-xpu-test.sh
+1
-2
@@ -44,7 +44,6 @@ docker run \
44
pytest -v -s v1/structured_output
45
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_eagle.py --ignore=v1/spec_decode/test_tree_attention.py
46
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector.py --ignore=v1/kv_connector/unit/test_shared_storage_connector.py
47
+ pytest -v -s v1/test_metrics
48
pytest -v -s v1/test_serial_utils.py
48
- pytest -v -s v1/test_utils.py
49
- pytest -v -s v1/test_metrics_reader.py
49
'
.buildkite/test-pipeline.yaml
+8
-15
@@ -159,10 +159,7 @@ steps:
159
- examples/offline_inference/rlhf.py
160
- examples/offline_inference/rlhf_colocate.py
161
- tests/examples/offline_inference/data_parallel.py
162
- - tests/v1/test_async_llm_dp.py
163
- - tests/v1/test_external_lb_dp.py
164
- - tests/v1/test_internal_lb_dp.py
165
- - tests/v1/test_hybrid_lb_dp.py
162
+ - tests/v1/distributed
163
- tests/v1/engine/test_engine_core_client.py
164
- tests/distributed/test_symm_mem_allreduce.py
165
commands:
@@ -180,10 +177,10 @@ steps:
177
- TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
178
# test with internal dp
179
- python3 ../examples/offline_inference/data_parallel.py --enforce-eager
183
- - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/test_async_llm_dp.py
184
- - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/test_external_lb_dp.py
185
- - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/test_internal_lb_dp.py
186
- - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/test_hybrid_lb_dp.py
180
+ - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
181
+ - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
182
+ - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
183
+ - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
184
- pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
185
- pytest -v -s distributed/test_utils.py
186
- pytest -v -s compile/test_basic_correctness.py
@@ -300,12 +297,9 @@ steps:
297
- pytest -v -s v1/spec_decode
298
- pytest -v -s v1/kv_connector/unit
299
- pytest -v -s v1/metrics
303
- - pytest -v -s v1/test_kv_sharing.py
304
- - pytest -v -s v1/test_metrics_reader.py
300
- pytest -v -s v1/test_oracle.py
301
- pytest -v -s v1/test_request.py
302
- pytest -v -s v1/test_serial_utils.py
308
- - pytest -v -s v1/test_utils.py
303
# Integration test for streaming correctness (requires special branch).
304
- pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
305
- pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
@@ -895,14 +889,13 @@ steps:
889
- tests/compile/test_wrapper.py
890
- tests/distributed/
891
- tests/entrypoints/llm/test_collective_rpc.py
898
- - tests/v1/test_async_llm_dp.py
899
- - tests/v1/test_external_lb_dp.py
892
+ - tests/v1/distributed
893
- tests/v1/entrypoints/openai/test_multi_api_servers.py
894
- tests/v1/shutdown
895
- tests/v1/worker/test_worker_memory_snapshot.py
896
commands:
904
- - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/test_async_llm_dp.py
905
- - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/test_external_lb_dp.py
897
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
898
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
899
- DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py
900
- pytest -v -s entrypoints/llm/test_collective_rpc.py
901
- pytest -v -s ./compile/test_basic_correctness.py
tests/v1/core/test_kv_sharing.py
renamed
tests/v1/distributed/__init__.py
tests/v1/distributed/test_async_llm_dp.py
renamed
tests/v1/distributed/test_external_lb_dp.py
renamed
tests/v1/distributed/test_hybrid_lb_dp.py
renamed
+1
-1
@@ -12,7 +12,7 @@ import pytest_asyncio
12
import requests
13
14
from tests.utils import RemoteOpenAIServer
15
-from tests.v1.test_utils import check_request_balancing
15
+from tests.v1.utils import check_request_balancing
16
from vllm.platforms import current_platform
17
18
MODEL_NAME = "ibm-research/PowerMoE-3b"
tests/v1/distributed/test_internal_lb_dp.py
renamed
+1
-1
@@ -13,7 +13,7 @@ import pytest_asyncio
13
import requests
14
15
from tests.utils import RemoteOpenAIServer
16
-from tests.v1.test_utils import check_request_balancing
16
+from tests.v1.utils import check_request_balancing
17
from vllm.platforms import current_platform
18
19
MODEL_NAME = "ibm-research/PowerMoE-3b"
tests/v1/entrypoints/openai/test_multi_api_servers.py
+1
-1
@@ -8,7 +8,7 @@ import pytest
8
import pytest_asyncio
9
10
from tests.utils import RemoteOpenAIServer
11
-from tests.v1.test_utils import check_request_balancing
11
+from tests.v1.utils import check_request_balancing
12
13
MODEL_NAME = "ibm-research/PowerMoE-3b"
14
tests/v1/metrics/test_metrics_reader.py
renamed
tests/v1/utils.py
renamed
-61
@@ -1,71 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
3
import pytest
4
import regex as re
5
import requests
7
-import torch
6
7
from tests.utils import RemoteOpenAIServer
10
-from vllm.v1.worker.utils import bind_kv_cache
11
-
12
-
13
-def test_bind_kv_cache():
14
- from vllm.attention import Attention
15
-
16
- ctx = {
17
- 'layers.0.self_attn': Attention(32, 128, 0.1),
18
- 'layers.1.self_attn': Attention(32, 128, 0.1),
19
- 'layers.2.self_attn': Attention(32, 128, 0.1),
20
- 'layers.3.self_attn': Attention(32, 128, 0.1),
21
- }
22
- kv_cache = {
23
- 'layers.0.self_attn': torch.zeros((1, )),
24
- 'layers.1.self_attn': torch.zeros((1, )),
25
- 'layers.2.self_attn': torch.zeros((1, )),
26
- 'layers.3.self_attn': torch.zeros((1, )),
27
- }
28
- runner_kv_caches: list[torch.Tensor] = []
29
- bind_kv_cache(kv_cache, ctx, runner_kv_caches)
30
- assert ctx['layers.0.self_attn'].kv_cache[0] is kv_cache[
31
- 'layers.0.self_attn']
32
- assert ctx['layers.1.self_attn'].kv_cache[0] is kv_cache[
33
- 'layers.1.self_attn']
34
- assert ctx['layers.2.self_attn'].kv_cache[0] is kv_cache[
35
- 'layers.2.self_attn']
36
- assert ctx['layers.3.self_attn'].kv_cache[0] is kv_cache[
37
- 'layers.3.self_attn']
38
-
39
- assert runner_kv_caches[0] is kv_cache['layers.0.self_attn']
40
- assert runner_kv_caches[1] is kv_cache['layers.1.self_attn']
41
- assert runner_kv_caches[2] is kv_cache['layers.2.self_attn']
42
- assert runner_kv_caches[3] is kv_cache['layers.3.self_attn']
43
-
44
-
45
-def test_bind_kv_cache_non_attention():
46
- from vllm.attention import Attention
47
-
48
- # example from Jamba PP=2
49
- ctx = {
50
- 'model.layers.20.attn': Attention(32, 128, 0.1),
51
- 'model.layers.28.attn': Attention(32, 128, 0.1),
52
- }
53
- kv_cache = {
54
- 'model.layers.20.attn': torch.zeros((1, )),
55
- 'model.layers.28.attn': torch.zeros((1, )),
56
- }
57
-
58
- runner_kv_caches: list[torch.Tensor] = []
59
- bind_kv_cache(kv_cache, ctx, runner_kv_caches)
60
-
61
- assert ctx['model.layers.20.attn'].kv_cache[0] is kv_cache[
62
- 'model.layers.20.attn']
63
- assert ctx['model.layers.28.attn'].kv_cache[0] is kv_cache[
64
- 'model.layers.28.attn']
65
-
66
- assert runner_kv_caches[0] is kv_cache['model.layers.20.attn']
67
- assert runner_kv_caches[1] is kv_cache['model.layers.28.attn']
68
-
8
9
# Prometheus metrics utilities for testing
10
tests/v1/worker/test_utils.py
new
+63
@@ -0,0 +1,63 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import torch
5
+
6
+from vllm.v1.worker.utils import bind_kv_cache
7
+
8
+
9
+def test_bind_kv_cache():
10
+ from vllm.attention import Attention
11
+
12
+ ctx = {
13
+ 'layers.0.self_attn': Attention(32, 128, 0.1),
14
+ 'layers.1.self_attn': Attention(32, 128, 0.1),
15
+ 'layers.2.self_attn': Attention(32, 128, 0.1),
16
+ 'layers.3.self_attn': Attention(32, 128, 0.1),
17
+ }
18
+ kv_cache = {
19
+ 'layers.0.self_attn': torch.zeros((1, )),
20
+ 'layers.1.self_attn': torch.zeros((1, )),
21
+ 'layers.2.self_attn': torch.zeros((1, )),
22
+ 'layers.3.self_attn': torch.zeros((1, )),
23
+ }
24
+ runner_kv_caches: list[torch.Tensor] = []
25
+ bind_kv_cache(kv_cache, ctx, runner_kv_caches)
26
+ assert ctx['layers.0.self_attn'].kv_cache[0] is kv_cache[
27
+ 'layers.0.self_attn']
28
+ assert ctx['layers.1.self_attn'].kv_cache[0] is kv_cache[
29
+ 'layers.1.self_attn']
30
+ assert ctx['layers.2.self_attn'].kv_cache[0] is kv_cache[
31
+ 'layers.2.self_attn']
32
+ assert ctx['layers.3.self_attn'].kv_cache[0] is kv_cache[
33
+ 'layers.3.self_attn']
34
+
35
+ assert runner_kv_caches[0] is kv_cache['layers.0.self_attn']
36
+ assert runner_kv_caches[1] is kv_cache['layers.1.self_attn']
37
+ assert runner_kv_caches[2] is kv_cache['layers.2.self_attn']
38
+ assert runner_kv_caches[3] is kv_cache['layers.3.self_attn']
39
+
40
+
41
+def test_bind_kv_cache_non_attention():
42
+ from vllm.attention import Attention
43
+
44
+ # example from Jamba PP=2
45
+ ctx = {
46
+ 'model.layers.20.attn': Attention(32, 128, 0.1),
47
+ 'model.layers.28.attn': Attention(32, 128, 0.1),
48
+ }
49
+ kv_cache = {
50
+ 'model.layers.20.attn': torch.zeros((1, )),
51
+ 'model.layers.28.attn': torch.zeros((1, )),
52
+ }
53
+
54
+ runner_kv_caches: list[torch.Tensor] = []
55
+ bind_kv_cache(kv_cache, ctx, runner_kv_caches)
56
+
57
+ assert ctx['model.layers.20.attn'].kv_cache[0] is kv_cache[
58
+ 'model.layers.20.attn']
59
+ assert ctx['model.layers.28.attn'].kv_cache[0] is kv_cache[
60
+ 'model.layers.28.attn']
61
+
62
+ assert runner_kv_caches[0] is kv_cache['model.layers.20.attn']
63
+ assert runner_kv_caches[1] is kv_cache['model.layers.28.attn']