@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5c65a72bb1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[V0 Deprecation] Remove more V0 tests (#25117)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Sep 17, 2025 at 22:05 UTC 5c65a72bb17b34bc6eb0d7ca43b10938c88dc7e3
18 files changed -1500
.buildkite/test-pipeline.yaml
-6
@@ -46,22 +46,18 @@ steps:
46 mirror_hardwares: [amdexperimental]
47 source_file_dependencies:
48 - vllm/
49 - - tests/async_engine
49 - tests/test_inputs.py
50 - tests/test_outputs.py
51 - tests/multimodal
52 - tests/utils_
54 - - tests/worker
53 - tests/standalone_tests/lazy_imports.py
54 - tests/transformers_utils
55 commands:
56 - python3 standalone_tests/lazy_imports.py
59 - - pytest -v -s async_engine # AsyncLLMEngine
57 - pytest -v -s test_inputs.py
58 - pytest -v -s test_outputs.py
59 - pytest -v -s multimodal
60 - pytest -v -s utils_ # Utils
64 - - pytest -v -s worker # Worker
61 - pytest -v -s transformers_utils # transformers_utils
62
63 - label: Python-only Installation Test # 10min
@@ -82,14 +78,12 @@ steps:
78 - vllm/
79 - tests/basic_correctness/test_basic_correctness
80 - tests/basic_correctness/test_cpu_offload
85 - - tests/basic_correctness/test_preemption
81 - tests/basic_correctness/test_cumem.py
82 commands:
83 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
84 - pytest -v -s basic_correctness/test_cumem.py
85 - pytest -v -s basic_correctness/test_basic_correctness.py
86 - pytest -v -s basic_correctness/test_cpu_offload.py
92 - - VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
87
88 - label: Entrypoints Unit Tests # 5min
89 timeout_in_minutes: 10
.github/CODEOWNERS
-2
@@ -41,7 +41,6 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson
41
42 # Test ownership
43 /.buildkite/lm-eval-harness @mgoin @simon-mo
44 -/tests/async_engine @njhill @robertgshaw2-redhat @simon-mo
44 /tests/distributed/test_multi_node_assignment.py @youkaichao
45 /tests/distributed/test_pipeline_parallel.py @youkaichao
46 /tests/distributed/test_same_node.py @youkaichao
@@ -50,7 +49,6 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson
49 /tests/kernels @mgoin @tlrmchlsmth @WoosukKwon @yewentao256
50 /tests/models @DarkLight1337 @ywang96
51 /tests/multimodal @DarkLight1337 @ywang96 @NickLucche
53 -/tests/prefix_caching @comaniac @KuntaiDu
52 /tests/quantization @mgoin @robertgshaw2-redhat @yewentao256
53 /tests/test_inputs.py @DarkLight1337 @ywang96
54 /tests/v1/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
tests/async_engine/__init__.py
tests/async_engine/api_server_async_engine.py deleted
-54
@@ -1,54 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""vllm.entrypoints.api_server with some extra logging for testing."""
4 -from collections.abc import Iterable
5 -from typing import Any
6 -
7 -import uvicorn
8 -from fastapi.responses import JSONResponse, Response
9 -
10 -import vllm.entrypoints.api_server
11 -import vllm.envs as envs
12 -from vllm.engine.arg_utils import AsyncEngineArgs
13 -from vllm.engine.async_llm_engine import AsyncLLMEngine
14 -from vllm.utils import FlexibleArgumentParser
15 -
16 -app = vllm.entrypoints.api_server.app
17 -
18 -
19 -class AsyncLLMEngineWithStats(AsyncLLMEngine):
20 -
21 - def __init__(self, *args, **kwargs):
22 - super().__init__(*args, **kwargs)
23 - self._num_aborts = 0
24 -
25 - async def _engine_abort(self, request_ids: Iterable[str]):
26 - ids = list(request_ids)
27 - self._num_aborts += len(ids)
28 - await super()._engine_abort(ids)
29 -
30 - def testing_stats(self) -> dict[str, Any]:
31 - return {"num_aborted_requests": self._num_aborts}
32 -
33 -
34 -@app.get("/stats")
35 -def stats() -> Response:
36 - """Get the statistics of the engine."""
37 - return JSONResponse(engine.testing_stats())
38 -
39 -
40 -if __name__ == "__main__":
41 - parser = FlexibleArgumentParser()
42 - parser.add_argument("--host", type=str, default="localhost")
43 - parser.add_argument("--port", type=int, default=8000)
44 - parser = AsyncEngineArgs.add_cli_args(parser)
45 - args = parser.parse_args()
46 -
47 - engine_args = AsyncEngineArgs.from_cli_args(args)
48 - engine = AsyncLLMEngineWithStats.from_engine_args(engine_args)
49 - vllm.entrypoints.api_server.engine = engine
50 - uvicorn.run(app,
51 - host=args.host,
52 - port=args.port,
53 - log_level="debug",
54 - timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE)
tests/async_engine/conftest.py deleted
-12
@@ -1,12 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import pytest
4 -
5 -
6 -@pytest.fixture(scope="function", autouse=True)
7 -def use_v0_only(monkeypatch):
8 - """
9 - Since this module is V0 only, set VLLM_USE_V1=0 for
10 - all tests in the module.
11 - """
12 - monkeypatch.setenv('VLLM_USE_V1', '0')
tests/async_engine/test_api_server.py deleted
-139
@@ -1,139 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import copyreg
5 -import os
6 -import subprocess
7 -import sys
8 -import time
9 -from multiprocessing import Pool
10 -from pathlib import Path
11 -
12 -import pytest
13 -import requests
14 -import urllib3.exceptions
15 -
16 -
17 -def _pickle_new_connection_error(obj):
18 - """Custom pickler for NewConnectionError to fix tblib compatibility."""
19 - # Extract the original message by removing the "conn: " prefix
20 - full_message = obj.args[0] if obj.args else ""
21 - if ': ' in full_message:
22 - # Split off the connection part and keep the actual message
23 - _, actual_message = full_message.split(': ', 1)
24 - else:
25 - actual_message = full_message
26 - return _unpickle_new_connection_error, (actual_message, )
27 -
28 -
29 -def _unpickle_new_connection_error(message):
30 - """Custom unpickler for NewConnectionError."""
31 - # Create with None as conn and the actual message
32 - return urllib3.exceptions.NewConnectionError(None, message)
33 -
34 -
35 -# Register the custom pickle/unpickle functions for tblib compatibility
36 -copyreg.pickle(urllib3.exceptions.NewConnectionError,
37 - _pickle_new_connection_error)
38 -
39 -
40 -def _query_server(prompt: str, max_tokens: int = 5) -> dict:
41 - response = requests.post("http://localhost:8000/generate",
42 - json={
43 - "prompt": prompt,
44 - "max_tokens": max_tokens,
45 - "temperature": 0,
46 - "ignore_eos": True
47 - })
48 - response.raise_for_status()
49 - return response.json()
50 -
51 -
52 -def _query_server_long(prompt: str) -> dict:
53 - return _query_server(prompt, max_tokens=500)
54 -
55 -
56 -@pytest.fixture
57 -def api_server(distributed_executor_backend: str):
58 - script_path = Path(__file__).parent.joinpath(
59 - "api_server_async_engine.py").absolute()
60 - commands = [
61 - sys.executable,
62 - "-u",
63 - str(script_path),
64 - "--model",
65 - "facebook/opt-125m",
66 - "--host",
67 - "127.0.0.1",
68 - "--distributed-executor-backend",
69 - distributed_executor_backend,
70 - ]
71 -
72 - # API Server Test Requires V0.
73 - my_env = os.environ.copy()
74 - my_env["VLLM_USE_V1"] = "0"
75 - uvicorn_process = subprocess.Popen(commands, env=my_env)
76 - yield
77 - uvicorn_process.terminate()
78 -
79 -
80 -@pytest.mark.timeout(300)
81 -@pytest.mark.parametrize("distributed_executor_backend", ["mp", "ray"])
82 -def test_api_server(api_server, distributed_executor_backend: str):
83 - """
84 - Run the API server and test it.
85 -
86 - We run both the server and requests in separate processes.
87 -
88 - We test that the server can handle incoming requests, including
89 - multiple requests at the same time, and that it can handle requests
90 - being cancelled without crashing.
91 - """
92 - with Pool(32) as pool:
93 - # Wait until the server is ready
94 - prompts = ["warm up"] * 1
95 - result = None
96 - while not result:
97 - try:
98 - for r in pool.map(_query_server, prompts):
99 - result = r
100 - break
101 - except requests.exceptions.ConnectionError:
102 - time.sleep(1)
103 -
104 - # Actual tests start here
105 - # Try with 1 prompt
106 - for result in pool.map(_query_server, prompts):
107 - assert result
108 -
109 - num_aborted_requests = requests.get(
110 - "http://localhost:8000/stats").json()["num_aborted_requests"]
111 - assert num_aborted_requests == 0
112 -
113 - # Try with 100 prompts
114 - prompts = ["test prompt"] * 100
115 - for result in pool.map(_query_server, prompts):
116 - assert result
117 -
118 - with Pool(32) as pool:
119 - # Cancel requests
120 - prompts = ["canceled requests"] * 100
121 - pool.map_async(_query_server_long, prompts)
122 - time.sleep(0.01)
123 - pool.terminate()
124 - pool.join()
125 -
126 - # check cancellation stats
127 - # give it some time to update the stats
128 - time.sleep(1)
129 -
130 - num_aborted_requests = requests.get(
131 - "http://localhost:8000/stats").json()["num_aborted_requests"]
132 - assert num_aborted_requests > 0
133 -
134 - # check that server still runs after cancellations
135 - with Pool(32) as pool:
136 - # Try with 100 prompts
137 - prompts = ["test prompt after canceled"] * 100
138 - for result in pool.map(_query_server, prompts):
139 - assert result
tests/async_engine/test_request_tracker.py deleted
-71
@@ -1,71 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import pytest
5 -
6 -from vllm.engine.async_llm_engine import RequestTracker
7 -from vllm.outputs import RequestOutput
8 -
9 -
10 -@pytest.mark.asyncio
11 -async def test_request_tracker():
12 - tracker = RequestTracker()
13 - stream_1 = tracker.add_request("1")
14 - assert tracker.new_requests_event.is_set()
15 - await tracker.wait_for_new_requests()
16 - new, aborted = tracker.get_new_and_aborted_requests()
17 - assert not tracker.new_requests_event.is_set()
18 - assert len(new) == 1
19 - assert new[0]["request_id"] == "1"
20 - assert not aborted
21 - assert not stream_1.finished
22 -
23 - stream_2 = tracker.add_request("2")
24 - stream_3 = tracker.add_request("3")
25 - assert tracker.new_requests_event.is_set()
26 - await tracker.wait_for_new_requests()
27 - new, aborted = tracker.get_new_and_aborted_requests()
28 - assert not tracker.new_requests_event.is_set()
29 - assert len(new) == 2
30 - assert new[0]["request_id"] == "2"
31 - assert new[1]["request_id"] == "3"
32 - assert not aborted
33 - assert not stream_2.finished
34 - assert not stream_3.finished
35 -
36 - # request_ids must be unique
37 - with pytest.raises(KeyError):
38 - tracker.add_request("1")
39 - assert not tracker.new_requests_event.is_set()
40 -
41 - tracker.abort_request("1")
42 - new, aborted = tracker.get_new_and_aborted_requests()
43 - assert len(aborted) == 1
44 - assert "1" in aborted
45 - assert not new
46 - assert stream_1.finished
47 -
48 - stream_4 = tracker.add_request("4")
49 - tracker.abort_request("4")
50 - assert tracker.new_requests_event.is_set()
51 - await tracker.wait_for_new_requests()
52 - new, aborted = tracker.get_new_and_aborted_requests()
53 - # aborted new requests will cancel each other out -
54 - # there's no need for them to propagate into the
55 - # engine
56 - assert not aborted
57 - assert not new
58 - assert stream_4.finished
59 -
60 - stream_5 = tracker.add_request("5")
61 - assert tracker.new_requests_event.is_set()
62 - tracker.process_request_output(
63 - RequestOutput("2", "output", [], [], [], finished=True))
64 - await tracker.wait_for_new_requests()
65 - new, aborted = tracker.get_new_and_aborted_requests()
66 - assert not tracker.new_requests_event.is_set()
67 - assert not aborted
68 - assert len(new) == 1
69 - assert new[0]["request_id"] == "5"
70 - assert stream_2.finished
71 - assert not stream_5.finished
tests/basic_correctness/test_preemption.py deleted
-189
@@ -1,189 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Compare the short outputs of HF and vLLM when using greedy sampling.
4 -
5 -VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 has to be set before running this test.
6 -
7 -Run `VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1
8 -pytest tests/basic_correctness/test_preemption.py`.
9 -"""
10 -import pytest
11 -from prometheus_client import REGISTRY
12 -
13 -import vllm.envs as envs
14 -from vllm import SamplingParams
15 -from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
16 - ENABLE_ARTIFICIAL_PREEMPT)
17 -
18 -from ..models.utils import check_outputs_equal
19 -
20 -MODELS = [
21 - "distilbert/distilgpt2",
22 -]
23 -
24 -
25 -@pytest.fixture(scope="function", autouse=True)
26 -def use_v0_only(monkeypatch):
27 - """
28 - We should enable this for V1, but VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT,
29 - so use VLLM_USE_V1=0 for all tests in the file.
30 - """
31 - monkeypatch.setenv('VLLM_USE_V1', '0')
32 -
33 -
34 -@pytest.fixture(scope="module", autouse=True)
35 -def check_settings():
36 - assert ENABLE_ARTIFICIAL_PREEMPT is True, (
37 - "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1."
38 - "`VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 "
39 - "pytest tests/basic_correctness/test_preemption.py`")
40 -
41 -
42 -@pytest.fixture
43 -def distributed_executor_backend() -> str:
44 - # When SPMD worker is used, use distributed_executor_backend="ray"
45 - # to test delta input optimization works with preemption.
46 - return "ray" if envs.VLLM_USE_RAY_SPMD_WORKER else "mp"
47 -
48 -
49 -@pytest.mark.parametrize("model", MODELS)
50 -@pytest.mark.parametrize("dtype", ["half"])
51 -@pytest.mark.parametrize("max_tokens", [96])
52 -@pytest.mark.parametrize("chunked_prefill_token_size", [16])
53 -def test_chunked_prefill_recompute(
54 - hf_runner,
55 - vllm_runner,
56 - example_prompts,
57 - model: str,
58 - dtype: str,
59 - max_tokens: int,
60 - chunked_prefill_token_size: int,
61 - distributed_executor_backend: str,
62 -) -> None:
63 - """Ensure that chunked prefill works with preemption."""
64 - max_num_seqs = min(chunked_prefill_token_size, 256)
65 - enable_chunked_prefill = False
66 - max_num_batched_tokens = None
67 - if chunked_prefill_token_size != -1:
68 - enable_chunked_prefill = True
69 - max_num_batched_tokens = chunked_prefill_token_size
70 -
71 - with hf_runner(model, dtype=dtype) as hf_model:
72 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
73 -
74 - with vllm_runner(
75 - model,
76 - dtype=dtype,
77 - max_num_batched_tokens=max_num_batched_tokens,
78 - enable_chunked_prefill=enable_chunked_prefill,
79 - max_num_seqs=max_num_seqs,
80 - distributed_executor_backend=distributed_executor_backend,
81 - disable_log_stats=False,
82 - ) as vllm_model:
83 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
84 - assert (vllm_model.llm.llm_engine.scheduler[0].artificial_preempt_cnt
85 - < ARTIFICIAL_PREEMPTION_MAX_CNT)
86 -
87 - for i in range(len(example_prompts)):
88 - hf_output_ids, hf_output_str = hf_outputs[i]
89 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
90 - assert hf_output_str == vllm_output_str, (
91 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
92 - assert hf_output_ids == vllm_output_ids, (
93 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
94 -
95 -
96 -@pytest.mark.parametrize("model", MODELS)
97 -@pytest.mark.parametrize("dtype", ["float"])
98 -@pytest.mark.parametrize("max_tokens", [96])
99 -def test_preemption(
100 - caplog_vllm,
101 - hf_runner,
102 - vllm_runner,
103 - example_prompts,
104 - model: str,
105 - dtype: str,
106 - max_tokens: int,
107 - distributed_executor_backend: str,
108 -) -> None:
109 - """By default, recompute preemption is enabled"""
110 -
111 - with hf_runner(model, dtype=dtype) as hf_model:
112 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
113 -
114 - with vllm_runner(
115 - model,
116 - dtype=dtype,
117 - disable_log_stats=False,
118 - distributed_executor_backend=distributed_executor_backend,
119 - ) as vllm_model:
120 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
121 - assert (vllm_model.llm.llm_engine.scheduler[0].artificial_preempt_cnt
122 - < ARTIFICIAL_PREEMPTION_MAX_CNT)
123 - total_preemption = (
124 - vllm_model.llm.llm_engine.scheduler[0].num_cumulative_preemption)
125 -
126 - check_outputs_equal(
127 - outputs_0_lst=hf_outputs,
128 - outputs_1_lst=vllm_outputs,
129 - name_0="hf",
130 - name_1="vllm",
131 - )
132 -
133 - assert ("is preempted by PreemptionMode.RECOMPUTE mode because there "
134 - "is not enough KV cache space." in caplog_vllm.text)
135 - # Ensure the count bucket of request-level histogram metrics matches
136 - # the number of requests as a simple sanity check to ensure metrics are
137 - # generated
138 - preemption_metrics = None
139 - for m in REGISTRY.collect():
140 - if m.name == "vllm:num_preemptions":
141 - preemption_metrics = m
142 - assert preemption_metrics is not None
143 - total_recorded_preemption = 0
144 - for sample in preemption_metrics.samples:
145 - total_recorded_preemption += sample.value
146 - assert total_preemption == total_recorded_preemption
147 -
148 -
149 -@pytest.mark.parametrize("model", MODELS)
150 -@pytest.mark.parametrize("dtype", ["float"])
151 -@pytest.mark.parametrize("max_tokens", [96])
152 -def test_preemption_infeasible(
153 - vllm_runner,
154 - example_prompts,
155 - model: str,
156 - dtype: str,
157 - max_tokens: int,
158 - distributed_executor_backend: str,
159 -) -> None:
160 - """Verify infeasible preemption request will be ignored."""
161 - BLOCK_SIZE = 16
162 - prefill_blocks = 2
163 - decode_blocks = max_tokens // BLOCK_SIZE
164 - with vllm_runner(
165 - model,
166 - dtype=dtype,
167 - block_size=BLOCK_SIZE,
168 - # Not enough gpu blocks to complete a single sequence.
169 - # preemption should happen, and the sequence should be
170 - # ignored instead of hanging forever.
171 - num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
172 - max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
173 - distributed_executor_backend=distributed_executor_backend,
174 - ) as vllm_model:
175 - sampling_params = SamplingParams(max_tokens=max_tokens,
176 - ignore_eos=True)
177 - req_outputs = vllm_model.llm.generate(
178 - example_prompts,
179 - sampling_params=sampling_params,
180 - )
181 -
182 - assert (vllm_model.llm.llm_engine.scheduler[0].artificial_preempt_cnt
183 - < ARTIFICIAL_PREEMPTION_MAX_CNT)
184 -
185 - # Verify the request is ignored and not hang.
186 - for req_output in req_outputs:
187 - outputs = req_output.outputs
188 - assert len(outputs) == 1
189 - assert outputs[0].finish_reason == "length"
tests/detokenizer/conftest.py deleted
-11
@@ -1,11 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import pytest
4 -
5 -
6 -@pytest.fixture(autouse=True)
7 -def v1(run_with_both_engines):
8 - # Simple autouse wrapper to run both engines for each test
9 - # This can be promoted up to conftest.py to run for every
10 - # test in a package
11 - pass
tests/detokenizer/test_stop_checker.py deleted
-83
@@ -1,83 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import pytest
5 -
6 -from vllm.engine.output_processor.stop_checker import StopChecker
7 -from vllm.inputs import token_inputs
8 -from vllm.sampling_params import SamplingParams
9 -from vllm.sequence import Logprob, Sequence, SequenceStatus
10 -
11 -
12 -def sequence_with_eos(text: str, eos_token: str,
13 - eos_token_id: int) -> Sequence:
14 - """
15 - Create a Sequence that ends with an EOS token.
16 - """
17 - seq = Sequence(
18 - seq_id=0,
19 - inputs=token_inputs([]),
20 - block_size=16,
21 - eos_token_id=eos_token_id,
22 - )
23 - seq.output_text = text + eos_token
24 -
25 - offset = eos_token_id + 1
26 - for i in range(offset, len(text) + offset):
27 - seq.append_token_id(token_id=i, logprobs={i: Logprob(0.0)})
28 - seq.append_token_id(token_id=eos_token_id,
29 - logprobs={eos_token_id: Logprob(0.0)})
30 -
31 - seq.status = SequenceStatus.RUNNING
32 -
33 - return seq
34 -
35 -
36 -@pytest.mark.parametrize(["text_wo_eos", "eos_token", "eos_token_id"], [
37 - ("This text ends with EOS token", "</s>", 2),
38 -])
39 -@pytest.mark.parametrize("ignore_eos", [True, False])
40 -@pytest.mark.parametrize("include_stop_str_in_output", [True, False])
41 -@pytest.mark.skip_global_cleanup
42 -def test_stop_on_eos_token(text_wo_eos: str, eos_token: str, eos_token_id: int,
43 - ignore_eos: bool, include_stop_str_in_output: bool):
44 - """
45 - Test the behavior of the StopChecker's maybe_stop_sequence method
46 - when an EOS token is encountered.
47 -
48 - This test covers:
49 - - When the EOS token should stop the sequence and be removed from the output
50 - - When the EOS token should stop the sequence and be included in the output
51 - - When the EOS token should be ignored, and the sequence continues
52 - """
53 -
54 - stop_checker = StopChecker(max_model_len=1024)
55 -
56 - seq = sequence_with_eos(
57 - text=text_wo_eos,
58 - eos_token=eos_token,
59 - eos_token_id=eos_token_id,
60 - )
61 - new_char_count = len(eos_token)
62 -
63 - # Note that `stop` and `stop_token_ids` are not specified
64 - sampling_params = SamplingParams(
65 - min_tokens=1,
66 - ignore_eos=ignore_eos,
67 - include_stop_str_in_output=include_stop_str_in_output)
68 -
69 - stop_checker.maybe_stop_sequence(
70 - seq=seq,
71 - new_char_count=new_char_count,
72 - sampling_params=sampling_params,
73 - )
74 -
75 - if ignore_eos:
76 - assert seq.status == SequenceStatus.RUNNING
77 - assert seq.output_text == text_wo_eos + eos_token
78 - elif include_stop_str_in_output:
79 - assert seq.status == SequenceStatus.FINISHED_STOPPED
80 - assert seq.output_text == text_wo_eos + eos_token
81 - else:
82 - assert seq.status == SequenceStatus.FINISHED_STOPPED
83 - assert seq.output_text == text_wo_eos
tests/entrypoints/openai/correctness/test_lmeval.py
-10
@@ -81,13 +81,3 @@ def test_lm_eval_accuracy_v1_engine(monkeypatch: pytest.MonkeyPatch):
81 more_args = ["--max-num-seqs", "64"]
82
83 run_test(more_args)
84 -
85 -
86 -@pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
87 -def test_lm_eval_accuracy_v0_engine(monkeypatch: pytest.MonkeyPatch,
88 - more_args):
89 - """Run with the V0 Engine."""
90 -
91 - with monkeypatch.context() as m:
92 - m.setenv("VLLM_USE_V1", "0")
93 - run_test(more_args)
tests/samplers/test_logprobs.py deleted
-182
@@ -1,182 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import pytest
5 -import torch
6 -
7 -from vllm import SamplingParams
8 -
9 -from ..conftest import VllmRunner
10 -
11 -MODELS = ["distilbert/distilgpt2"]
12 -
13 -
14 -@pytest.fixture(scope="function", autouse=True)
15 -def use_v0_only(monkeypatch):
16 - """
17 - This module is V0 only since it uses dtype=float, so
18 - set VLLM_USE_V1=0 for all tests in the module.
19 - """
20 - monkeypatch.setenv('VLLM_USE_V1', '0')
21 -
22 -
23 -@pytest.mark.parametrize("model", MODELS)
24 -@pytest.mark.parametrize("dtype",
25 - ["float"]) # needed for comparing logprobs with HF
26 -@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
27 -@pytest.mark.parametrize("num_top_logprobs", [0, 6]) # 32000 == vocab_size
28 -@pytest.mark.parametrize("detokenize", [True, False])
29 -def test_get_prompt_logprobs(
30 - hf_runner,
31 - vllm_runner,
32 - model,
33 - dtype,
34 - chunked_prefill_token_size: int,
35 - num_top_logprobs: int,
36 - detokenize: bool,
37 - example_prompts,
38 -):
39 - max_num_seqs = 256
40 - enable_chunked_prefill = False
41 - max_num_batched_tokens = None
42 - if chunked_prefill_token_size != -1:
43 - enable_chunked_prefill = True
44 - max_num_seqs = min(chunked_prefill_token_size, max_num_seqs)
45 - max_num_batched_tokens = chunked_prefill_token_size
46 -
47 - max_tokens = 5
48 - with hf_runner(model, dtype=dtype) as hf_model:
49 - hf_logprobs = hf_model.generate_greedy_logprobs(
50 - example_prompts,
51 - max_tokens=max_tokens,
52 - )
53 -
54 - with vllm_runner(
55 - model,
56 - dtype=dtype,
57 - max_logprobs=num_top_logprobs,
58 - enable_chunked_prefill=enable_chunked_prefill,
59 - max_num_batched_tokens=max_num_batched_tokens,
60 - max_num_seqs=max_num_seqs,
61 - ) as vllm_model:
62 - vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
63 - logprobs=num_top_logprobs,
64 - prompt_logprobs=num_top_logprobs,
65 - temperature=0.0,
66 - detokenize=detokenize)
67 - vllm_results = vllm_model.llm.generate(
68 - example_prompts, sampling_params=vllm_sampling_params)
69 -
70 - # Test whether logprobs are included in the results.
71 - for result in vllm_results:
72 - assert result.prompt_logprobs is not None
73 - assert result.outputs[0].logprobs is not None
74 - assert len(result.outputs[0].logprobs) == max_tokens
75 - for logprobs in result.outputs[0].logprobs:
76 - # If the output token is not included in the top X
77 - # logprob, it can return 1 more data
78 - assert (len(logprobs) == num_top_logprobs
79 - or len(logprobs) == num_top_logprobs + 1)
80 - output_text = result.outputs[0].text
81 - output_string_from_most_likely_tokens_lst: list[str] = []
82 - for top_logprobs in result.outputs[0].logprobs:
83 - top_logprob = next(iter(top_logprobs.values()))
84 - output_string_from_most_likely_tokens_lst.append(
85 - top_logprob.decoded_token)
86 -
87 - if detokenize:
88 - output_string_from_most_likely_tokens = "".join(
89 - output_string_from_most_likely_tokens_lst)
90 - assert output_text == output_string_from_most_likely_tokens, (
91 - "The output text from the top logprob for each token position "
92 - "should be the same as the output text in the result.")
93 - else:
94 - assert output_text == ''
95 - assert output_string_from_most_likely_tokens_lst == ([None] *
96 - max_tokens)
97 -
98 - # The first prompt logprob is always None
99 - assert result.prompt_logprobs[0] is None
100 - for prompt_logprobs in result.prompt_logprobs[1:]:
101 - # If the prompt token is not included in the top X
102 - # logprob, it can return 1 more data
103 - assert (len(prompt_logprobs) == num_top_logprobs
104 - or len(prompt_logprobs) == num_top_logprobs + 1)
105 -
106 - # Test whether prompt logprobs are consistent with HF
107 - for vllm_result, hf_logprob in zip(vllm_results, hf_logprobs):
108 - # Check prompt logprobs
109 - # The first prompt logprob is always None, so we compare it from 1:.
110 - vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
111 - for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
112 - for token_id, logprob in vllm_prompt_logprob_dict.items():
113 - torch.testing.assert_close(logprob.logprob,
114 - hf_logprob[0][i][token_id].item(),
115 - atol=1e-2,
116 - rtol=1e-2)
117 - vllm_sample_logprobs = vllm_result.outputs[0].logprobs
118 - for i, top_logprobs in enumerate(vllm_sample_logprobs):
119 - for token_id, sample_logprob in top_logprobs.items():
120 - logprob = sample_logprob.logprob
121 - torch.testing.assert_close(logprob,
122 - hf_logprob[i][-1][token_id].item(),
123 - atol=1e-2,
124 - rtol=1e-2)
125 - if detokenize:
126 - assert isinstance(sample_logprob.decoded_token, str), (
127 - "The token should be decoded by the time it is returned"
128 - " to the user.")
129 -
130 - # Test if prompt logprobs are correctly set.
131 - for vllm_result in vllm_results:
132 - token_ids = vllm_result.prompt_token_ids
133 - prompt_logprobs = vllm_result.prompt_logprobs
134 -
135 - # The first token doesn't have logprob.
136 - assert prompt_logprobs[0] is None
137 -
138 - for token_id, logprob_dict in zip(token_ids[1:], prompt_logprobs[1:]):
139 - assert token_id in logprob_dict
140 -
141 -
142 -def test_max_logprobs():
143 - runner = VllmRunner("facebook/opt-125m", max_logprobs=1)
144 - vllm_sampling_params = SamplingParams(logprobs=1)
145 - # should pass
146 - runner.generate(["Hello world"], sampling_params=vllm_sampling_params)
147 -
148 - bad_sampling_params = SamplingParams(logprobs=2)
149 - with pytest.raises(ValueError):
150 - runner.generate(["Hello world"], sampling_params=bad_sampling_params)
151 -
152 -
153 -@pytest.mark.parametrize("model", MODELS)
154 -@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
155 -@pytest.mark.parametrize("detokenize", [True, False])
156 -def test_none_logprobs(vllm_runner, model, chunked_prefill_token_size: int,
157 - detokenize: bool, example_prompts):
158 - max_num_seqs = 256
159 - enable_chunked_prefill = False
160 - max_num_batched_tokens = None
161 - if chunked_prefill_token_size != -1:
162 - enable_chunked_prefill = True
163 - max_num_seqs = min(chunked_prefill_token_size, max_num_seqs)
164 - max_num_batched_tokens = chunked_prefill_token_size
165 - max_tokens = 5
166 -
167 - with vllm_runner(
168 - model,
169 - enable_chunked_prefill=enable_chunked_prefill,
170 - max_num_batched_tokens=max_num_batched_tokens,
171 - max_num_seqs=max_num_seqs,
172 - ) as vllm_model:
173 - sampling_params_logprobs_none = SamplingParams(max_tokens=max_tokens,
174 - logprobs=None,
175 - temperature=0.0,
176 - detokenize=detokenize)
177 - results_logprobs_none = vllm_model.llm.generate(
178 - example_prompts, sampling_params=sampling_params_logprobs_none)
179 -
180 - for i in range(len(results_logprobs_none)):
181 - assert results_logprobs_none[i].outputs[0].logprobs is None
182 - assert results_logprobs_none[i].outputs[0].cumulative_logprob is None
tests/worker/__init__.py
tests/worker/conftest.py deleted
-11
@@ -1,11 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import pytest
4 -
5 -
6 -@pytest.fixture(scope="function", autouse=True)
7 -def use_v0_only(monkeypatch):
8 - """
9 - This module tests V0 internals, so set VLLM_USE_V1=0.
10 - """
11 - monkeypatch.setenv('VLLM_USE_V1', '0')
\ No newline at end of file
tests/worker/test_model_input.py deleted
-113
@@ -1,113 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import dataclasses
5 -
6 -import torch
7 -
8 -from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
9 -from vllm.attention.backends.abstract import AttentionBackend
10 -from vllm.attention.backends.utils import CommonAttentionState
11 -from vllm.model_executor import SamplingMetadata
12 -from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
13 -
14 -
15 -class MockAttentionBackend(AttentionBackend):
16 -
17 - @staticmethod
18 - def get_name() -> str:
19 - raise NotImplementedError
20 -
21 - @staticmethod
22 - def get_impl_cls():
23 - raise NotImplementedError
24 -
25 - @staticmethod
26 - def get_metadata_cls() -> type["AttentionMetadata"]:
27 - return AttentionMetadata
28 -
29 - @staticmethod
30 - def get_builder_cls() -> type["AttentionMetadataBuilder"]:
31 - return AttentionMetadataBuilder
32 -
33 - @staticmethod
34 - def get_state_cls() -> type["CommonAttentionState"]:
35 - return CommonAttentionState
36 -
37 - @staticmethod
38 - def get_kv_cache_shape(
39 - num_blocks: int,
40 - block_size: int,
41 - num_kv_heads: int,
42 - head_size: int,
43 - ) -> tuple[int, ...]:
44 - raise NotImplementedError
45 -
46 - @staticmethod
47 - def swap_blocks(
48 - src_kv_cache: torch.Tensor,
49 - dst_kv_cache: torch.Tensor,
50 - src_to_dst: torch.Tensor,
51 - ) -> None:
52 - pass
53 -
54 - @staticmethod
55 - def copy_blocks(
56 - kv_caches: list[torch.Tensor],
57 - src_to_dists: torch.Tensor,
58 - ) -> None:
59 - pass
60 -
61 -
62 -def test_model_runner_input():
63 - sampling_metadata = SamplingMetadata(
64 - ["seq_group"],
65 - "selected_token_indices",
66 - "categorized_sample_indices",
67 - "num_prompts",
68 - )
69 - attn_metadata = AttentionMetadata(
70 - num_prefills=1,
71 - num_prefill_tokens=2,
72 - num_decode_tokens=3,
73 - slot_mapping=torch.zeros(1),
74 - multi_modal_placeholder_index_maps=None,
75 - enable_kv_scales_calculation=True,
76 - )
77 - model_input = ModelInputForGPUWithSamplingMetadata(
78 - input_tokens=torch.ones(10),
79 - input_positions=torch.ones(10),
80 - sampling_metadata=sampling_metadata,
81 - attn_metadata=attn_metadata)
82 -
83 - assert isinstance(model_input, ModelInputForGPUWithSamplingMetadata)
84 -
85 - # Test round trip serialization.
86 - tensor_dict = model_input.as_broadcastable_tensor_dict()
87 - attn_backend = MockAttentionBackend()
88 - received_model_input = (
89 - ModelInputForGPUWithSamplingMetadata.from_broadcasted_tensor_dict(
90 - tensor_dict, attn_backend=attn_backend))
91 - # Check that received copy has correct values.
92 - assert isinstance(received_model_input,
93 - ModelInputForGPUWithSamplingMetadata)
94 - assert received_model_input.input_tokens is not None
95 - assert (
96 - received_model_input.input_tokens == model_input.input_tokens).all()
97 - assert received_model_input.input_positions is not None
98 - assert (received_model_input.input_positions == model_input.input_positions
99 - ).all()
100 - assert received_model_input.multi_modal_kwargs is None
101 - assert (received_model_input.multi_modal_kwargs ==
102 - model_input.multi_modal_kwargs)
103 - assert received_model_input.lora_requests is None
104 - assert received_model_input.lora_requests == model_input.lora_requests
105 - assert received_model_input.lora_mapping is None
106 - assert received_model_input.lora_mapping == model_input.lora_mapping
107 - for field in dataclasses.fields(AttentionMetadata):
108 - assert getattr(received_model_input.attn_metadata, field.name,
109 - None) == getattr(attn_metadata, field.name, None)
110 - # For sampling metadata, only selected_token_indices is copied.
111 - assert (received_model_input.sampling_metadata.selected_token_indices ==
112 - sampling_metadata.selected_token_indices)
113 - assert received_model_input.sampling_metadata.seq_groups is None
tests/worker/test_model_runner.py deleted
-462
@@ -1,462 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import pytest
5 -import torch
6 -
7 -from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
8 - init_distributed_environment)
9 -from vllm.engine.arg_utils import EngineArgs
10 -from vllm.model_executor.sampling_metadata import SamplingMetadata
11 -from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12 -from vllm.utils import get_open_port
13 -from vllm.worker.model_runner import ModelRunner
14 -
15 -
16 -def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
17 - engine_args = EngineArgs(model, *args, **kwargs)
18 - engine_config = engine_args.create_engine_config()
19 - model_runner = ModelRunner(
20 - vllm_config=engine_config,
21 - is_driver_worker=True,
22 - )
23 - return model_runner
24 -
25 -
26 -def test_deepseek_mla_attn_backend_module():
27 - model_runner = _create_model_runner(
28 - "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
29 - trust_remote_code=True,
30 - enable_chunked_prefill=False,
31 - )
32 - assert model_runner.attn_backend.__name__ == "TritonMLABackend"
33 -
34 -
35 -@pytest.mark.parametrize("batch_size", list(range(1, 257, 3)))
36 -@pytest.mark.parametrize("use_prompt_embeds", [True, False])
37 -def test_prepare_prompt(batch_size, use_prompt_embeds, monkeypatch):
38 - if use_prompt_embeds:
39 - # Prompt Embeddings is only currently supported on V0
40 - monkeypatch.setenv("VLLM_USE_V1", "0")
41 -
42 - model_runner = _create_model_runner(
43 - "facebook/opt-125m",
44 - max_num_batched_tokens=100000,
45 - max_num_seqs=100000,
46 - enable_chunked_prefill=False,
47 - enable_prompt_embeds=True,
48 - )
49 -
50 - seq_lens: list[int] = []
51 - seq_group_metadata_list: list[SequenceGroupMetadata] = []
52 - block_tables = {0: [1]}
53 - expected_input_embeds_len = 0
54 - for i in range(batch_size):
55 - # make sure all tokens fit into one block
56 - seq_len = i % (model_runner.block_size - 1) + 1
57 - seq_lens.append(seq_len)
58 - if use_prompt_embeds:
59 - seq_data = SequenceData.from_seqs(
60 - prompt_token_ids=[0] * seq_len,
61 - prompt_embeds=torch.rand(seq_len, 10),
62 - )
63 - expected_input_embeds_len += seq_len
64 - else:
65 - seq_data = SequenceData.from_seqs(prompt_token_ids=range(seq_len))
66 -
67 - seq_group_metadata = SequenceGroupMetadata(
68 - request_id=f"test_{i}",
69 - is_prompt=True,
70 - seq_data={0: seq_data},
71 - sampling_params=SamplingParams(temperature=0),
72 - block_tables=block_tables,
73 - )
74 - assert seq_group_metadata.token_chunk_size == seq_data.get_len()
75 - seq_group_metadata_list.append(seq_group_metadata)
76 -
77 - expected_selected_token_indices = []
78 - selected_token_start_idx = 0
79 - for seq_len in seq_lens:
80 - expected_selected_token_indices.append(selected_token_start_idx +
81 - seq_len - 1)
82 - selected_token_start_idx += seq_len
83 - model_input = model_runner._prepare_model_input_tensors(
84 - seq_group_metadata_list)
85 - input_tokens = model_input.input_tokens
86 - input_positions = model_input.input_positions
87 - input_embeds = model_input.inputs_embeds
88 - attn_metadata = model_input.attn_metadata
89 - return_seq_lens = model_input.seq_lens
90 - slot_mapping = attn_metadata.slot_mapping
91 - assert return_seq_lens == seq_lens
92 - assert len(slot_mapping) == len(input_tokens)
93 -
94 - # Verify input metadata is correct for prompts.
95 - device = model_runner.device
96 - assert attn_metadata.num_prefills > 0
97 - assert attn_metadata.num_decode_tokens == 0
98 - torch.testing.assert_close(
99 - attn_metadata.seq_lens_tensor,
100 - torch.tensor(seq_lens, device=device, dtype=torch.int))
101 - assert attn_metadata.seq_lens == seq_lens
102 - assert attn_metadata.max_prefill_seq_len == max(seq_lens)
103 - assert attn_metadata.max_decode_seq_len == 0
104 -
105 - # Test subquery start locs.
106 - start_idx = 0
107 - start_loc = [start_idx]
108 - for seq_len in seq_lens:
109 - start_idx += seq_len
110 - start_loc.append(start_idx)
111 - torch.testing.assert_close(
112 - attn_metadata.query_start_loc,
113 - torch.tensor(start_loc, dtype=torch.int32, device=device))
114 -
115 - # Test seq start locs. Note that for normal prefill it is
116 - # equivalent to query_start_loc.
117 - start_idx = 0
118 - seq_start_loc = [start_idx]
119 - for seq_len in seq_lens:
120 - start_idx += seq_len
121 - seq_start_loc.append(start_idx)
122 -
123 - torch.testing.assert_close(
124 - attn_metadata.seq_start_loc,
125 - torch.tensor(start_loc, dtype=torch.int32, device=device))
126 - torch.testing.assert_close(
127 - attn_metadata.context_lens_tensor,
128 - torch.zeros(attn_metadata.context_lens_tensor.shape[0],
129 - dtype=torch.int,
130 - device=device))
131 -
132 - expected = torch.tensor([[] for _ in range(len(seq_group_metadata_list))],
133 - dtype=torch.int32,
134 - device=model_runner.device)
135 - torch.testing.assert_close(attn_metadata.block_tables, expected)
136 - # Cuda graph should not be used for prerill.
137 - assert attn_metadata.use_cuda_graph is False
138 -
139 - assert len(input_tokens) == sum(seq_lens)
140 - assert len(input_positions) == sum(seq_lens)
141 - if expected_input_embeds_len == 0:
142 - torch.testing.assert_close(input_tokens, input_positions)
143 - assert input_embeds is None
144 - else:
145 - assert len(input_embeds) == expected_input_embeds_len
146 -
147 - sampling_metadata = SamplingMetadata.prepare(
148 - seq_group_metadata_list,
149 - seq_lens,
150 - query_lens=seq_lens,
151 - device=model_runner.device,
152 - pin_memory=model_runner.pin_memory)
153 - assert len(input_tokens) == sum(seq_lens)
154 - assert len(input_positions) == sum(seq_lens)
155 - actual = sampling_metadata.selected_token_indices
156 - expected = torch.tensor(expected_selected_token_indices,
157 - device=actual.device,
158 - dtype=actual.dtype)
159 - torch.testing.assert_close(actual, expected)
160 - torch.allclose(input_tokens, input_positions)
161 -
162 - actual = sampling_metadata.selected_token_indices
163 - expected = torch.tensor(expected_selected_token_indices,
164 - device=actual.device,
165 - dtype=actual.dtype)
166 - torch.testing.assert_close(actual, expected)
167 -
168 -
169 -@pytest.mark.parametrize("batch_size", list(range(1, 257, 3)))
170 -@pytest.mark.parametrize("use_prompt_embeds", [True, False])
171 -def test_prepare_decode_cuda_graph(batch_size, use_prompt_embeds, monkeypatch):
172 - if use_prompt_embeds:
173 - # Prompt Embeddings is only currently supported on V0
174 - monkeypatch.setenv("VLLM_USE_V1", "0")
175 -
176 - model_runner = _create_model_runner(
177 - "facebook/opt-125m",
178 - seed=0,
179 - dtype="float16",
180 - enforce_eager=False,
181 - max_num_batched_tokens=100000,
182 - max_num_seqs=100000,
183 - enable_chunked_prefill=False,
184 - enable_prompt_embeds=True,
185 - )
186 -
187 - context_lens: list[int] = []
188 - seq_group_metadata_list: list[SequenceGroupMetadata] = []
189 - # Assume each seq group finishes prefill.
190 - for i in range(batch_size):
191 - # make sure all tokens fit into one block
192 - context_len = i % (model_runner.block_size - 1) + 1
193 - context_lens.append(context_len)
194 - if use_prompt_embeds:
195 - seq_data = SequenceData.from_seqs(
196 - prompt_token_ids=[0] * context_len,
197 - prompt_embeds=torch.rand(context_len, 10),
198 - )
199 - output_embed = torch.rand(10)
200 - else:
201 - seq_data = SequenceData.from_seqs(
202 - prompt_token_ids=range(context_len))
203 - output_embed = None
204 - seq_data.update_num_computed_tokens(context_len)
205 - # Append one token ID since prefill is finished.
206 - seq_data.append_token_id(1, 0, output_embed)
207 - seq_group_metadata = SequenceGroupMetadata(
208 - request_id=f"test_{i}",
209 - is_prompt=False,
210 - seq_data={0: seq_data},
211 - sampling_params=SamplingParams(temperature=0),
212 - block_tables={0: [1]},
213 - )
214 - assert seq_group_metadata.token_chunk_size == 1
215 - seq_group_metadata_list.append(seq_group_metadata)
216 -
217 - model_input = model_runner._prepare_model_input_tensors(
218 - seq_group_metadata_list)
219 - input_tokens = model_input.input_tokens
220 - input_positions = model_input.input_positions
221 - input_embeds = model_input.inputs_embeds
222 - attn_metadata = model_input.attn_metadata
223 - slot_mapping = attn_metadata.slot_mapping
224 -
225 - assert len(slot_mapping) == len(input_tokens)
226 -
227 - expected_bs = model_runner.vllm_config.pad_for_cudagraph(
228 - len(seq_group_metadata_list))
229 - # Verify input metadata is correct for prompts.
230 - device = model_runner.device
231 - assert attn_metadata.num_prefills == 0
232 - assert attn_metadata.num_prefill_tokens == 0
233 - seq_lens = [context_len + 1 for context_len in context_lens]
234 - # seq_lens are padded to expected_bs
235 - for _ in range(expected_bs - len(seq_lens)):
236 - seq_lens.append(1)
237 - assert attn_metadata.seq_lens == seq_lens
238 - assert attn_metadata.num_decode_tokens == len(seq_lens)
239 - start_idx = 0
240 - start_loc = [start_idx]
241 - for _ in context_lens:
242 - # decode has only 1 token for query.
243 - start_idx += 1
244 - start_loc.append(start_idx)
245 - torch.testing.assert_close(
246 - attn_metadata.query_start_loc,
247 - torch.tensor(start_loc, dtype=torch.int32, device=device))
248 -
249 - start_idx = 0
250 - seq_start_loc = [start_idx]
251 - for seq_len in seq_lens:
252 - start_idx += seq_len
253 - seq_start_loc.append(start_idx)
254 - torch.testing.assert_close(
255 - attn_metadata.seq_start_loc,
256 - torch.tensor(seq_start_loc, dtype=torch.int32, device=device))
257 -
258 - torch.testing.assert_close(
259 - attn_metadata.context_lens_tensor,
260 - torch.tensor(context_lens, dtype=torch.int, device=device))
261 - assert attn_metadata.max_decode_seq_len == max(seq_lens)
262 - torch.testing.assert_close(
263 - attn_metadata.seq_lens_tensor[:len(seq_lens)],
264 - torch.tensor(seq_lens, dtype=torch.int, device=device))
265 -
266 - # block table's first index corresponds to each batch, meaning in
267 - # decoding it is each token.
268 - assert attn_metadata.block_tables.shape[0] == len(input_tokens)
269 - # Block table's second dim corresponds to each token's block number.
270 - # It is padded up to
271 - assert attn_metadata.block_tables.shape[1] == (
272 - model_runner.get_max_block_per_batch())
273 - assert attn_metadata.use_cuda_graph is True
274 -
275 - assert len(input_tokens) == expected_bs
276 - assert len(input_positions) == expected_bs
277 - if use_prompt_embeds:
278 - expected_input_embeds_length = start_loc[-1]
279 - assert len(input_embeds) == expected_input_embeds_length
280 - assert expected_input_embeds_length <= expected_bs
281 - else:
282 - assert input_embeds is None
283 -
284 - # Verify Sampling
285 - expected_selected_token_indices = []
286 - for selected_token_start_idx, _ in enumerate(context_lens):
287 - expected_selected_token_indices.append(selected_token_start_idx)
288 - sampling_metadata = SamplingMetadata.prepare(
289 - seq_group_metadata_list,
290 - seq_lens,
291 - # query lens is all 1 for decode.
292 - query_lens=[1 for _ in range(len(context_lens))],
293 - device=model_runner.device,
294 - pin_memory=model_runner.pin_memory)
295 - actual = sampling_metadata.selected_token_indices
296 - expected = torch.tensor(expected_selected_token_indices,
297 - device=actual.device,
298 - dtype=actual.dtype)
299 - torch.testing.assert_close(actual, expected)
300 -
301 -
302 -def test_empty_seq_group():
303 - """Verify prepare prompt and decode returns empty output."""
304 - model_runner = _create_model_runner(
305 - "facebook/opt-125m",
306 - seed=0,
307 - dtype="float16",
308 - enforce_eager=False,
309 - )
310 - seq_group_metadata_list: list[SequenceGroupMetadata] = []
311 - model_input = model_runner._prepare_model_input_tensors(
312 - seq_group_metadata_list)
313 -
314 - input_tokens = model_input.input_tokens
315 - input_positions = model_input.input_positions
316 - attn_metadata = model_input.attn_metadata
317 -
318 - assert input_tokens is None
319 - assert input_positions is None
320 - assert attn_metadata is None
321 -
322 - model_input = model_runner._prepare_model_input_tensors(
323 - seq_group_metadata_list)
324 -
325 - input_tokens = model_input.input_tokens
326 - input_positions = model_input.input_positions
327 - input_embeds = model_input.inputs_embeds
328 - attn_metadata = model_input.attn_metadata
329 - return_seq_lens = model_input.seq_lens
330 -
331 - assert input_tokens is None
332 - assert input_positions is None
333 - assert input_embeds is None
334 - assert attn_metadata is None
335 - assert return_seq_lens is None
336 -
337 -
338 -@pytest.fixture
339 -def distributed_init():
340 - init_distributed_environment(
341 - world_size=1,
342 - rank=0,
343 - distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
344 - local_rank=0)
345 - ensure_model_parallel_initialized(1, 1)
346 -
347 -
348 -@pytest.mark.parametrize("batch_size", list(range(2, 128, 3)))
349 -@pytest.mark.parametrize("enforce_eager", [True, False])
350 -@pytest.mark.parametrize('use_prompt_embeds', [True, False])
351 -def test_hybrid_batches(batch_size, enforce_eager, use_prompt_embeds,
352 - distributed_init, monkeypatch):
353 - if use_prompt_embeds:
354 - # Prompt Embeddings is only currently supported on V0
355 - monkeypatch.setenv("VLLM_USE_V1", "0")
356 -
357 - model_runner = _create_model_runner(
358 - "facebook/opt-125m",
359 - seed=0,
360 - dtype="float16",
361 - enforce_eager=enforce_eager,
362 - max_num_batched_tokens=100000,
363 - max_num_seqs=100000,
364 - enable_chunked_prefill=True,
365 - enable_prompt_embeds=True,
366 - )
367 -
368 - # Add prefill requests.
369 - seq_lens: list[int] = []
370 - seq_group_metadata_list: list[SequenceGroupMetadata] = []
371 - prefill_metadata_list: list[SequenceGroupMetadata] = []
372 - decode_metadata_list: list[SequenceGroupMetadata] = []
373 - block_tables = {0: [1]}
374 - prefill_batch_size = batch_size // 2
375 - decode_batch_size = batch_size - prefill_batch_size
376 - expected_input_embeds_len = 0
377 - for i in range(prefill_batch_size):
378 - # make sure all tokens fit into one block
379 - seq_len = i % (model_runner.block_size - 1) + 1
380 - seq_lens.append(seq_len)
381 - if use_prompt_embeds:
382 - seq_data = SequenceData.from_seqs(
383 - prompt_token_ids=[0] * seq_len,
384 - prompt_embeds=torch.rand(seq_len, 10),
385 - )
386 - expected_input_embeds_len += seq_len
387 - else:
388 - seq_data = SequenceData.from_seqs(
389 - prompt_token_ids=range(seq_len), )
390 - seq_group_metadata = SequenceGroupMetadata(
391 - request_id=f"test_{i}",
392 - is_prompt=True,
393 - seq_data={0: seq_data},
394 - sampling_params=SamplingParams(temperature=0),
395 - block_tables=block_tables,
396 - )
397 - assert seq_group_metadata.token_chunk_size == seq_data.get_len()
398 - seq_group_metadata_list.append(seq_group_metadata)
399 - prefill_metadata_list.append(seq_group_metadata)
400 -
401 - # Add decode requests
402 - for i in range(prefill_batch_size, batch_size):
403 - # make sure all tokens fit into one block
404 - context_len = i % (model_runner.block_size - 1) + 1
405 - if use_prompt_embeds:
406 - seq_data = SequenceData.from_seqs(
407 - prompt_token_ids=[0] * context_len,
408 - prompt_embeds=torch.rand(context_len, 10),
409 - )
410 - output_embed = torch.rand(10)
411 - # This also iterates the expected input_embeds, because the model
412 - # needs both the input and output embeddings passed into together
413 - expected_input_embeds_len += 1
414 - else:
415 - seq_data = SequenceData.from_seqs(
416 - prompt_token_ids=range(context_len), )
417 - output_embed = None
418 - assert len(seq_data.prompt_token_ids) == context_len
419 - seq_data.append_token_id(1, 0, output_embed)
420 - seq_data.update_num_computed_tokens(context_len)
421 - seq_group_metadata = SequenceGroupMetadata(
422 - request_id=f"test_{i}",
423 - is_prompt=False,
424 - seq_data={0: seq_data},
425 - sampling_params=SamplingParams(temperature=0),
426 - block_tables={0: [1]},
427 - )
428 - assert seq_group_metadata.token_chunk_size == 1
429 - seq_group_metadata_list.append(seq_group_metadata)
430 - decode_metadata_list.append(seq_group_metadata)
431 -
432 - model_input = model_runner.prepare_model_input(seq_group_metadata_list)
433 -
434 - input_tokens = model_input.input_tokens
435 - input_positions = model_input.input_positions
436 - input_embeds = model_input.inputs_embeds
437 - attn_metadata = model_input.attn_metadata
438 -
439 - prefill_meta_actual = attn_metadata.prefill_metadata
440 - decode_meta_actual = attn_metadata.decode_metadata
441 -
442 - assert len(attn_metadata.slot_mapping) == len(input_tokens)
443 - assert len(input_positions) == len(input_tokens)
444 - assert attn_metadata.num_prefills == prefill_batch_size
445 - assert attn_metadata.num_decode_tokens == decode_batch_size
446 - assert attn_metadata.num_prefill_tokens == sum(seq_lens)
447 - if expected_input_embeds_len == 0:
448 - assert input_embeds is None
449 - else:
450 - assert len(input_embeds) == expected_input_embeds_len
451 -
452 - # Verify attn metadata is consistent. We don't need to test individual
453 - # values here because they are tested above.
454 - attn_metadata = model_runner._prepare_model_input_tensors(
455 - seq_group_metadata_list).attn_metadata
456 -
457 - for attr_expected, attr_actual in zip(vars(attn_metadata.prefill_metadata),
458 - vars(prefill_meta_actual)):
459 - assert attr_expected[1] == attr_actual[1]
460 - for attr_expected, attr_actual in zip(vars(attn_metadata.decode_metadata),
461 - vars(decode_meta_actual)):
462 - assert attr_expected[1] == attr_actual[1]
tests/worker/test_profile.py deleted
-68
@@ -1,68 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import torch
5 -
6 -from vllm.engine.arg_utils import EngineArgs
7 -from vllm.utils import get_distributed_init_method, get_ip, get_open_port
8 -from vllm.worker.cache_engine import CacheEngine
9 -from vllm.worker.worker import Worker
10 -
11 -
12 -def test_gpu_memory_profiling():
13 - # Tests the gpu profiling that happens in order to determine the number of
14 - # KV cache blocks that we can allocate on the GPU.
15 - # This test mocks the maximum available gpu memory so that it can run on
16 - # any gpu setup.
17 -
18 - # Set up engine args to build a worker.
19 - engine_args = EngineArgs(model="facebook/opt-125m",
20 - dtype="half",
21 - load_format="dummy")
22 - engine_config = engine_args.create_engine_config()
23 - engine_config.cache_config.num_gpu_blocks = 1000
24 - engine_config.cache_config.num_cpu_blocks = 1000
25 -
26 - # Create the worker.
27 - distributed_init_method = get_distributed_init_method(
28 - get_ip(), get_open_port())
29 - worker = Worker(
30 - vllm_config=engine_config,
31 - local_rank=0,
32 - rank=0,
33 - distributed_init_method=distributed_init_method,
34 - is_driver_worker=True,
35 - )
36 -
37 - # Set 10GiB as the total gpu ram to be device-agnostic
38 - def mock_mem_info():
39 - current_usage = torch.cuda.memory_stats(
40 - )["allocated_bytes.all.current"]
41 - mock_total_bytes = 10 * 1024**3
42 - free = mock_total_bytes - current_usage
43 -
44 - return (free, mock_total_bytes)
45 -
46 - from unittest.mock import patch
47 - with patch("torch.cuda.mem_get_info", side_effect=mock_mem_info):
48 - # Load the model so we can profile it
49 - worker.init_device()
50 - worker.load_model()
51 - gpu_blocks, _ = worker.determine_num_available_blocks()
52 -
53 - # Peak vram usage by torch should be 0.47 GiB
54 - # Model weights take 0.25 GiB
55 - # No memory should be allocated outside of torch
56 - # 9.0 GiB should be the utilization target
57 - # 8.28 GiB should be available for the KV cache
58 - block_size = CacheEngine.get_cache_block_size(
59 - engine_config.cache_config, engine_config.model_config,
60 - engine_config.parallel_config)
61 -
62 - expected_blocks = (8.28 * 1024**3) // block_size
63 -
64 - # Check within a small tolerance for portability
65 - # Hardware, kernel, or dependency changes could all affect memory
66 - # utilization.
67 - # A 100 block tolerance here should be about 60MB of wiggle room.
68 - assert abs(gpu_blocks - expected_blocks) < 100
tests/worker/test_swap.py deleted
-87
@@ -1,87 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import torch
5 -
6 -from vllm.engine.arg_utils import EngineArgs
7 -from vllm.sequence import ExecuteModelRequest
8 -from vllm.utils import get_distributed_init_method, get_ip, get_open_port
9 -from vllm.worker.worker import Worker
10 -
11 -
12 -def test_swap() -> None:
13 - # Configure the engine.
14 - engine_args = EngineArgs(model="distilbert/distilgpt2",
15 - dtype="half",
16 - load_format="dummy")
17 - engine_config = engine_args.create_engine_config()
18 - engine_config.cache_config.num_gpu_blocks = 1000
19 - engine_config.cache_config.num_cpu_blocks = 1000
20 -
21 - # Create the worker.
22 - distributed_init_method = get_distributed_init_method(
23 - get_ip(), get_open_port())
24 - worker = Worker(
25 - vllm_config=engine_config,
26 - local_rank=0,
27 - rank=0,
28 - distributed_init_method=distributed_init_method,
29 - is_driver_worker=True,
30 - )
31 -
32 - # Initialize the worker.
33 - worker.init_device()
34 - worker.load_model()
35 - worker.initialize_cache(
36 - num_gpu_blocks=engine_config.cache_config.num_gpu_blocks,
37 - num_cpu_blocks=engine_config.cache_config.num_cpu_blocks)
38 -
39 - # Randomly initialize the cache.
40 - gpu_cache = worker.cache_engine[0].gpu_cache
41 - cpu_cache = worker.cache_engine[0].cpu_cache
42 - num_layers = len(gpu_cache)
43 - for i in range(num_layers):
44 - gpu_key_cache, gpu_value_cache = gpu_cache[i]
45 - gpu_key_cache.random_()
46 - gpu_value_cache.random_()
47 - cpu_key_cache, cpu_value_cache = cpu_cache[i]
48 - cpu_key_cache.random_()
49 - cpu_value_cache.random_()
50 -
51 - allclose = lambda a, b: torch.allclose(
52 - a.cuda(), b.cuda(), rtol=0.0, atol=0.0)
53 -
54 - # Test swap out.
55 - blocks_to_swap_out = [(3, 72), (56, 35), (84, 34)]
56 - execute_model_req = ExecuteModelRequest(
57 - seq_group_metadata_list=[],
58 - blocks_to_swap_in=[],
59 - blocks_to_swap_out=blocks_to_swap_out,
60 - blocks_to_copy=[],
61 - )
62 - worker.execute_model(execute_model_req=execute_model_req)
63 -
64 - for i in range(num_layers):
65 - gpu_key_cache, gpu_value_cache = gpu_cache[i]
66 - cpu_key_cache, cpu_value_cache = cpu_cache[i]
67 - for src, dst in blocks_to_swap_out:
68 - assert allclose(gpu_key_cache[src], cpu_key_cache[dst])
69 - assert allclose(gpu_value_cache[src], cpu_value_cache[dst])
70 -
71 - # Test swap in.
72 - execute_model_req.blocks_to_swap_out = []
73 - execute_model_req.blocks_to_swap_in = [
74 - (19, 45),
75 - (67, 23),
76 - (12, 78),
77 - (40, 99),
78 - (1, 71),
79 - ]
80 - worker.execute_model(execute_model_req=execute_model_req)
81 -
82 - for i in range(num_layers):
83 - gpu_key_cache, gpu_value_cache = gpu_cache[i]
84 - cpu_key_cache, cpu_value_cache = cpu_cache[i]
85 - for src, dst in execute_model_req.blocks_to_swap_in:
86 - assert allclose(gpu_key_cache[dst], cpu_key_cache[src])
87 - assert allclose(gpu_value_cache[dst], cpu_value_cache[src])