A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V0 Deprecation] Remove `VLLM_USE_V1` from tests (#26341)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 7, 2025 at 23:42 UTC
1e4ecca1d01b9941e621614a062dd4e6a4557c46
51 files changed
+861
-1319
.buildkite/test-pipeline.yaml
+2
-1
@@ -296,6 +296,7 @@ steps:
296
- tests/v1
297
commands:
298
# split the test to avoid interference
299
+ - pytest -v -s -m 'not cpu_test' v1/core
300
- pytest -v -s v1/executor
301
- pytest -v -s v1/kv_offload
302
- pytest -v -s v1/sample
@@ -317,7 +318,7 @@ steps:
318
no_gpu: true
319
commands:
320
# split the test to avoid interference
320
- - pytest -v -s v1/core
321
+ - pytest -v -s -m 'cpu_test' v1/core
322
- pytest -v -s v1/structured_output
323
- pytest -v -s v1/test_serial_utils.py
324
- pytest -v -s -m 'cpu_test' v1/kv_connector/unit
tests/basic_correctness/test_basic_correctness.py
+2
-7
@@ -13,7 +13,7 @@ import pytest
13
import torch
14
15
from vllm import LLM
16
-from vllm.v1.engine.llm_engine import LLMEngine as LLMEngineV1
16
+from vllm.v1.engine.llm_engine import LLMEngine
17
18
from ..conftest import HfRunner, VllmRunner
19
from ..models.utils import check_outputs_equal
@@ -211,16 +211,11 @@ def test_models_distributed(
211
212
213
def test_failed_model_execution(vllm_runner, monkeypatch) -> None:
214
- from vllm.envs import VLLM_USE_V1
215
-
216
- if not VLLM_USE_V1:
217
- pytest.skip("Skipping V0 test, dump input not supported")
218
-
214
# Needed to mock an error in the same process
215
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
216
217
with vllm_runner("facebook/opt-125m", enforce_eager=True) as vllm_model:
223
- if isinstance(vllm_model.llm.llm_engine, LLMEngineV1):
218
+ if isinstance(vllm_model.llm.llm_engine, LLMEngine):
219
v1_test_failed_model_execution(vllm_model)
220
221
tests/basic_correctness/test_cumem.py
+48
-57
@@ -117,68 +117,59 @@ def test_cumem_with_cudagraph():
117
118
@create_new_process_for_each_test()
119
@pytest.mark.parametrize(
120
- "model, use_v1",
120
+ "model",
121
[
122
# sleep mode with safetensors
123
- ("meta-llama/Llama-3.2-1B", True),
123
+ "meta-llama/Llama-3.2-1B",
124
# sleep mode with pytorch checkpoint
125
- ("facebook/opt-125m", True),
125
+ "facebook/opt-125m",
126
],
127
)
128
-def test_end_to_end(monkeypatch: pytest.MonkeyPatch, model: str, use_v1: bool):
129
- with monkeypatch.context() as m:
130
- assert use_v1
131
- m.setenv("VLLM_USE_V1", "1")
132
- free, total = torch.cuda.mem_get_info()
133
- used_bytes_baseline = total - free # in case other process is running
134
- llm = LLM(model, enable_sleep_mode=True)
135
- prompt = "How are you?"
136
- sampling_params = SamplingParams(temperature=0, max_tokens=10)
137
- output = llm.generate(prompt, sampling_params)
138
-
139
- # the benefit of `llm.sleep(level=2)` is mainly CPU memory usage,
140
- # which is difficult to measure in the test. therefore, we only
141
- # test sleep level 1 here.
142
- llm.sleep(level=1)
143
-
144
- free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
145
- used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
146
- # now the memory usage is mostly cudagraph memory pool,
147
- # and it should be less than the model weights (1B model, 2GiB weights)
148
-
149
- # NOTE: In V1, the memory buffer for logits (max_num_reqs x vocab_size)
150
- # is captured but cannot be releasesd from PyTorch due to a known bug,
151
- # therefore high memory usage after `llm.sleep` is called is expected.
152
- # FIXME(youkaichao & ywang96): Fix memory buffer issue with sleep mode
153
- # in V1.
154
- if use_v1:
155
- assert used_bytes < 7 * GiB_bytes
156
- else:
157
- assert used_bytes < 2 * GiB_bytes
158
-
159
- llm.wake_up()
160
- output2 = llm.generate(prompt, sampling_params)
161
- # cmp output
162
- assert output[0].outputs[0].text == output2[0].outputs[0].text
163
-
164
- llm.sleep(level=1)
165
- llm.wake_up(tags=["weights"])
166
-
167
- free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
168
- used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
169
-
170
- # should just reallocate memory for weights (1B model, ~2GiB weights)
171
- if use_v1:
172
- assert used_bytes < 10 * GiB_bytes
173
- else:
174
- assert used_bytes < 6 * GiB_bytes
175
-
176
- # now allocate kv cache memory
177
- llm.wake_up(tags=["kv_cache"])
178
- output3 = llm.generate(prompt, sampling_params)
179
-
180
- # cmp output
181
- assert output[0].outputs[0].text == output3[0].outputs[0].text
128
+def test_end_to_end(model: str):
129
+ free, total = torch.cuda.mem_get_info()
130
+ used_bytes_baseline = total - free # in case other process is running
131
+ llm = LLM(model, enable_sleep_mode=True)
132
+ prompt = "How are you?"
133
+ sampling_params = SamplingParams(temperature=0, max_tokens=10)
134
+ output = llm.generate(prompt, sampling_params)
135
+
136
+ # the benefit of `llm.sleep(level=2)` is mainly CPU memory usage,
137
+ # which is difficult to measure in the test. therefore, we only
138
+ # test sleep level 1 here.
139
+ llm.sleep(level=1)
140
+
141
+ free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
142
+ used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
143
+ # now the memory usage is mostly cudagraph memory pool,
144
+ # and it should be less than the model weights (1B model, 2GiB weights)
145
+
146
+ # NOTE: In V1, the memory buffer for logits (max_num_reqs x vocab_size)
147
+ # is captured but cannot be releasesd from PyTorch due to a known bug,
148
+ # therefore high memory usage after `llm.sleep` is called is expected.
149
+ # FIXME(youkaichao & ywang96): Fix memory buffer issue with sleep mode
150
+ # in V1.
151
+ assert used_bytes < 7 * GiB_bytes
152
+
153
+ llm.wake_up()
154
+ output2 = llm.generate(prompt, sampling_params)
155
+ # cmp output
156
+ assert output[0].outputs[0].text == output2[0].outputs[0].text
157
+
158
+ llm.sleep(level=1)
159
+ llm.wake_up(tags=["weights"])
160
+
161
+ free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
162
+ used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
163
+
164
+ # should just reallocate memory for weights (1B model, ~2GiB weights)
165
+ assert used_bytes < 10 * GiB_bytes
166
+
167
+ # now allocate kv cache memory
168
+ llm.wake_up(tags=["kv_cache"])
169
+ output3 = llm.generate(prompt, sampling_params)
170
+
171
+ # cmp output
172
+ assert output[0].outputs[0].text == output3[0].outputs[0].text
173
174
175
@create_new_process_for_each_test()
tests/compile/piecewise/test_full_cudagraph.py
-2
@@ -66,7 +66,6 @@ def llm_pair(request):
66
pytest.skip("Only Blackwell GPUs support Cutlass MLA")
67
68
env_vars = {
69
- "VLLM_USE_V1": "1",
69
# Force native sampler to avoid potential nondeterminism in FlashInfer
70
# when per-request generators are not used in V1.
71
"VLLM_USE_FLASHINFER_SAMPLER": "0",
@@ -161,7 +160,6 @@ def test_full_cudagraph_with_invalid_backend():
160
with (
161
temporary_environ(
162
{
164
- "VLLM_USE_V1": "1",
163
"VLLM_ATTENTION_BACKEND": "FLEX_ATTENTION",
164
# Flex_Attention is not supported with full cuda graph
165
}
tests/compile/piecewise/test_simple.py
-3
@@ -18,7 +18,6 @@ from vllm.config import (
18
VllmConfig,
19
set_current_vllm_config,
20
)
21
-from vllm.envs import VLLM_USE_V1
21
from vllm.forward_context import BatchDescriptor, set_forward_context
22
from vllm.utils import is_torch_equal_or_newer
23
@@ -127,7 +126,6 @@ def _run_simple_model(
126
@pytest.mark.parametrize("use_inductor", [True, False])
127
@torch.inference_mode()
128
def test_simple_piecewise_compile(use_inductor):
130
- assert VLLM_USE_V1
129
_run_simple_model(
130
splitting_ops=["silly.attention"],
131
use_inductor_graph_partition=False,
@@ -146,7 +144,6 @@ def test_simple_piecewise_compile(use_inductor):
144
@torch.inference_mode()
145
@pytest.mark.parametrize("splitting_ops", [["silly.attention"], []])
146
def test_simple_inductor_graph_partition(splitting_ops):
149
- assert VLLM_USE_V1
147
if not is_torch_equal_or_newer("2.9.0.dev"):
148
pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
149
tests/compile/test_async_tp.py
+1
-7
@@ -388,10 +388,6 @@ def test_async_tp_pass_correctness(
388
"pass_config": {"enable_async_tp": async_tp_enabled},
389
}
390
391
- async_tp_env = tp_env = {
392
- "VLLM_USE_V1": "1",
393
- }
394
-
391
async_tp_args = [
392
*common_args,
393
"--tensor-parallel-size",
@@ -410,6 +406,4 @@ def test_async_tp_pass_correctness(
406
"mp",
407
]
408
413
- compare_two_settings(
414
- model_id, async_tp_args, tp_args, async_tp_env, tp_env, method="generate"
415
- )
409
+ compare_two_settings(model_id, async_tp_args, tp_args, method="generate")
tests/compile/test_config.py
+1
-11
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import pytest
4
5
-import vllm
5
from vllm.compilation.counter import compilation_counter
6
from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig
7
from vllm.utils import _is_torch_equal_or_newer
@@ -16,15 +15,10 @@ def test_version():
15
assert not _is_torch_equal_or_newer("2.7.1", "2.8.0.dev")
16
17
19
-def test_use_cudagraphs_dynamic(monkeypatch):
20
- assert vllm.envs.VLLM_USE_V1
18
+def test_use_cudagraphs_dynamic():
19
vllm_config = VllmConfig()
20
assert vllm_config.compilation_config.use_cudagraph
21
24
- monkeypatch.setenv("VLLM_USE_V1", "0")
25
- vllm_config = VllmConfig()
26
- assert not vllm_config.compilation_config.use_cudagraph
27
-
22
23
def test_custom_op():
24
# proper syntax
@@ -41,8 +35,6 @@ def test_custom_op():
35
# may be influenced by other tests.
36
@pytest.mark.parametrize("val", ["1"])
37
def test_VLLM_DISABLE_COMPILE_CACHE(vllm_runner, monkeypatch, val):
44
- assert vllm.envs.VLLM_USE_V1
45
-
38
# Disable multiprocessing so that the counter is in the same process
39
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
40
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", val)
@@ -68,8 +60,6 @@ def test_VLLM_DISABLE_COMPILE_CACHE(vllm_runner, monkeypatch, val):
60
@pytest.mark.forked
61
@pytest.mark.parametrize("enabled", [True, False])
62
def test_use_cudagraphs(vllm_runner, monkeypatch, enabled):
71
- assert vllm.envs.VLLM_USE_V1
72
-
63
# Disable multiprocessing so that the counter is in the same process
64
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
65
tests/compile/test_fusion_attn.py
-3
@@ -303,7 +303,6 @@ def test_attention_quant_pattern(
303
model_class: type[AttentionQuantPatternModel],
304
backend: _Backend,
305
use_inductor_graph_partition: bool,
306
- monkeypatch,
306
dist_init,
307
caplog_vllm,
308
):
@@ -312,8 +311,6 @@ def test_attention_quant_pattern(
311
if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
312
pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
313
315
- monkeypatch.setenv("VLLM_USE_V1", "1")
316
-
314
device = torch.device("cuda:0")
315
torch.manual_seed(42)
316
tests/config/test_mp_reducer.py
+1
-4
@@ -8,16 +8,13 @@ from vllm.engine.arg_utils import AsyncEngineArgs
8
from vllm.v1.engine.async_llm import AsyncLLM
9
10
11
-def test_mp_reducer(monkeypatch):
11
+def test_mp_reducer():
12
"""
13
Test that _reduce_config reducer is registered when AsyncLLM is instantiated
14
without transformers_modules. This is a regression test for
15
https://github.com/vllm-project/vllm/pull/18640.
16
"""
17
18
- # Use V1 AsyncLLM which calls maybe_register_config_serialize_by_value
19
- monkeypatch.setenv("VLLM_USE_V1", "1")
20
-
18
# Ensure transformers_modules is not in sys.modules
19
if "transformers_modules" in sys.modules:
20
del sys.modules["transformers_modules"]
tests/detokenizer/test_stop_strings.py
+2
-4
@@ -5,7 +5,7 @@ from typing import Any, Optional
5
6
import pytest
7
8
-from vllm import LLM, SamplingParams, envs
8
+from vllm import LLM, SamplingParams
9
10
MODEL = "meta-llama/llama-2-7b-hf"
11
MAX_TOKENS = 200
@@ -111,9 +111,7 @@ def _stop_token_id(llm):
111
112
@pytest.mark.skip_global_cleanup
113
def test_stop_strings():
114
- # If V0, must set enforce_eager=False since we use
115
- # async output processing below.
116
- llm = LLM(MODEL, enforce_eager=envs.VLLM_USE_V1)
114
+ llm = LLM(MODEL, enforce_eager=True)
115
116
_stop_basic(llm)
117
_stop_multi_tokens(llm)
tests/distributed/test_context_parallel.py
+8
-45
@@ -42,24 +42,10 @@ class CPTestOptions(NamedTuple):
42
@dataclass
43
class CPTestSettings:
44
parallel_setups: list[ParallelSetup]
45
- # NOTE: the length of distributed_backends and
46
- # vllm_major_versions should be the same, and they
47
- # are first zipped together to iterate over all
48
- # test settings.
45
distributed_backends: list[str]
50
- # vllm major version: "0" for V0, "1" for V1
51
- vllm_major_versions: list[str]
46
runner: RunnerOption
47
test_options: CPTestOptions
48
55
- def __post_init__(self):
56
- if len(self.distributed_backends) != len(self.vllm_major_versions):
57
- raise ValueError(
58
- f"Length mismatch: distributed_backends "
59
- f"({len(self.distributed_backends)}) != "
60
- f"vllm_major_versions ({len(self.vllm_major_versions)})"
61
- )
62
-
49
@staticmethod
50
def detailed(
51
*,
@@ -87,7 +73,6 @@ class CPTestSettings:
73
return CPTestSettings(
74
parallel_setups=parallel_setups,
75
distributed_backends=["mp"],
90
- vllm_major_versions=["1"],
76
runner=runner,
77
test_options=CPTestOptions(
78
multi_node_only=multi_node_only, load_format=load_format
@@ -98,14 +83,11 @@ class CPTestSettings:
83
opts = self.test_options
84
85
for parallel_setup in self.parallel_setups:
101
- for backend, vllm_major_version in zip(
102
- self.distributed_backends, self.vllm_major_versions
103
- ):
86
+ for backend in self.distributed_backends:
87
yield (
88
model_id,
89
parallel_setup,
90
backend,
108
- vllm_major_version,
91
self.runner,
92
opts,
93
)
@@ -115,7 +97,6 @@ def _compare_cp_with_tp(
97
model_id: str,
98
parallel_setup: ParallelSetup,
99
distributed_backend: str,
118
- vllm_major_version: str,
100
runner: RunnerOption,
101
test_options: CPTestOptions,
102
num_gpus_available: int,
@@ -191,10 +172,6 @@ def _compare_cp_with_tp(
172
if hf_overrides:
173
common_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
174
194
- cp_env = tp_env = {
195
- "VLLM_USE_V1": vllm_major_version, # Note(hc): DCP only support V1 engine only
196
- }
197
-
175
cp_args = [
176
*common_args,
177
"--tensor-parallel-size",
@@ -217,24 +194,13 @@ def _compare_cp_with_tp(
194
distributed_backend,
195
]
196
220
- try:
221
- compare_two_settings(
222
- model_id,
223
- cp_args,
224
- tp_args,
225
- cp_env,
226
- tp_env,
227
- method=method,
228
- max_wait_seconds=720,
229
- )
230
- except Exception:
231
- testing_ray_compiled_graph = cp_env is not None
232
- if testing_ray_compiled_graph and vllm_major_version == "0":
233
- # Ray Compiled Graph tests are flaky for V0,
234
- # so we don't want to fail the test
235
- logger.exception("Ray Compiled Graph tests failed")
236
- else:
237
- raise
197
+ compare_two_settings(
198
+ model_id,
199
+ cp_args,
200
+ tp_args,
201
+ method=method,
202
+ max_wait_seconds=720,
203
+ )
204
205
206
CP_TEXT_GENERATION_MODELS = {
@@ -257,7 +223,6 @@ CP_TEST_MODELS = [
223
"model_id",
224
"parallel_setup",
225
"distributed_backend",
260
- "vllm_major_version",
226
"runner",
227
"test_options",
228
),
@@ -274,7 +239,6 @@ def test_cp_generation(
239
model_id: str,
240
parallel_setup: ParallelSetup,
241
distributed_backend: str,
277
- vllm_major_version: str,
242
runner: RunnerOption,
243
test_options: CPTestOptions,
244
num_gpus_available,
@@ -283,7 +247,6 @@ def test_cp_generation(
247
model_id,
248
parallel_setup,
249
distributed_backend,
286
- vllm_major_version,
250
runner,
251
test_options,
252
num_gpus_available,
tests/distributed/test_pipeline_parallel.py
+2
-7
@@ -307,7 +307,6 @@ def _compare_tp(
307
if distributed_backend == "ray":
308
# For V1, test Ray Compiled Graph for all the tests
309
pp_env = {
310
- "VLLM_USE_V1": "1",
310
"VLLM_USE_RAY_COMPILED_DAG": "1",
311
"VLLM_USE_RAY_SPMD_WORKER": "1",
312
"VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL": "1",
@@ -316,15 +315,11 @@ def _compare_tp(
315
# terminate because of a Ray Compiled Graph issue.
316
common_args.append("--disable-frontend-multiprocessing")
317
elif distributed_backend == "mp":
319
- pp_env = {
320
- "VLLM_USE_V1": "1",
321
- }
318
+ pp_env = None
319
else:
320
pp_env = None
321
325
- tp_env = {
326
- "VLLM_USE_V1": "1",
327
- }
322
+ tp_env = None
323
324
pp_args = [
325
*common_args,
tests/distributed/test_sequence_parallel.py
+2
-44
@@ -42,24 +42,10 @@ class SPTestOptions(NamedTuple):
42
@dataclass
43
class SPTestSettings:
44
parallel_setups: list[ParallelSetup]
45
- # NOTE: the length of distributed_backends and
46
- # vllm_major_versions should be the same, and they
47
- # are first zipped together to iterate over all
48
- # test settings.
45
distributed_backends: list[str]
50
- # vllm major version: "0" for V0, "1" for V1
51
- vllm_major_versions: list[str]
46
runner: RunnerOption
47
test_options: SPTestOptions
48
55
- def __post_init__(self):
56
- if len(self.distributed_backends) != len(self.vllm_major_versions):
57
- raise ValueError(
58
- f"Length mismatch: distributed_backends "
59
- f"({len(self.distributed_backends)}) != "
60
- f"vllm_major_versions ({len(self.vllm_major_versions)})"
61
- )
62
-
49
@staticmethod
50
def detailed(
51
*,
@@ -85,7 +71,6 @@ class SPTestSettings:
71
return SPTestSettings(
72
parallel_setups=parallel_setups,
73
distributed_backends=["mp", "ray"],
88
- vllm_major_versions=["1", "1"],
74
runner=runner,
75
test_options=SPTestOptions(
76
multi_node_only=multi_node_only, load_format=load_format
@@ -117,7 +102,6 @@ class SPTestSettings:
102
return SPTestSettings(
103
parallel_setups=parallel_setups,
104
distributed_backends=["mp", "ray"],
120
- vllm_major_versions=["1", "1"],
105
runner=runner,
106
test_options=SPTestOptions(
107
multi_node_only=multi_node_only, load_format=load_format
@@ -147,7 +131,6 @@ class SPTestSettings:
131
return SPTestSettings(
132
parallel_setups=parallel_setups,
133
distributed_backends=["mp", "ray"],
150
- vllm_major_versions=["1", "1"],
134
runner=runner,
135
test_options=SPTestOptions(
136
multi_node_only=multi_node_only, load_format=load_format
@@ -158,14 +141,11 @@ class SPTestSettings:
141
opts = self.test_options
142
143
for parallel_setup in self.parallel_setups:
161
- for backend, vllm_major_version in zip(
162
- self.distributed_backends, self.vllm_major_versions
163
- ):
144
+ for backend in self.distributed_backends:
145
yield (
146
model_id,
147
parallel_setup,
148
backend,
168
- vllm_major_version,
149
self.runner,
150
opts,
151
)
@@ -175,7 +155,6 @@ def _compare_sp(
155
model_id: str,
156
parallel_setup: ParallelSetup,
157
distributed_backend: str,
178
- vllm_major_version: str,
158
runner: RunnerOption,
159
test_options: SPTestOptions,
160
num_gpus_available: int,
@@ -265,10 +244,6 @@ def _compare_sp(
244
},
245
}
246
268
- tp_sp_env = tp_env = {
269
- "VLLM_USE_V1": vllm_major_version,
270
- }
271
-
247
tp_sp_args = [
248
*common_args,
249
"--tensor-parallel-size",
@@ -281,9 +256,6 @@ def _compare_sp(
256
json.dumps(compilation_config),
257
]
258
284
- tp_env = {
285
- "VLLM_USE_V1": vllm_major_version,
286
- }
259
tp_args = [
260
*common_args,
261
"--tensor-parallel-size",
@@ -292,18 +264,7 @@ def _compare_sp(
264
"mp",
265
]
266
295
- try:
296
- compare_two_settings(
297
- model_id, tp_sp_args, tp_args, tp_sp_env, tp_env, method=method
298
- )
299
- except Exception:
300
- testing_ray_compiled_graph = tp_sp_env is not None
301
- if testing_ray_compiled_graph and vllm_major_version == "0":
302
- # Ray Compiled Graph tests are flaky for V0,
303
- # so we don't want to fail the test
304
- logger.exception("Ray Compiled Graph tests failed")
305
- else:
306
- raise
267
+ compare_two_settings(model_id, tp_sp_args, tp_args, method=method)
268
269
270
SP_TEXT_GENERATION_MODELS = {
@@ -325,7 +286,6 @@ SP_TEST_MODELS = [
286
"model_id",
287
"parallel_setup",
288
"distributed_backend",
328
- "vllm_major_version",
289
"runner",
290
"test_options",
291
),
@@ -341,7 +301,6 @@ def test_tp_sp_generation(
301
model_id: str,
302
parallel_setup: ParallelSetup,
303
distributed_backend: str,
344
- vllm_major_version: str,
304
runner: RunnerOption,
305
test_options: SPTestOptions,
306
num_gpus_available,
@@ -350,7 +309,6 @@ def test_tp_sp_generation(
309
model_id,
310
parallel_setup,
311
distributed_backend,
353
- vllm_major_version,
312
runner,
313
test_options,
314
num_gpus_available,
tests/entrypoints/llm/test_accuracy.py
+18
-34
@@ -61,50 +61,34 @@ def run_test(model_name, more_args=None):
61
TPU_TP_TEST_STR = "" # "tensor_parallel_size=4"
62
63
64
-@pytest.mark.skipif(
65
- not current_platform.is_cuda() and not current_platform.is_tpu(),
66
- reason="V1 is currently only supported on CUDA and TPU",
67
-)
64
@pytest.mark.parametrize("model", MODEL_NAMES)
69
-def test_lm_eval_accuracy_v1_engine(model, monkeypatch: pytest.MonkeyPatch):
65
+def test_lm_eval_accuracy_v1_engine(model):
66
"""Run with the V1 Engine."""
67
72
- with monkeypatch.context() as m:
73
- m.setenv("VLLM_USE_V1", "1")
68
+ more_args = None
69
+ if current_platform.is_tpu():
70
+ # Limit compilation time for TPU V1
71
75
- more_args = None
76
- if current_platform.is_tpu():
77
- # Limit compilation time for TPU V1
72
+ more_args = "max_model_len=2048,max_num_seqs=64"
73
79
- more_args = "max_model_len=2048,max_num_seqs=64"
74
+ # Add TP test (if provided)
75
+ if TPU_TP_TEST_STR:
76
+ more_args += ",{}".format(TPU_TP_TEST_STR)
77
81
- # Add TP test (if provided)
82
- if TPU_TP_TEST_STR:
83
- more_args += ",{}".format(TPU_TP_TEST_STR)
78
+ run_test(model, more_args)
79
85
- run_test(model, more_args)
80
87
-
88
-@pytest.mark.skipif(
89
- not current_platform.is_cuda() and not current_platform.is_tpu(),
90
- reason="V1 is currently only supported on CUDA and TPU",
91
-)
81
@pytest.mark.parametrize("model", FP8_KV_MODEL_NAMES)
93
-def test_lm_eval_accuracy_v1_engine_fp8_kv_cache(
94
- model, monkeypatch: pytest.MonkeyPatch
95
-):
82
+def test_lm_eval_accuracy_v1_engine_fp8_kv_cache(model):
83
"""Run with the V1 Engine."""
84
98
- with monkeypatch.context() as m:
99
- m.setenv("VLLM_USE_V1", "1")
100
-
101
- more_args = None
102
- if current_platform.is_tpu():
103
- # Limit compilation time for TPU V1
104
- more_args = "max_model_len=2048,max_num_seqs=128,kv_cache_dtype=fp8"
85
+ more_args = None
86
+ if current_platform.is_tpu():
87
+ # Limit compilation time for TPU V1
88
+ more_args = "max_model_len=2048,max_num_seqs=128,kv_cache_dtype=fp8"
89
106
- # Add TP test (if provided)
107
- if TPU_TP_TEST_STR:
108
- more_args += ",{}".format(TPU_TP_TEST_STR)
90
+ # Add TP test (if provided)
91
+ if TPU_TP_TEST_STR:
92
+ more_args += ",{}".format(TPU_TP_TEST_STR)
93
110
- run_test(model, more_args)
94
+ run_test(model, more_args)
tests/entrypoints/openai/correctness/test_lmeval.py
+6
-15
@@ -10,7 +10,6 @@ AsyncLLMEngine are working correctly.
10
"""
11
12
import lm_eval
13
-import pytest
13
14
from vllm.platforms import current_platform
15
@@ -67,21 +66,13 @@ def run_test(more_args):
66
), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
67
68
70
-@pytest.mark.skipif(
71
- not current_platform.is_cuda()
72
- and not current_platform.is_tpu()
73
- and not current_platform.is_xpu(),
74
- reason="V1 currently only supported on CUDA, XPU and TPU",
75
-)
76
-def test_lm_eval_accuracy_v1_engine(monkeypatch: pytest.MonkeyPatch):
69
+def test_lm_eval_accuracy_v1_engine():
70
"""Run with the V1 Engine."""
71
79
- with monkeypatch.context() as m:
80
- m.setenv("VLLM_USE_V1", "1")
81
- more_args = []
72
+ more_args = []
73
83
- # Limit compilation time for V1
84
- if current_platform.is_tpu():
85
- more_args = ["--max-num-seqs", "64"]
74
+ # Limit compilation time for V1
75
+ if current_platform.is_tpu():
76
+ more_args = ["--max-num-seqs", "64"]
77
87
- run_test(more_args)
78
+ run_test(more_args)
tests/entrypoints/openai/test_chat.py
+1
-12
@@ -21,18 +21,7 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
21
22
23
@pytest.fixture(scope="module")
24
-def monkeypatch_module():
25
- from _pytest.monkeypatch import MonkeyPatch
26
-
27
- mpatch = MonkeyPatch()
28
- yield mpatch
29
- mpatch.undo()
30
-
31
-
32
-@pytest.fixture(scope="module")
33
-def server(monkeypatch_module, zephyr_lora_files): # noqa: F811
34
- monkeypatch_module.setenv("VLLM_USE_V1", "1")
35
-
24
+def server(zephyr_lora_files): # noqa: F811
25
args = [
26
# use half precision for speed and memory savings in CI environment
27
"--dtype",
tests/entrypoints/openai/test_lora_adapters.py
+1
-14
@@ -37,21 +37,8 @@ BADREQUEST_CASES = [
37
]
38
39
40
-@pytest.fixture(scope="module")
41
-def monkeypatch_module():
42
- from _pytest.monkeypatch import MonkeyPatch
43
-
44
- mpatch = MonkeyPatch()
45
- yield mpatch
46
- mpatch.undo()
47
-
48
-
40
@pytest.fixture(scope="module", params=[True])
50
-def server_with_lora_modules_json(request, monkeypatch_module, zephyr_lora_files):
51
- use_v1 = request.param
52
- assert use_v1
53
- monkeypatch_module.setenv("VLLM_USE_V1", "1")
54
-
41
+def server_with_lora_modules_json(request, zephyr_lora_files):
42
# Define the json format LoRA module configurations
43
lora_module_1 = {
44
"name": "zephyr-lora",
tests/entrypoints/openai/test_metrics.py
+21
-94
@@ -22,24 +22,6 @@ MODEL_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
22
PREV_MINOR_VERSION = version._prev_minor_version()
23
24
25
-@pytest.fixture(scope="module", params=[True])
26
-def use_v1(request):
27
- # Module-scoped variant of run_with_both_engines
28
- #
29
- # Use this fixture to run a test with both v0 and v1, and
30
- # also to conditionalize the test logic e.g.
31
- #
32
- # def test_metrics_exist(use_v1, server, client):
33
- # ...
34
- # expected = EXPECTED_V1_METRICS if use_v1 else EXPECTED_METRICS
35
- # for metric in expected:
36
- # assert metric in response.text
37
- #
38
- # @skip_v1 wouldn't work here because this is a module-level
39
- # fixture - per-function decorators would have no effect
40
- yield request.param
41
-
42
-
25
@pytest.fixture(scope="module")
26
def default_server_args():
27
return [
@@ -63,13 +45,11 @@ def default_server_args():
45
f"--show-hidden-metrics-for-version={PREV_MINOR_VERSION}",
46
],
47
)
66
-def server(use_v1, default_server_args, request):
48
+def server(default_server_args, request):
49
if request.param:
50
default_server_args.append(request.param)
69
- env_dict = dict(VLLM_USE_V1="1" if use_v1 else "0")
70
- with RemoteOpenAIServer(
71
- MODEL_NAME, default_server_args, env_dict=env_dict
72
- ) as remote_server:
51
+
52
+ with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
53
yield remote_server
54
55
@@ -129,7 +109,8 @@ EXPECTED_VALUES = {
109
110
@pytest.mark.asyncio
111
async def test_metrics_counts(
132
- server: RemoteOpenAIServer, client: openai.AsyncClient, use_v1: bool
112
+ server: RemoteOpenAIServer,
113
+ client: openai.AsyncClient,
114
):
115
for _ in range(_NUM_REQUESTS):
116
# sending a request triggers the metrics to be logged.
@@ -145,7 +126,7 @@ async def test_metrics_counts(
126
127
# Loop over all expected metric_families
128
for metric_family, suffix_values_list in EXPECTED_VALUES.items():
148
- if (use_v1 and metric_family not in EXPECTED_METRICS_V1) or (
129
+ if (metric_family not in EXPECTED_METRICS_V1) or (
130
not server.show_hidden_metrics
131
and metric_family in HIDDEN_DEPRECATED_METRICS
132
):
@@ -183,62 +164,6 @@ async def test_metrics_counts(
164
assert found_metric, f"Did not find {metric_family} in prom endpoint"
165
166
186
-EXPECTED_METRICS = [
187
- "vllm:num_requests_running",
188
- "vllm:num_requests_waiting",
189
- "vllm:gpu_cache_usage_perc",
190
- "vllm:time_to_first_token_seconds_sum",
191
- "vllm:time_to_first_token_seconds_bucket",
192
- "vllm:time_to_first_token_seconds_count",
193
- "vllm:time_per_output_token_seconds_sum",
194
- "vllm:time_per_output_token_seconds_bucket",
195
- "vllm:time_per_output_token_seconds_count",
196
- "vllm:e2e_request_latency_seconds_sum",
197
- "vllm:e2e_request_latency_seconds_bucket",
198
- "vllm:e2e_request_latency_seconds_count",
199
- "vllm:request_queue_time_seconds_sum",
200
- "vllm:request_queue_time_seconds_bucket",
201
- "vllm:request_queue_time_seconds_count",
202
- "vllm:request_inference_time_seconds_sum",
203
- "vllm:request_inference_time_seconds_bucket",
204
- "vllm:request_inference_time_seconds_count",
205
- "vllm:request_prefill_time_seconds_sum",
206
- "vllm:request_prefill_time_seconds_bucket",
207
- "vllm:request_prefill_time_seconds_count",
208
- "vllm:request_decode_time_seconds_sum",
209
- "vllm:request_decode_time_seconds_bucket",
210
- "vllm:request_decode_time_seconds_count",
211
- "vllm:request_prompt_tokens_sum",
212
- "vllm:request_prompt_tokens_bucket",
213
- "vllm:request_prompt_tokens_count",
214
- "vllm:request_generation_tokens_sum",
215
- "vllm:request_generation_tokens_bucket",
216
- "vllm:request_generation_tokens_count",
217
- "vllm:request_params_n_sum",
218
- "vllm:request_params_n_bucket",
219
- "vllm:request_params_n_count",
220
- "vllm:request_params_max_tokens_sum",
221
- "vllm:request_params_max_tokens_bucket",
222
- "vllm:request_params_max_tokens_count",
223
- "vllm:iteration_tokens_total",
224
- "vllm:num_preemptions_total",
225
- "vllm:prompt_tokens_total",
226
- "vllm:generation_tokens_total",
227
- "vllm:request_success_total",
228
- "vllm:cache_config_info",
229
- # labels in cache_config_info
230
- "block_size",
231
- "cache_dtype",
232
- "cpu_offload_gb",
233
- "enable_prefix_caching",
234
- "gpu_memory_utilization",
235
- "num_cpu_blocks",
236
- "num_gpu_blocks",
237
- "num_gpu_blocks_override",
238
- "sliding_window",
239
- "swap_space_bytes",
240
-]
241
-
167
EXPECTED_METRICS_V1 = [
168
"vllm:num_requests_running",
169
"vllm:num_requests_waiting",
@@ -304,17 +229,21 @@ HIDDEN_DEPRECATED_METRICS: list[str] = [
229
230
@pytest.mark.asyncio
231
async def test_metrics_exist(
307
- server: RemoteOpenAIServer, client: openai.AsyncClient, use_v1: bool
232
+ server: RemoteOpenAIServer,
233
+ client: openai.AsyncClient,
234
):
235
# sending a request triggers the metrics to be logged.
236
await client.completions.create(
311
- model=MODEL_NAME, prompt="Hello, my name is", max_tokens=5, temperature=0.0
237
+ model=MODEL_NAME,
238
+ prompt="Hello, my name is",
239
+ max_tokens=5,
240
+ temperature=0.0,
241
)
242
243
response = requests.get(server.url_for("metrics"))
244
assert response.status_code == HTTPStatus.OK
245
317
- for metric in EXPECTED_METRICS_V1 if use_v1 else EXPECTED_METRICS:
246
+ for metric in EXPECTED_METRICS_V1:
247
if metric in HIDDEN_DEPRECATED_METRICS and not server.show_hidden_metrics:
248
continue
249
assert metric in response.text
@@ -322,10 +251,11 @@ async def test_metrics_exist(
251
252
@pytest.mark.asyncio
253
async def test_abort_metrics_reset(
325
- server: RemoteOpenAIServer, client: openai.AsyncClient, use_v1: bool
254
+ server: RemoteOpenAIServer,
255
+ client: openai.AsyncClient,
256
):
257
running_requests, waiting_requests, kv_cache_usage = _get_running_metrics_from_api(
328
- server, use_v1
258
+ server
259
)
260
261
# Expect no running requests or kvcache usage
@@ -351,7 +281,7 @@ async def test_abort_metrics_reset(
281
282
# Check that we have running requests
283
running_requests, waiting_requests, kv_cache_usage = _get_running_metrics_from_api(
354
- server, use_v1
284
+ server
285
)
286
287
# Expect running requests and kvcache usage
@@ -371,7 +301,7 @@ async def test_abort_metrics_reset(
301
302
# Verify running and waiting requests counts and KV cache usage are zero
303
running_requests_after, waiting_requests_after, kv_cache_usage_after = (
374
- _get_running_metrics_from_api(server, use_v1)
304
+ _get_running_metrics_from_api(server)
305
)
306
307
assert running_requests_after == 0, (
@@ -385,7 +315,7 @@ async def test_abort_metrics_reset(
315
)
316
317
388
-def _get_running_metrics_from_api(server: RemoteOpenAIServer, use_v1: bool):
318
+def _get_running_metrics_from_api(server: RemoteOpenAIServer):
319
"""Return (running_count, waiting_count, kv_cache_usage)"""
320
321
response = requests.get(server.url_for("metrics"))
@@ -394,9 +324,7 @@ def _get_running_metrics_from_api(server: RemoteOpenAIServer, use_v1: bool):
324
# Verify running and waiting requests counts and KV cache usage are zero
325
running_requests, waiting_requests, kv_cache_usage = None, None, None
326
397
- kv_cache_usage_metric = (
398
- "vllm:kv_cache_usage_perc" if use_v1 else "vllm:gpu_cache_usage_perc"
399
- )
327
+ kv_cache_usage_metric = "vllm:kv_cache_usage_perc"
328
329
for family in text_string_to_metric_families(response.text):
330
if family.name == "vllm:num_requests_running":
@@ -422,7 +350,7 @@ def _get_running_metrics_from_api(server: RemoteOpenAIServer, use_v1: bool):
350
return running_requests, waiting_requests, kv_cache_usage
351
352
425
-def test_metrics_exist_run_batch(use_v1: bool):
353
+def test_metrics_exist_run_batch():
354
input_batch = """{"custom_id": "request-0", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/multilingual-e5-small", "input": "You are a helpful assistant."}}""" # noqa: E501
355
356
base_url = "0.0.0.0"
@@ -452,7 +380,6 @@ def test_metrics_exist_run_batch(use_v1: bool):
380
"--port",
381
port,
382
],
455
- env={"VLLM_USE_V1": "1"},
383
)
384
385
def is_server_up(url):
tests/entrypoints/openai/test_prompt_validation.py
-5
@@ -15,11 +15,6 @@ from vllm.entrypoints.renderer import BaseRenderer
15
from ...utils import RemoteOpenAIServer
16
17
18
-@pytest.fixture(scope="function", autouse=True)
19
-def use_v1_only(monkeypatch):
20
- monkeypatch.setenv("VLLM_USE_V1", "1")
21
-
22
-
18
@pytest.mark.asyncio
19
async def test_empty_prompt():
20
model_name = "gpt2"
tests/kernels/attention/test_attention_selector.py
+9
-20
@@ -80,7 +80,6 @@ def test_env(
80
):
81
"""Test attention backend selection with valid device-backend pairs."""
82
with monkeypatch.context() as m:
83
- m.setenv("VLLM_USE_V1", "1")
83
m.setenv(STR_BACKEND_ENV_VAR, name)
84
m.setenv("VLLM_MLA_DISABLE", "1" if use_mla else "0")
85
@@ -212,30 +211,21 @@ def test_env(
211
212
213
@pytest.mark.parametrize("device", ["cpu", "cuda"])
215
-def test_fp32_fallback(
216
- device: str,
217
- monkeypatch: pytest.MonkeyPatch,
218
-):
214
+def test_fp32_fallback(device: str):
215
"""Test attention backend selection with fp32."""
220
- with monkeypatch.context() as m:
221
- m.setenv("VLLM_USE_V1", "1")
216
+ if device == "cpu":
217
+ with patch("vllm.attention.selector.current_platform", CpuPlatform()):
218
+ backend = get_attn_backend(16, torch.float32, None, 16)
219
+ assert backend.get_name() == "TORCH_SDPA"
220
223
- if device == "cpu":
224
- with patch("vllm.attention.selector.current_platform", CpuPlatform()):
225
- backend = get_attn_backend(16, torch.float32, None, 16)
226
- assert backend.get_name() == "TORCH_SDPA"
227
-
228
- elif device == "cuda":
229
- with patch("vllm.attention.selector.current_platform", CudaPlatform()):
230
- backend = get_attn_backend(16, torch.float32, None, 16)
231
- assert backend.get_name() == "FLEX_ATTENTION"
221
+ elif device == "cuda":
222
+ with patch("vllm.attention.selector.current_platform", CudaPlatform()):
223
+ backend = get_attn_backend(16, torch.float32, None, 16)
224
+ assert backend.get_name() == "FLEX_ATTENTION"
225
226
227
def test_flash_attn(monkeypatch: pytest.MonkeyPatch):
228
"""Test FlashAttn validation."""
236
- # TODO: When testing for v1, pipe in `use_v1` as an argument to
237
- # get_attn_backend
238
-
229
pytest.skip(
230
"Skipping as current backend selector does not "
231
"handle fallbacks when a backend is set via env var."
@@ -289,7 +279,6 @@ def test_invalid_env(monkeypatch: pytest.MonkeyPatch):
279
monkeypatch.context() as m,
280
patch("vllm.attention.selector.current_platform", CudaPlatform()),
281
):
292
- m.setenv("VLLM_USE_V1", "1")
282
m.setenv(STR_BACKEND_ENV_VAR, STR_INVALID_VAL)
283
284
# Should raise ValueError for invalid backend
tests/kernels/test_flex_attention.py
+6
-8
@@ -55,7 +55,6 @@ def test_flex_attention_vs_default_backend(vllm_runner, monkeypatch):
55
56
# Run with flex attention
57
with monkeypatch.context() as m:
58
- m.setenv("VLLM_USE_V1", "1")
58
m.setenv("VLLM_ATTENTION_BACKEND", "FLEX_ATTENTION")
59
60
set_seed(seed)
@@ -72,7 +71,6 @@ def test_flex_attention_vs_default_backend(vllm_runner, monkeypatch):
71
72
# Run with default backend
73
with monkeypatch.context() as m:
75
- m.setenv("VLLM_USE_V1", "1")
74
set_seed(seed)
75
with vllm_runner(
76
model_name,
@@ -113,7 +111,6 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner, monkeypatch):
111
112
# Run with flex attention
113
with monkeypatch.context() as m:
116
- m.setenv("VLLM_USE_V1", "1")
114
m.setenv("VLLM_ATTENTION_BACKEND", "FLEX_ATTENTION")
115
with vllm_runner(
116
model_name,
@@ -126,17 +123,18 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner, monkeypatch):
123
flex_outputs = llm_flex.embed(prompts)
124
125
# Run with default backend
129
- with monkeypatch.context() as m:
130
- m.setenv("VLLM_USE_V1", "1")
131
- with vllm_runner(
126
+ with (
127
+ monkeypatch.context() as m,
128
+ vllm_runner(
129
model_name,
130
runner="pooling",
131
dtype=torch.bfloat16,
132
tensor_parallel_size=1,
133
max_model_len=100,
134
enforce_eager=True,
138
- ) as llm_default:
139
- default_outputs = llm_default.embed(prompts)
135
+ ) as llm_default,
136
+ ):
137
+ default_outputs = llm_default.embed(prompts)
138
139
check_embeddings_close(
140
embeddings_0_lst=flex_outputs,
tests/models/multimodal/generation/test_maverick.py
-1
@@ -613,7 +613,6 @@ def test_dummy_maverick(
613
profile: bool = False,
614
) -> None:
615
# Disable multiprocessing allows us to access model executor from LLM engine
616
- monkeypatch.setenv("VLLM_USE_V1", "1")
616
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
617
618
model_path = create_reduced_maverick_model(
tests/plugins/vllm_add_dummy_platform/vllm_add_dummy_platform/dummy_platform.py
+1
-5
@@ -8,7 +8,6 @@ if TYPE_CHECKING:
8
from vllm.config import VllmConfig
9
else:
10
VllmConfig = None
11
-from vllm import envs
11
12
13
class DummyPlatform(Platform):
@@ -19,10 +18,7 @@ class DummyPlatform(Platform):
18
19
@classmethod
20
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
22
- if envs.VLLM_USE_V1:
23
- compilation_config = vllm_config.compilation_config
24
- # Activate custom ops for v1.
25
- compilation_config.custom_ops = ["all"]
21
+ vllm_config.compilation_config.custom_ops = ["all"]
22
23
def get_attn_backend_cls(
24
self,
tests/plugins_tests/test_scheduler_plugins.py
-1
@@ -16,7 +16,6 @@ class DummyV1Scheduler(Scheduler):
16
17
def test_scheduler_plugins_v1(monkeypatch: pytest.MonkeyPatch):
18
with monkeypatch.context() as m:
19
- m.setenv("VLLM_USE_V1", "1")
19
# Explicitly turn off engine multiprocessing so
20
# that the scheduler runs in this process
21
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
tests/samplers/test_no_bad_words.py
-7
@@ -8,18 +8,11 @@ Run `pytest tests/samplers/test_no_bad_words.py`.
8
9
from typing import Optional
10
11
-import pytest
11
from transformers import AutoTokenizer
12
13
from vllm import LLM, SamplingParams
14
15
17
-@pytest.fixture(autouse=True)
18
-def v1(monkeypatch):
19
- """Only run on vLLM v1."""
20
- monkeypatch.setenv("VLLM_USE_V1", "1")
21
-
22
-
16
def _generate(
17
llm: LLM,
18
prompt: str,
tests/tpu/lora/test_lora.py
-11
@@ -17,17 +17,6 @@ from vllm.lora.request import LoRARequest
17
# 100 training iterations with a training batch size of 100.
18
19
20
-@pytest.fixture(scope="function", autouse=True)
21
-def use_v1_only(monkeypatch: pytest.MonkeyPatch):
22
- """
23
- Since Multi-LoRA is only supported on the v1 TPU backend, set VLLM_USE_V1=1
24
- for all tests in this file
25
- """
26
- with monkeypatch.context() as m:
27
- m.setenv("VLLM_USE_V1", "1")
28
- yield
29
-
30
-
20
def setup_vllm(num_loras: int, tp: int) -> vllm.LLM:
21
return vllm.LLM(
22
model="Qwen/Qwen2.5-3B-Instruct",
tests/v1/attention/utils.py
-1
@@ -305,7 +305,6 @@ full_cg_backend_configs = {
305
"CutlassMLA": BackendConfig(
306
name="CutlassMLA",
307
env_vars={
308
- "VLLM_USE_V1": "1",
308
"VLLM_ATTENTION_BACKEND": "CUTLASS_MLA",
309
"FORCE_NUM_KV_SPLITS": "1", # TODO: remove this when hang issue is fixed
310
},
tests/v1/core/test_kv_sharing.py
+3
@@ -1,11 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import pytest
5
import torch
6
7
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
8
from vllm.v1.worker.utils import add_kv_sharing_layers_to_kv_cache_groups
9
10
+pytestmark = pytest.mark.cpu_test
11
+
12
13
def new_kv_cache_spec():
14
return FullAttentionSpec(16, 1, 1, torch.float32, False)
tests/v1/core/test_scheduler_e2e.py
-4
@@ -1,14 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import os
3
4
import pytest
5
6
from vllm import LLM
7
9
-if os.getenv("VLLM_USE_V1", "0") != "1":
10
- pytest.skip("Test package requires V1", allow_module_level=True)
11
-
8
MODEL = "meta-llama/Llama-3.2-1B"
9
PROMPT = "Hello my name is Robert and I"
10
tests/v1/cudagraph/test_cudagraph_mode.py
+2
-2
@@ -60,7 +60,7 @@ def test_backend_and_cudagraph_mode_combo(backend_name, cudagraph_mode, supporte
60
):
61
pytest.skip("Only Hopper GPUs support FA3 and FlashMLA")
62
63
- env_vars = {"VLLM_USE_V1": "1", **backend_configs[backend_name].env_vars}
63
+ env_vars = backend_configs[backend_name].env_vars
64
65
with temporary_environ(env_vars), ExitStack() as stack:
66
if not supported:
@@ -117,7 +117,7 @@ combo_cases_2 = [
117
def test_cudagraph_compilation_combo(combo_case):
118
backend_name, cudagraph_mode, compilation_level, supported = combo_case
119
120
- env_vars = {"VLLM_USE_V1": "1", **backend_configs[backend_name].env_vars}
120
+ env_vars = backend_configs[backend_name].env_vars
121
122
with temporary_environ(env_vars), ExitStack() as stack:
123
if not supported:
tests/v1/e2e/test_cascade_attention.py
-1
@@ -20,7 +20,6 @@ def test_cascade_attention(example_system_message, monkeypatch, attn_backend):
20
)
21
22
with monkeypatch.context() as m:
23
- m.setenv("VLLM_USE_V1", "1")
23
m.setenv("VLLM_ATTENTION_BACKEND", attn_backend)
24
25
llm = LLM(model="Qwen/Qwen2-1.5B-Instruct")
tests/v1/e2e/test_correctness_sliding_window.py
+29
-34
@@ -32,7 +32,7 @@ model_config = {
32
@pytest.mark.parametrize("seed", [1])
33
@pytest.mark.parametrize("disable_hybrid_kv_cache_manager", [True, False])
34
def test_sliding_window_retrieval(
35
- monkeypatch, model, batch_size, seed, disable_hybrid_kv_cache_manager
35
+ model, batch_size, seed, disable_hybrid_kv_cache_manager
36
):
37
"""
38
The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
@@ -40,39 +40,34 @@ def test_sliding_window_retrieval(
40
If we tell it upfront which we are going to be looking for, then
41
it answers correctly (mostly).
42
"""
43
- with monkeypatch.context() as m:
44
- m.setenv("VLLM_USE_V1", "1")
45
-
46
- test_config = model_config[model]
47
-
48
- llm = LLM(
49
- model=model, disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
50
- )
51
- sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
52
-
53
- prompts, answer, indices = prep_prompts(
54
- batch_size, ln_range=test_config.ln_range
55
- )
56
-
57
- check_length(prompts, llm, test_config.sliding_window)
58
-
59
- # Fresh generation
60
- responses = llm.generate(prompts, sampling_params)
61
- check_answers(
62
- indices,
63
- answer,
64
- [response.outputs[0].text for response in responses],
65
- accept_rate=1.0,
66
- )
67
-
68
- # Re-generate with the same prompts to test prefix caching
69
- responses = llm.generate(prompts, sampling_params)
70
- check_answers(
71
- indices,
72
- answer,
73
- [response.outputs[0].text for response in responses],
74
- accept_rate=1.0,
75
- )
43
+ test_config = model_config[model]
44
+
45
+ llm = LLM(
46
+ model=model, disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
47
+ )
48
+ sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
49
+
50
+ prompts, answer, indices = prep_prompts(batch_size, ln_range=test_config.ln_range)
51
+
52
+ check_length(prompts, llm, test_config.sliding_window)
53
+
54
+ # Fresh generation
55
+ responses = llm.generate(prompts, sampling_params)
56
+ check_answers(
57
+ indices,
58
+ answer,
59
+ [response.outputs[0].text for response in responses],
60
+ accept_rate=1.0,
61
+ )
62
+
63
+ # Re-generate with the same prompts to test prefix caching
64
+ responses = llm.generate(prompts, sampling_params)
65
+ check_answers(
66
+ indices,
67
+ answer,
68
+ [response.outputs[0].text for response in responses],
69
+ accept_rate=1.0,
70
+ )
71
72
73
def check_length(prompts: list[str], llm: LLM, sliding_window: int):
tests/v1/e2e/test_kv_sharing_fast_prefill.py
-2
@@ -81,8 +81,6 @@ def test_kv_sharing_fast_prefill(
81
)
82
83
with monkeypatch.context() as m:
84
- m.setenv("VLLM_USE_V1", "1")
85
-
84
# Make scheduling deterministic for reproducibility
85
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
86
tests/v1/e2e/test_min_tokens.py
+1
-5
@@ -13,7 +13,6 @@ Covers:
13
5) Multiple stop conditions
14
"""
15
16
-import os
16
from typing import Optional, Union
17
18
import pytest
@@ -161,9 +160,6 @@ MIN_TOKENS_TEST_CASES = [
160
@pytest.fixture(scope="module")
161
def llm_v1():
162
"""Create V1 LLM instance for testing"""
164
- # Ensure V1 engine is used
165
- os.environ["VLLM_USE_V1"] = "1"
166
-
163
llm = LLM(
164
model=TEST_MODEL,
165
tensor_parallel_size=1,
@@ -503,6 +499,6 @@ if __name__ == "__main__":
499
500
Usage:
501
cd vllm/
506
- VLLM_USE_V1=1 python -m pytest tests/v1/e2e/test_min_tokens.py -v
502
+ python -m pytest tests/v1/e2e/test_min_tokens.py -v
503
"""
504
pytest.main([__file__, "-v"])
tests/v1/e2e/test_spec_decode.py
-1
@@ -301,7 +301,6 @@ def test_mtp_correctness(
301
model_setup: (method, model_name, tp_size)
302
"""
303
with monkeypatch.context() as m:
304
- m.setenv("VLLM_USE_V1", "1")
304
m.setenv("VLLM_MLA_DISABLE", "1")
305
306
method, model_name, tp_size = model_setup
tests/v1/engine/test_async_llm.py
+14
-44
@@ -95,17 +95,11 @@ async def generate(
95
)
96
@pytest.mark.asyncio
97
async def test_load(
98
- monkeypatch: pytest.MonkeyPatch,
98
output_kind: RequestOutputKind,
99
engine_args: AsyncEngineArgs,
100
prompt: PromptType,
101
):
103
- # TODO(rickyx): Remove monkeypatch once we have a better way to test V1
104
- # so that in the future when we switch, we don't have to change all the
105
- # tests.
106
- with monkeypatch.context() as m, ExitStack() as after:
107
- m.setenv("VLLM_USE_V1", "1")
108
-
102
+ with ExitStack() as after:
103
with set_default_torch_num_threads(1):
104
engine = AsyncLLM.from_engine_args(engine_args)
105
after.callback(engine.shutdown)
@@ -149,14 +143,11 @@ async def test_load(
143
)
144
@pytest.mark.asyncio
145
async def test_abort(
152
- monkeypatch: pytest.MonkeyPatch,
146
output_kind: RequestOutputKind,
147
engine_args: AsyncEngineArgs,
148
prompt: PromptType,
149
):
157
- with monkeypatch.context() as m, ExitStack() as after:
158
- m.setenv("VLLM_USE_V1", "1")
159
-
150
+ with ExitStack() as after:
151
with set_default_torch_num_threads(1):
152
engine = AsyncLLM.from_engine_args(engine_args)
153
after.callback(engine.shutdown)
@@ -222,13 +213,8 @@ async def test_abort(
213
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY]
214
)
215
@pytest.mark.asyncio
225
-async def test_multi_abort(
226
- monkeypatch: pytest.MonkeyPatch,
227
- output_kind: RequestOutputKind,
228
-):
229
- with monkeypatch.context() as m, ExitStack() as after:
230
- m.setenv("VLLM_USE_V1", "1")
231
-
216
+async def test_multi_abort(output_kind: RequestOutputKind):
217
+ with ExitStack() as after:
218
with set_default_torch_num_threads(1):
219
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
220
after.callback(engine.shutdown)
@@ -304,14 +290,11 @@ async def test_multi_abort(
290
)
291
@pytest.mark.asyncio
292
async def test_finished_flag(
307
- monkeypatch: pytest.MonkeyPatch,
293
n: int,
294
engine_args: AsyncEngineArgs,
295
prompt: PromptType,
296
):
312
- with monkeypatch.context() as m, ExitStack() as after:
313
- m.setenv("VLLM_USE_V1", "1")
314
-
297
+ with ExitStack() as after:
298
with set_default_torch_num_threads(1):
299
engine = AsyncLLM.from_engine_args(engine_args)
300
after.callback(engine.shutdown)
@@ -341,12 +324,10 @@ async def test_finished_flag(
324
)
325
@pytest.mark.asyncio
326
async def test_mid_stream_cancellation(
344
- monkeypatch: pytest.MonkeyPatch, engine_args: AsyncEngineArgs, prompt: PromptType
327
+ engine_args: AsyncEngineArgs, prompt: PromptType
328
):
329
"""Test that requests can be cancelled mid-stream."""
347
- with monkeypatch.context() as m, ExitStack() as after:
348
- m.setenv("VLLM_USE_V1", "1")
349
-
330
+ with ExitStack() as after:
331
with set_default_torch_num_threads(1):
332
engine = AsyncLLM.from_engine_args(engine_args)
333
after.callback(engine.shutdown)
@@ -411,9 +392,7 @@ async def test_customize_loggers(monkeypatch):
392
be added to the default loggers.
393
"""
394
414
- with monkeypatch.context() as m, ExitStack() as after:
415
- m.setenv("VLLM_USE_V1", "1")
416
-
395
+ with ExitStack() as after:
396
with set_default_torch_num_threads(1):
397
engine = AsyncLLM.from_engine_args(
398
TEXT_ENGINE_ARGS,
@@ -430,10 +409,8 @@ async def test_customize_loggers(monkeypatch):
409
410
411
@pytest.mark.asyncio(scope="module")
433
-async def test_dp_rank_argument(monkeypatch: pytest.MonkeyPatch):
434
- with monkeypatch.context() as m, ExitStack() as after:
435
- m.setenv("VLLM_USE_V1", "1")
436
-
412
+async def test_dp_rank_argument():
413
+ with ExitStack() as after:
414
with set_default_torch_num_threads(1):
415
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
416
after.callback(engine.shutdown)
@@ -466,7 +443,7 @@ async def test_dp_rank_argument(monkeypatch: pytest.MonkeyPatch):
443
444
445
@pytest.mark.asyncio
469
-async def test_check_health(monkeypatch: pytest.MonkeyPatch):
446
+async def test_check_health():
447
"""Test that check_health returns normally for healthy engine
448
and raises EngineDeadError when the engine is dead.
449
"""
@@ -474,9 +451,7 @@ async def test_check_health(monkeypatch: pytest.MonkeyPatch):
451
452
from vllm.v1.engine.exceptions import EngineDeadError
453
477
- with monkeypatch.context() as m, ExitStack() as after:
478
- m.setenv("VLLM_USE_V1", "1")
479
-
454
+ with ExitStack() as after:
455
with set_default_torch_num_threads(1):
456
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
457
after.callback(engine.shutdown)
@@ -503,15 +478,10 @@ async def test_check_health(monkeypatch: pytest.MonkeyPatch):
478
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY]
479
)
480
@pytest.mark.asyncio
506
-async def test_abort_final_output(
507
- monkeypatch: pytest.MonkeyPatch,
508
- output_kind: RequestOutputKind,
509
-):
481
+async def test_abort_final_output(output_kind: RequestOutputKind):
482
"""Test that abort() returns a final output with correct information."""
483
512
- with monkeypatch.context() as m, ExitStack() as after:
513
- m.setenv("VLLM_USE_V1", "1")
514
-
484
+ with ExitStack() as after:
485
with set_default_torch_num_threads(1):
486
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
487
after.callback(engine.shutdown)
tests/v1/engine/test_engine_args.py
-7
@@ -5,18 +5,11 @@ from argparse import ArgumentError
5
6
import pytest
7
8
-from vllm import envs
8
from vllm.config import VllmConfig
9
from vllm.engine.arg_utils import EngineArgs
10
from vllm.usage.usage_lib import UsageContext
11
from vllm.utils import FlexibleArgumentParser
12
14
-if not envs.VLLM_USE_V1:
15
- pytest.skip(
16
- "Skipping V1 tests. Rerun with `VLLM_USE_V1=1` to test.",
17
- allow_module_level=True,
18
- )
19
-
13
14
def test_prefix_caching_from_cli():
15
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
tests/v1/engine/test_engine_core.py
+300
-314
@@ -46,188 +46,184 @@ def make_request() -> EngineCoreRequest:
46
47
48
@create_new_process_for_each_test()
49
-def test_engine_core(monkeypatch: pytest.MonkeyPatch):
50
- with monkeypatch.context() as m:
51
- m.setenv("VLLM_USE_V1", "1")
52
- """Setup the EngineCore."""
53
- engine_args = EngineArgs(model=MODEL_NAME)
54
- vllm_config = engine_args.create_engine_config()
55
- executor_class = Executor.get_class(vllm_config)
56
-
57
- with set_default_torch_num_threads(1):
58
- engine_core = EngineCore(
59
- vllm_config=vllm_config, executor_class=executor_class, log_stats=True
60
- )
61
- """Test basic request lifecycle."""
62
-
63
- # First request.
64
- engine_core.add_request(*engine_core.preprocess_add_request(make_request()))
65
- assert len(engine_core.scheduler.waiting) == 1
66
- assert len(engine_core.scheduler.running) == 0
67
-
68
- _ = engine_core.step()
69
- assert len(engine_core.scheduler.waiting) == 0
70
- assert len(engine_core.scheduler.running) == 1
71
-
72
- # Second request.
73
- engine_core.add_request(*engine_core.preprocess_add_request(make_request()))
74
- assert len(engine_core.scheduler.waiting) == 1
75
- assert len(engine_core.scheduler.running) == 1
76
-
77
- _ = engine_core.step()
78
- assert len(engine_core.scheduler.waiting) == 0
79
- assert len(engine_core.scheduler.running) == 2
80
-
81
- # Add two requests in a row.
82
- engine_core.add_request(*engine_core.preprocess_add_request(make_request()))
83
- engine_core.add_request(*engine_core.preprocess_add_request(make_request()))
84
- assert len(engine_core.scheduler.waiting) == 2
85
- assert len(engine_core.scheduler.running) == 2
86
-
87
- _ = engine_core.step()
88
- assert len(engine_core.scheduler.waiting) == 0
89
- assert len(engine_core.scheduler.running) == 4
90
-
91
- # Loop through until they are all done.
92
- while (outs := engine_core.step()[0].get(0)) and outs.outputs:
93
- pass
49
+def test_engine_core():
50
+ """Setup the EngineCore."""
51
+ engine_args = EngineArgs(model=MODEL_NAME)
52
+ vllm_config = engine_args.create_engine_config()
53
+ executor_class = Executor.get_class(vllm_config)
54
+
55
+ with set_default_torch_num_threads(1):
56
+ engine_core = EngineCore(
57
+ vllm_config=vllm_config, executor_class=executor_class, log_stats=True
58
+ )
59
+ """Test basic request lifecycle."""
60
+
61
+ # First request.
62
+ engine_core.add_request(*engine_core.preprocess_add_request(make_request()))
63
+ assert len(engine_core.scheduler.waiting) == 1
64
+ assert len(engine_core.scheduler.running) == 0
65
+
66
+ _ = engine_core.step()
67
+ assert len(engine_core.scheduler.waiting) == 0
68
+ assert len(engine_core.scheduler.running) == 1
69
+
70
+ # Second request.
71
+ engine_core.add_request(*engine_core.preprocess_add_request(make_request()))
72
+ assert len(engine_core.scheduler.waiting) == 1
73
+ assert len(engine_core.scheduler.running) == 1
74
+
75
+ _ = engine_core.step()
76
+ assert len(engine_core.scheduler.waiting) == 0
77
+ assert len(engine_core.scheduler.running) == 2
78
+
79
+ # Add two requests in a row.
80
+ engine_core.add_request(*engine_core.preprocess_add_request(make_request()))
81
+ engine_core.add_request(*engine_core.preprocess_add_request(make_request()))
82
+ assert len(engine_core.scheduler.waiting) == 2
83
+ assert len(engine_core.scheduler.running) == 2
84
+
85
+ _ = engine_core.step()
86
+ assert len(engine_core.scheduler.waiting) == 0
87
+ assert len(engine_core.scheduler.running) == 4
88
+
89
+ # Loop through until they are all done.
90
+ while (outs := engine_core.step()[0].get(0)) and outs.outputs:
91
+ pass
92
+
93
+ assert len(engine_core.scheduler.waiting) == 0
94
+ assert len(engine_core.scheduler.running) == 0
95
+ """Test abort cycle."""
96
+
97
+ # Basic abort.
98
+ req = make_request()
99
+ request_id = req.request_id
100
+
101
+ engine_core.add_request(*engine_core.preprocess_add_request(req))
102
+ assert len(engine_core.scheduler.waiting) == 1
103
+ assert len(engine_core.scheduler.running) == 0
104
+ assert engine_core.scheduler.has_unfinished_requests()
105
+ assert not engine_core.scheduler.has_finished_requests()
106
+
107
+ _ = engine_core.step()
108
+ assert len(engine_core.scheduler.waiting) == 0
109
+ assert len(engine_core.scheduler.running) == 1
110
+ assert engine_core.scheduler.has_unfinished_requests()
111
+ assert not engine_core.scheduler.has_finished_requests()
112
+
113
+ engine_core.abort_requests([request_id])
114
+ assert len(engine_core.scheduler.waiting) == 0
115
+ assert len(engine_core.scheduler.running) == 0
116
+ assert not engine_core.scheduler.has_unfinished_requests()
117
+ assert engine_core.scheduler.has_finished_requests()
118
+
119
+ _ = engine_core.step()
120
+ assert not engine_core.scheduler.has_unfinished_requests()
121
+ assert not engine_core.scheduler.has_finished_requests()
122
+
123
+ # Add, step, abort 1 of the 3.
124
+ req0 = make_request()
125
+ req1 = make_request()
126
+ req2 = make_request()
127
+
128
+ engine_core.add_request(*engine_core.preprocess_add_request(req0))
129
+ engine_core.add_request(*engine_core.preprocess_add_request(req1))
130
+ assert len(engine_core.scheduler.waiting) == 2
131
+ assert len(engine_core.scheduler.running) == 0
132
+
133
+ _ = engine_core.step()
134
+ assert len(engine_core.scheduler.waiting) == 0
135
+ assert len(engine_core.scheduler.running) == 2
136
+
137
+ engine_core.add_request(*engine_core.preprocess_add_request(req2))
138
+ assert len(engine_core.scheduler.waiting) == 1
139
+ assert len(engine_core.scheduler.running) == 2
140
+
141
+ _ = engine_core.step()
142
+ assert len(engine_core.scheduler.waiting) == 0
143
+ assert len(engine_core.scheduler.running) == 3
144
+
145
+ # Abort just one.
146
+ engine_core.abort_requests([req1.request_id])
147
+ assert len(engine_core.scheduler.waiting) == 0
148
+ assert len(engine_core.scheduler.running) == 2
149
+
150
+ _ = engine_core.step()
151
+ assert len(engine_core.scheduler.waiting) == 0
152
+ assert len(engine_core.scheduler.running) == 2
153
+
154
+ # Abort the other requests at the same time.
155
+ engine_core.abort_requests([req2.request_id, req0.request_id])
156
+ assert len(engine_core.scheduler.waiting) == 0
157
+ assert len(engine_core.scheduler.running) == 0
158
+
159
+ # Sending duplicate requests with same request_id
160
+ req0 = make_request()
161
+ req1 = make_request()
162
+ req0.request_id = req1.request_id = "test"
163
+ engine_core.add_request(*engine_core.preprocess_add_request(req0))
164
+
165
+ while (outs := engine_core.step()[0].get(0)) and outs.outputs:
166
+ pass
167
+
168
+ engine_core.add_request(*engine_core.preprocess_add_request(req1))
169
+ while (outs := engine_core.step()[0].get(0)) and outs.outputs:
170
+ pass
171
+
172
+ assert len(engine_core.scheduler.waiting) == 0
173
+ assert len(engine_core.scheduler.running) == 0
174
95
- assert len(engine_core.scheduler.waiting) == 0
96
- assert len(engine_core.scheduler.running) == 0
97
- """Test abort cycle."""
175
99
- # Basic abort.
100
- req = make_request()
101
- request_id = req.request_id
176
+@create_new_process_for_each_test()
177
+def test_engine_core_advanced_sampling():
178
+ """
179
+ A basic end-to-end test to verify that the engine functions correctly
180
+ when additional sampling parameters, such as top_p, min_tokens, and
181
+ presence_penalty, are set.
182
+ """
183
+ """Setup the EngineCore."""
184
+ engine_args = EngineArgs(model=MODEL_NAME)
185
+ vllm_config = engine_args.create_engine_config()
186
+ executor_class = Executor.get_class(vllm_config)
187
+
188
+ with set_default_torch_num_threads(1):
189
+ engine_core = EngineCore(
190
+ vllm_config=vllm_config, executor_class=executor_class, log_stats=True
191
+ )
192
+ """Test basic request lifecycle."""
193
+ # First request.
194
+ request: EngineCoreRequest = make_request()
195
+ request.sampling_params = SamplingParams(
196
+ min_tokens=4,
197
+ presence_penalty=1.0,
198
+ frequency_penalty=1.0,
199
+ repetition_penalty=0.1,
200
+ stop_token_ids=[1001, 1002],
201
+ )
202
+ engine_core.add_request(*engine_core.preprocess_add_request(request))
203
103
- engine_core.add_request(*engine_core.preprocess_add_request(req))
204
+ def _check_engine_state():
205
assert len(engine_core.scheduler.waiting) == 1
206
assert len(engine_core.scheduler.running) == 0
106
- assert engine_core.scheduler.has_unfinished_requests()
107
- assert not engine_core.scheduler.has_finished_requests()
108
-
109
- _ = engine_core.step()
110
- assert len(engine_core.scheduler.waiting) == 0
111
- assert len(engine_core.scheduler.running) == 1
112
- assert engine_core.scheduler.has_unfinished_requests()
113
- assert not engine_core.scheduler.has_finished_requests()
114
-
115
- engine_core.abort_requests([request_id])
116
- assert len(engine_core.scheduler.waiting) == 0
117
- assert len(engine_core.scheduler.running) == 0
118
- assert not engine_core.scheduler.has_unfinished_requests()
119
- assert engine_core.scheduler.has_finished_requests()
120
-
121
- _ = engine_core.step()
122
- assert not engine_core.scheduler.has_unfinished_requests()
123
- assert not engine_core.scheduler.has_finished_requests()
124
-
125
- # Add, step, abort 1 of the 3.
126
- req0 = make_request()
127
- req1 = make_request()
128
- req2 = make_request()
129
-
130
- engine_core.add_request(*engine_core.preprocess_add_request(req0))
131
- engine_core.add_request(*engine_core.preprocess_add_request(req1))
132
- assert len(engine_core.scheduler.waiting) == 2
133
- assert len(engine_core.scheduler.running) == 0
134
-
135
- _ = engine_core.step()
136
- assert len(engine_core.scheduler.waiting) == 0
137
- assert len(engine_core.scheduler.running) == 2
138
-
139
- engine_core.add_request(*engine_core.preprocess_add_request(req2))
140
- assert len(engine_core.scheduler.waiting) == 1
141
- assert len(engine_core.scheduler.running) == 2
142
-
143
- _ = engine_core.step()
144
- assert len(engine_core.scheduler.waiting) == 0
145
- assert len(engine_core.scheduler.running) == 3
146
-
147
- # Abort just one.
148
- engine_core.abort_requests([req1.request_id])
149
- assert len(engine_core.scheduler.waiting) == 0
150
- assert len(engine_core.scheduler.running) == 2
151
-
152
- _ = engine_core.step()
153
- assert len(engine_core.scheduler.waiting) == 0
154
- assert len(engine_core.scheduler.running) == 2
155
-
156
- # Abort the other requests at the same time.
157
- engine_core.abort_requests([req2.request_id, req0.request_id])
158
- assert len(engine_core.scheduler.waiting) == 0
159
- assert len(engine_core.scheduler.running) == 0
160
-
161
- # Sending duplicate requests with same request_id
162
- req0 = make_request()
163
- req1 = make_request()
164
- req0.request_id = req1.request_id = "test"
165
- engine_core.add_request(*engine_core.preprocess_add_request(req0))
166
-
167
- while (outs := engine_core.step()[0].get(0)) and outs.outputs:
168
- pass
169
-
170
- engine_core.add_request(*engine_core.preprocess_add_request(req1))
207
+ # Loop through until they are all done.
208
while (outs := engine_core.step()[0].get(0)) and outs.outputs:
209
pass
173
-
210
assert len(engine_core.scheduler.waiting) == 0
211
assert len(engine_core.scheduler.running) == 0
212
213
+ _check_engine_state()
214
178
-@create_new_process_for_each_test()
179
-def test_engine_core_advanced_sampling(monkeypatch: pytest.MonkeyPatch):
180
- """
181
- A basic end-to-end test to verify that the engine functions correctly
182
- when additional sampling parameters, such as top_p, min_tokens, and
183
- presence_penalty, are set.
184
- """
185
- with monkeypatch.context() as m:
186
- m.setenv("VLLM_USE_V1", "1")
187
- """Setup the EngineCore."""
188
- engine_args = EngineArgs(model=MODEL_NAME)
189
- vllm_config = engine_args.create_engine_config()
190
- executor_class = Executor.get_class(vllm_config)
191
-
192
- with set_default_torch_num_threads(1):
193
- engine_core = EngineCore(
194
- vllm_config=vllm_config, executor_class=executor_class, log_stats=True
195
- )
196
- """Test basic request lifecycle."""
197
- # First request.
198
- request: EngineCoreRequest = make_request()
199
- request.sampling_params = SamplingParams(
200
- min_tokens=4,
201
- presence_penalty=1.0,
202
- frequency_penalty=1.0,
203
- repetition_penalty=0.1,
204
- stop_token_ids=[1001, 1002],
205
- )
206
- engine_core.add_request(*engine_core.preprocess_add_request(request))
207
-
208
- def _check_engine_state():
209
- assert len(engine_core.scheduler.waiting) == 1
210
- assert len(engine_core.scheduler.running) == 0
211
- # Loop through until they are all done.
212
- while (outs := engine_core.step()[0].get(0)) and outs.outputs:
213
- pass
214
- assert len(engine_core.scheduler.waiting) == 0
215
- assert len(engine_core.scheduler.running) == 0
216
-
217
- _check_engine_state()
218
-
219
- # Second request.
220
- request2 = make_request()
221
- request2.sampling_params = SamplingParams(
222
- top_p=0.99,
223
- top_k=50,
224
- )
225
- engine_core.add_request(*engine_core.preprocess_add_request(request2))
226
- _check_engine_state()
215
+ # Second request.
216
+ request2 = make_request()
217
+ request2.sampling_params = SamplingParams(
218
+ top_p=0.99,
219
+ top_k=50,
220
+ )
221
+ engine_core.add_request(*engine_core.preprocess_add_request(request2))
222
+ _check_engine_state()
223
224
225
@create_new_process_for_each_test()
230
-def test_engine_core_concurrent_batches(monkeypatch: pytest.MonkeyPatch):
226
+def test_engine_core_concurrent_batches():
227
"""
228
Test that the engine can handle multiple concurrent batches.
229
"""
@@ -272,173 +268,163 @@ def test_engine_core_concurrent_batches(monkeypatch: pytest.MonkeyPatch):
268
if hasattr(self, "thread_pool"):
269
self.thread_pool.shutdown(wait=False)
270
275
- with monkeypatch.context() as m:
276
- m.setenv("VLLM_USE_V1", "1")
277
-
278
- engine_args = EngineArgs(
279
- model=MODEL_NAME,
280
- # To test concurrent batches.
281
- max_num_seqs=2,
282
- # Avoid all requests being scheduled once.
283
- enable_prefix_caching=False,
284
- max_num_batched_tokens=10,
285
- # Reduce startup time.
286
- enforce_eager=True,
271
+ engine_args = EngineArgs(
272
+ model=MODEL_NAME,
273
+ # To test concurrent batches.
274
+ max_num_seqs=2,
275
+ # Avoid all requests being scheduled once.
276
+ enable_prefix_caching=False,
277
+ max_num_batched_tokens=10,
278
+ # Reduce startup time.
279
+ enforce_eager=True,
280
+ )
281
+ vllm_config = engine_args.create_engine_config()
282
+ with set_default_torch_num_threads(1):
283
+ engine_core = EngineCore(
284
+ vllm_config=vllm_config, log_stats=False, executor_class=DummyExecutor
285
)
288
- vllm_config = engine_args.create_engine_config()
289
- with set_default_torch_num_threads(1):
290
- engine_core = EngineCore(
291
- vllm_config=vllm_config, log_stats=False, executor_class=DummyExecutor
292
- )
293
- assert engine_core.batch_queue is not None
294
-
295
- # Add two requests in a row. Each request have 12 prompt tokens.
296
- req0 = make_request_with_max_tokens("0", 5)
297
- engine_core.add_request(*engine_core.preprocess_add_request(req0))
298
- req1 = make_request_with_max_tokens("1", 5)
299
- engine_core.add_request(*engine_core.preprocess_add_request(req1))
300
-
301
- # Schedule Batch 1: (10, req0)
302
- assert engine_core.step_with_batch_queue()[0] is None
303
- assert len(engine_core.batch_queue) == 1
304
- scheduler_output = engine_core.batch_queue[-1][1]
305
- assert scheduler_output.num_scheduled_tokens["0"] == 10
306
- # num_computed_tokens should have been updated immediately.
307
- assert engine_core.scheduler.requests[req0.request_id].num_computed_tokens == 10
308
-
309
- # Schedule Batch 2: (2, req0), (8, req1)
310
- assert engine_core.step_with_batch_queue()[0] == {}
311
- assert len(engine_core.batch_queue) == 1
312
- scheduler_output = engine_core.batch_queue[-1][1]
313
- assert scheduler_output.num_scheduled_tokens["0"] == 2
314
- assert scheduler_output.num_scheduled_tokens["1"] == 8
315
- # num_computed_tokens should have been updated immediately.
316
- assert engine_core.scheduler.requests["0"].num_computed_tokens == 12
317
- assert engine_core.scheduler.requests["1"].num_computed_tokens == 8
318
-
319
- assert engine_core.scheduler.get_num_unfinished_requests() == 2
320
-
321
- # Finish Batch 1 and schedule Batch 3: (4, req1).
322
- # Note that req0 cannot be scheduled
323
- # because it is in the decoding stage now.
324
- engine_core.step_with_batch_queue()
325
- assert len(engine_core.batch_queue) == 1
326
- scheduler_output = engine_core.batch_queue[-1][1]
327
- assert scheduler_output.num_scheduled_tokens["1"] == 4
328
-
329
- # Finish Batch 2. Get first token of req0.
330
- # Schedule Batch 4: (1, req0).
331
- output = engine_core.step_with_batch_queue()[0].get(0)
286
+ assert engine_core.batch_queue is not None
287
+
288
+ # Add two requests in a row. Each request have 12 prompt tokens.
289
+ req0 = make_request_with_max_tokens("0", 5)
290
+ engine_core.add_request(*engine_core.preprocess_add_request(req0))
291
+ req1 = make_request_with_max_tokens("1", 5)
292
+ engine_core.add_request(*engine_core.preprocess_add_request(req1))
293
+
294
+ # Schedule Batch 1: (10, req0)
295
+ assert engine_core.step_with_batch_queue()[0] is None
296
+ assert len(engine_core.batch_queue) == 1
297
+ scheduler_output = engine_core.batch_queue[-1][1]
298
+ assert scheduler_output.num_scheduled_tokens["0"] == 10
299
+ # num_computed_tokens should have been updated immediately.
300
+ assert engine_core.scheduler.requests[req0.request_id].num_computed_tokens == 10
301
+
302
+ # Schedule Batch 2: (2, req0), (8, req1)
303
+ assert engine_core.step_with_batch_queue()[0] == {}
304
+ assert len(engine_core.batch_queue) == 1
305
+ scheduler_output = engine_core.batch_queue[-1][1]
306
+ assert scheduler_output.num_scheduled_tokens["0"] == 2
307
+ assert scheduler_output.num_scheduled_tokens["1"] == 8
308
+ # num_computed_tokens should have been updated immediately.
309
+ assert engine_core.scheduler.requests["0"].num_computed_tokens == 12
310
+ assert engine_core.scheduler.requests["1"].num_computed_tokens == 8
311
+
312
+ assert engine_core.scheduler.get_num_unfinished_requests() == 2
313
+
314
+ # Finish Batch 1 and schedule Batch 3: (4, req1).
315
+ # Note that req0 cannot be scheduled
316
+ # because it is in the decoding stage now.
317
+ engine_core.step_with_batch_queue()
318
+ assert len(engine_core.batch_queue) == 1
319
+ scheduler_output = engine_core.batch_queue[-1][1]
320
+ assert scheduler_output.num_scheduled_tokens["1"] == 4
321
+
322
+ # Finish Batch 2. Get first token of req0.
323
+ # Schedule Batch 4: (1, req0).
324
+ output = engine_core.step_with_batch_queue()[0].get(0)
325
+ assert output is not None
326
+ assert len(output.outputs) == 1
327
+ assert engine_core.scheduler.requests[req0.request_id].num_tokens == 13
328
+ scheduler_output = engine_core.batch_queue[-1][1]
329
+ assert scheduler_output.num_scheduled_tokens["0"] == 1
330
+
331
+ # Finish Batch 3. Get first token of req1. Schedule Batch 5: (1, req1).
332
+ output = engine_core.step_with_batch_queue()[0].get(0)
333
+ assert output is not None
334
+ assert len(output.outputs) == 1
335
+ assert engine_core.scheduler.requests[req1.request_id].num_tokens == 13
336
+ scheduler_output = engine_core.batch_queue[-1][1]
337
+ assert scheduler_output.num_scheduled_tokens["1"] == 1
338
+
339
+ # Loop until req0 is finished.
340
+ req_id = 0
341
+ expected_num_tokens = [
342
+ engine_core.scheduler.requests["0"].num_tokens + 1,
343
+ engine_core.scheduler.requests["1"].num_tokens + 1,
344
+ ]
345
+ while engine_core.scheduler.get_num_unfinished_requests() == 2:
346
+ output = engine_core.step_with_batch_queue()[0]
347
+ # Every step consumes an output.
348
assert output is not None
333
- assert len(output.outputs) == 1
334
- assert engine_core.scheduler.requests[req0.request_id].num_tokens == 13
335
- scheduler_output = engine_core.batch_queue[-1][1]
336
- assert scheduler_output.num_scheduled_tokens["0"] == 1
337
-
338
- # Finish Batch 3. Get first token of req1. Schedule Batch 5: (1, req1).
339
- output = engine_core.step_with_batch_queue()[0].get(0)
340
- assert output is not None
341
- assert len(output.outputs) == 1
342
- assert engine_core.scheduler.requests[req1.request_id].num_tokens == 13
343
- scheduler_output = engine_core.batch_queue[-1][1]
344
- assert scheduler_output.num_scheduled_tokens["1"] == 1
345
-
346
- # Loop until req0 is finished.
347
- req_id = 0
348
- expected_num_tokens = [
349
- engine_core.scheduler.requests["0"].num_tokens + 1,
350
- engine_core.scheduler.requests["1"].num_tokens + 1,
351
- ]
352
- while engine_core.scheduler.get_num_unfinished_requests() == 2:
353
- output = engine_core.step_with_batch_queue()[0]
354
- # Every step consumes an output.
355
- assert output is not None
356
- assert len(output[0].outputs) == 1
357
- if req_id in engine_core.scheduler.requests:
358
- assert (
359
- engine_core.scheduler.requests[req_id].num_tokens
360
- == expected_num_tokens[req_id]
361
- )
362
- expected_num_tokens[req_id] += 1
363
- req_id = (req_id + 1) % 2
349
+ assert len(output[0].outputs) == 1
350
+ if req_id in engine_core.scheduler.requests:
351
+ assert (
352
+ engine_core.scheduler.requests[req_id].num_tokens
353
+ == expected_num_tokens[req_id]
354
+ )
355
+ expected_num_tokens[req_id] += 1
356
+ req_id = (req_id + 1) % 2
357
358
359
@multi_gpu_test(num_gpus=2)
367
-def test_engine_core_tp(monkeypatch: pytest.MonkeyPatch):
360
+def test_engine_core_tp():
361
"""
362
Test engine can initialize worker in tp properly
363
"""
364
372
- with monkeypatch.context() as m:
373
- m.setenv("VLLM_USE_V1", "1")
374
- """Setup the EngineCore."""
375
- engine_args = EngineArgs(
376
- model=MODEL_NAME,
377
- tensor_parallel_size=2,
378
- # Reduce startup time.
379
- enforce_eager=True,
380
- )
381
- vllm_config = engine_args.create_engine_config()
382
- executor_class = Executor.get_class(vllm_config)
365
+ """Setup the EngineCore."""
366
+ engine_args = EngineArgs(
367
+ model=MODEL_NAME,
368
+ tensor_parallel_size=2,
369
+ # Reduce startup time.
370
+ enforce_eager=True,
371
+ )
372
+ vllm_config = engine_args.create_engine_config()
373
+ executor_class = Executor.get_class(vllm_config)
374
384
- with set_default_torch_num_threads(1):
385
- engine_core = EngineCore(
386
- vllm_config=vllm_config, executor_class=executor_class, log_stats=True
387
- )
375
+ with set_default_torch_num_threads(1):
376
+ engine_core = EngineCore(
377
+ vllm_config=vllm_config, executor_class=executor_class, log_stats=True
378
+ )
379
389
- def get_worker_cache_config_field(worker, key: str):
390
- return getattr(worker.cache_config, key)
380
+ def get_worker_cache_config_field(worker, key: str):
381
+ return getattr(worker.cache_config, key)
382
392
- num_gpu_blocks = engine_core.collective_rpc(
393
- get_worker_cache_config_field, args=("num_gpu_blocks",)
394
- )
395
- num_cpu_blocks = engine_core.collective_rpc(
396
- get_worker_cache_config_field, args=("num_cpu_blocks",)
397
- )
398
- assert all(x is not None for x in num_gpu_blocks)
399
- assert all(x is not None for x in num_cpu_blocks)
383
+ num_gpu_blocks = engine_core.collective_rpc(
384
+ get_worker_cache_config_field, args=("num_gpu_blocks",)
385
+ )
386
+ num_cpu_blocks = engine_core.collective_rpc(
387
+ get_worker_cache_config_field, args=("num_cpu_blocks",)
388
+ )
389
+ assert all(x is not None for x in num_gpu_blocks)
390
+ assert all(x is not None for x in num_cpu_blocks)
391
392
393
@create_new_process_for_each_test()
403
-def test_engine_core_invalid_request_id_type(monkeypatch: pytest.MonkeyPatch):
394
+def test_engine_core_invalid_request_id_type():
395
"""Test that engine raises TypeError for non-string request_id."""
405
- with monkeypatch.context() as m:
406
- m.setenv("VLLM_USE_V1", "1")
396
+ engine_args = EngineArgs(model=MODEL_NAME)
397
+ vllm_config = engine_args.create_engine_config()
398
+ executor_class = Executor.get_class(vllm_config)
399
408
- engine_args = EngineArgs(model=MODEL_NAME)
409
- vllm_config = engine_args.create_engine_config()
410
- executor_class = Executor.get_class(vllm_config)
411
-
412
- with set_default_torch_num_threads(1):
413
- engine_core = EngineCore(
414
- vllm_config=vllm_config, executor_class=executor_class, log_stats=True
415
- )
400
+ with set_default_torch_num_threads(1):
401
+ engine_core = EngineCore(
402
+ vllm_config=vllm_config, executor_class=executor_class, log_stats=True
403
+ )
404
417
- # Test with UUID object (common mistake)
418
- uuid_request = make_request()
419
- uuid_request.request_id = uuid.uuid4() # UUID object instead of string
405
+ # Test with UUID object (common mistake)
406
+ uuid_request = make_request()
407
+ uuid_request.request_id = uuid.uuid4() # UUID object instead of string
408
421
- with pytest.raises(TypeError, match="request_id must be a string, got.*UUID"):
422
- engine_core.add_request(*engine_core.preprocess_add_request(uuid_request))
409
+ with pytest.raises(TypeError, match="request_id must be a string, got.*UUID"):
410
+ engine_core.add_request(*engine_core.preprocess_add_request(uuid_request))
411
424
- # Test with integer
425
- int_request = make_request()
426
- int_request.request_id = 12345
412
+ # Test with integer
413
+ int_request = make_request()
414
+ int_request.request_id = 12345
415
428
- with pytest.raises(TypeError, match="request_id must be a string, got.*int"):
429
- engine_core.add_request(*engine_core.preprocess_add_request(int_request))
416
+ with pytest.raises(TypeError, match="request_id must be a string, got.*int"):
417
+ engine_core.add_request(*engine_core.preprocess_add_request(int_request))
418
431
- # Test with None
432
- none_request = make_request()
433
- none_request.request_id = None
419
+ # Test with None
420
+ none_request = make_request()
421
+ none_request.request_id = None
422
435
- with pytest.raises(
436
- TypeError, match="request_id must be a string, got.*NoneType"
437
- ):
438
- engine_core.add_request(*engine_core.preprocess_add_request(none_request))
423
+ with pytest.raises(TypeError, match="request_id must be a string, got.*NoneType"):
424
+ engine_core.add_request(*engine_core.preprocess_add_request(none_request))
425
440
- # Verify engine is still functional after errors
441
- valid_request = make_request()
442
- engine_core.add_request(*engine_core.preprocess_add_request(valid_request))
443
- assert len(engine_core.scheduler.waiting) == 1
444
- assert len(engine_core.scheduler.running) == 0
426
+ # Verify engine is still functional after errors
427
+ valid_request = make_request()
428
+ engine_core.add_request(*engine_core.preprocess_add_request(valid_request))
429
+ assert len(engine_core.scheduler.waiting) == 1
430
+ assert len(engine_core.scheduler.running) == 0
tests/v1/engine/test_engine_core_client.py
+125
-146
@@ -130,8 +130,6 @@ def test_engine_core_client(
130
monkeypatch: pytest.MonkeyPatch, multiprocessing_mode: bool
131
):
132
with monkeypatch.context() as m:
133
- m.setenv("VLLM_USE_V1", "1")
134
-
133
# Monkey-patch core engine utility function to test.
134
m.setattr(EngineCore, "echo", echo, raising=False)
135
@@ -218,8 +216,6 @@ def test_engine_core_client(
216
@pytest.mark.asyncio(loop_scope="function")
217
async def test_engine_core_client_asyncio(monkeypatch: pytest.MonkeyPatch):
218
with monkeypatch.context() as m:
221
- m.setenv("VLLM_USE_V1", "1")
222
-
219
# Monkey-patch core engine utility function to test.
220
m.setattr(EngineCore, "echo", echo, raising=False)
221
@@ -373,8 +369,6 @@ async def test_engine_core_client_util_method_custom_return(
369
monkeypatch: pytest.MonkeyPatch,
370
):
371
with monkeypatch.context() as m:
376
- m.setenv("VLLM_USE_V1", "1")
377
-
372
# Must set insecure serialization to allow returning custom types.
373
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
374
@@ -422,8 +416,6 @@ async def test_engine_core_client_util_method_custom_dict_return(
416
monkeypatch: pytest.MonkeyPatch,
417
):
418
with monkeypatch.context() as m:
425
- m.setenv("VLLM_USE_V1", "1")
426
-
419
# Must set insecure serialization to allow returning custom types.
420
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
421
@@ -480,8 +472,6 @@ async def test_engine_core_client_util_method_nested_structures(
472
monkeypatch: pytest.MonkeyPatch,
473
):
474
with monkeypatch.context() as m:
483
- m.setenv("VLLM_USE_V1", "1")
484
-
475
# Must set insecure serialization to allow returning custom types.
476
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
477
@@ -592,76 +582,71 @@ async def test_engine_core_client_util_method_nested_structures(
582
indirect=["publisher_config"],
583
)
584
def test_kv_cache_events(
595
- monkeypatch: pytest.MonkeyPatch,
585
multiprocessing_mode: bool,
586
publisher_config,
587
):
599
- with monkeypatch.context() as m:
600
- m.setenv("VLLM_USE_V1", "1")
601
- block_size = 16
602
- num_blocks = 2
603
-
604
- engine_args = EngineArgs(
605
- model=MODEL_NAME,
606
- enforce_eager=True,
607
- enable_prefix_caching=True,
608
- block_size=block_size,
609
- )
610
- engine_args.kv_events_config = publisher_config
588
+ block_size = 16
589
+ num_blocks = 2
590
+
591
+ engine_args = EngineArgs(
592
+ model=MODEL_NAME,
593
+ enforce_eager=True,
594
+ enable_prefix_caching=True,
595
+ block_size=block_size,
596
+ )
597
+ engine_args.kv_events_config = publisher_config
598
612
- vllm_config = engine_args.create_engine_config(UsageContext.UNKNOWN_CONTEXT)
599
+ vllm_config = engine_args.create_engine_config(UsageContext.UNKNOWN_CONTEXT)
600
614
- executor_class = Executor.get_class(vllm_config)
615
- with set_default_torch_num_threads(1):
616
- client = EngineCoreClient.make_client(
617
- multiprocess_mode=multiprocessing_mode,
618
- asyncio_mode=False,
619
- vllm_config=vllm_config,
620
- executor_class=executor_class,
621
- log_stats=False,
622
- )
623
- endpoint = publisher_config.endpoint.replace("*", "127.0.0.1")
624
- subscriber = MockSubscriber(
625
- endpoint, topic=publisher_config.topic, decode_type=KVEventBatch
601
+ executor_class = Executor.get_class(vllm_config)
602
+ with set_default_torch_num_threads(1):
603
+ client = EngineCoreClient.make_client(
604
+ multiprocess_mode=multiprocessing_mode,
605
+ asyncio_mode=False,
606
+ vllm_config=vllm_config,
607
+ executor_class=executor_class,
608
+ log_stats=False,
609
)
610
+ endpoint = publisher_config.endpoint.replace("*", "127.0.0.1")
611
+ subscriber = MockSubscriber(
612
+ endpoint, topic=publisher_config.topic, decode_type=KVEventBatch
613
+ )
614
628
- try:
629
- custom_tokens = list(range(num_blocks * block_size))
630
- sampling_params = SamplingParams(max_tokens=1)
631
- request = make_request(sampling_params, custom_tokens)
632
- client.add_request(request)
615
+ try:
616
+ custom_tokens = list(range(num_blocks * block_size))
617
+ sampling_params = SamplingParams(max_tokens=1)
618
+ request = make_request(sampling_params, custom_tokens)
619
+ client.add_request(request)
620
634
- outputs: dict[str, list] = {request.request_id: []}
635
- loop_until_done(client, outputs)
621
+ outputs: dict[str, list] = {request.request_id: []}
622
+ loop_until_done(client, outputs)
623
637
- result = subscriber.receive_one(timeout=1000)
638
- assert result is not None, "No message received"
624
+ result = subscriber.receive_one(timeout=1000)
625
+ assert result is not None, "No message received"
626
640
- seq, received = result
627
+ seq, received = result
628
642
- assert seq == 0, "Sequence number mismatch"
643
- assert len(received.events) == 1, (
644
- "We should have exactly one BlockStored event"
645
- )
646
- event = received.events[0]
647
- assert isinstance(event, BlockStored), "We should have a BlockStored event"
648
- assert len(event.block_hashes) == num_blocks, (
649
- "We should have a BlockStored event with 2 block_hashes"
650
- )
651
- assert event.block_size == block_size, (
652
- "Block size should be the same as the block size"
653
- )
654
- assert event.parent_block_hash is None, "Parent block hash should be None"
655
- assert event.lora_id is None, "Lora id should be None"
656
- assert len(event.token_ids) == num_blocks * block_size, (
657
- "Token ids should be the same as the custom tokens"
658
- )
659
- assert event.token_ids == custom_tokens, (
660
- "Token ids should be the same as the custom tokens"
661
- )
662
- finally:
663
- client.shutdown()
664
- subscriber.close()
629
+ assert seq == 0, "Sequence number mismatch"
630
+ assert len(received.events) == 1, "We should have exactly one BlockStored event"
631
+ event = received.events[0]
632
+ assert isinstance(event, BlockStored), "We should have a BlockStored event"
633
+ assert len(event.block_hashes) == num_blocks, (
634
+ "We should have a BlockStored event with 2 block_hashes"
635
+ )
636
+ assert event.block_size == block_size, (
637
+ "Block size should be the same as the block size"
638
+ )
639
+ assert event.parent_block_hash is None, "Parent block hash should be None"
640
+ assert event.lora_id is None, "Lora id should be None"
641
+ assert len(event.token_ids) == num_blocks * block_size, (
642
+ "Token ids should be the same as the custom tokens"
643
+ )
644
+ assert event.token_ids == custom_tokens, (
645
+ "Token ids should be the same as the custom tokens"
646
+ )
647
+ finally:
648
+ client.shutdown()
649
+ subscriber.close()
650
651
652
@pytest.mark.asyncio
@@ -672,101 +657,96 @@ def test_kv_cache_events(
657
)
658
@multi_gpu_test(num_gpus=4)
659
async def test_kv_cache_events_dp(
675
- monkeypatch: pytest.MonkeyPatch,
660
multiprocessing_mode: bool,
661
publisher_config,
662
):
679
- with monkeypatch.context() as m:
680
- m.setenv("VLLM_USE_V1", "1")
681
- block_size = 16
682
- num_blocks = 2
683
- dp_size = 2
684
- tp_size = 2
685
-
686
- engine_args = EngineArgs(
687
- model=MODEL_NAME,
688
- enforce_eager=True,
689
- enable_prefix_caching=True,
690
- data_parallel_size=dp_size,
691
- tensor_parallel_size=tp_size,
692
- block_size=block_size,
693
- )
694
- engine_args.kv_events_config = publisher_config
663
+ block_size = 16
664
+ num_blocks = 2
665
+ dp_size = 2
666
+ tp_size = 2
667
+
668
+ engine_args = EngineArgs(
669
+ model=MODEL_NAME,
670
+ enforce_eager=True,
671
+ enable_prefix_caching=True,
672
+ data_parallel_size=dp_size,
673
+ tensor_parallel_size=tp_size,
674
+ block_size=block_size,
675
+ )
676
+ engine_args.kv_events_config = publisher_config
677
696
- vllm_config = engine_args.create_engine_config(UsageContext.UNKNOWN_CONTEXT)
678
+ vllm_config = engine_args.create_engine_config(UsageContext.UNKNOWN_CONTEXT)
679
698
- executor_class = Executor.get_class(vllm_config)
699
- with set_default_torch_num_threads(1):
700
- client = EngineCoreClient.make_client(
701
- multiprocess_mode=multiprocessing_mode,
702
- asyncio_mode=True,
703
- vllm_config=vllm_config,
704
- executor_class=executor_class,
705
- log_stats=False,
706
- )
707
- await asyncio.sleep(1)
680
+ executor_class = Executor.get_class(vllm_config)
681
+ with set_default_torch_num_threads(1):
682
+ client = EngineCoreClient.make_client(
683
+ multiprocess_mode=multiprocessing_mode,
684
+ asyncio_mode=True,
685
+ vllm_config=vllm_config,
686
+ executor_class=executor_class,
687
+ log_stats=False,
688
+ )
689
+ await asyncio.sleep(1)
690
709
- # Build endpoints for all DP ranks
710
- base_endpoint = publisher_config.endpoint.replace("*", "127.0.0.1")
711
- endpoints = []
712
- for i in range(dp_size):
713
- offset_endpoint = ZmqEventPublisher.offset_endpoint_port(base_endpoint, i)
714
- endpoints.append(offset_endpoint)
691
+ # Build endpoints for all DP ranks
692
+ base_endpoint = publisher_config.endpoint.replace("*", "127.0.0.1")
693
+ endpoints = []
694
+ for i in range(dp_size):
695
+ offset_endpoint = ZmqEventPublisher.offset_endpoint_port(base_endpoint, i)
696
+ endpoints.append(offset_endpoint)
697
716
- subscriber = MockSubscriber(
717
- endpoints, topic=publisher_config.topic, decode_type=KVEventBatch
718
- )
698
+ subscriber = MockSubscriber(
699
+ endpoints, topic=publisher_config.topic, decode_type=KVEventBatch
700
+ )
701
720
- try:
721
- custom_tokens = list(range(num_blocks * block_size))
722
- sampling_params = SamplingParams(max_tokens=1)
723
- all_request_ids = []
702
+ try:
703
+ custom_tokens = list(range(num_blocks * block_size))
704
+ sampling_params = SamplingParams(max_tokens=1)
705
+ all_request_ids = []
706
725
- # Create and add 25 requests
726
- # NOTE: attempts to force routing to both dp groups but can be flaky
727
- for i in range(25):
728
- await asyncio.sleep(0.01)
729
- request = make_request(sampling_params, custom_tokens)
730
- await client.add_request_async(request)
731
- all_request_ids.append(request.request_id)
707
+ # Create and add 25 requests
708
+ # NOTE: attempts to force routing to both dp groups but can be flaky
709
+ for i in range(25):
710
+ await asyncio.sleep(0.01)
711
+ request = make_request(sampling_params, custom_tokens)
712
+ await client.add_request_async(request)
713
+ all_request_ids.append(request.request_id)
714
733
- await asyncio.sleep(0.1)
715
+ await asyncio.sleep(0.1)
716
735
- # Initialize outputs dict for all requests
736
- outputs: dict[str, list] = {req_id: [] for req_id in all_request_ids}
717
+ # Initialize outputs dict for all requests
718
+ outputs: dict[str, list] = {req_id: [] for req_id in all_request_ids}
719
738
- print("processing requests...")
739
- await asyncio.wait_for(
740
- loop_until_fully_done_async(client, outputs), timeout=20.0
741
- )
720
+ print("processing requests...")
721
+ await asyncio.wait_for(
722
+ loop_until_fully_done_async(client, outputs), timeout=20.0
723
+ )
724
743
- # Receive from subscriber until no more messages
744
- print("collecting results...")
745
- results = []
746
- while True:
747
- result = subscriber.receive_one(timeout=1)
748
- print(result)
749
- if result is None:
750
- break
751
- results.append(result)
752
-
753
- # Collect all events and data_parallel_ranks from all results
754
- all_dp_ranks = [received.data_parallel_rank for (_, received) in results]
755
- unique_dps = set(all_dp_ranks)
756
- assert len(unique_dps) == 2, (
757
- f"Expected 2 unique data_parallel_ranks, got {len(unique_dps)}"
758
- )
725
+ # Receive from subscriber until no more messages
726
+ print("collecting results...")
727
+ results = []
728
+ while True:
729
+ result = subscriber.receive_one(timeout=1)
730
+ print(result)
731
+ if result is None:
732
+ break
733
+ results.append(result)
734
+
735
+ # Collect all events and data_parallel_ranks from all results
736
+ all_dp_ranks = [received.data_parallel_rank for (_, received) in results]
737
+ unique_dps = set(all_dp_ranks)
738
+ assert len(unique_dps) == 2, (
739
+ f"Expected 2 unique data_parallel_ranks, got {len(unique_dps)}"
740
+ )
741
760
- finally:
761
- client.shutdown()
762
- subscriber.close()
742
+ finally:
743
+ client.shutdown()
744
+ subscriber.close()
745
746
747
@pytest.mark.timeout(20)
748
def test_startup_failure(monkeypatch: pytest.MonkeyPatch):
749
with monkeypatch.context() as m, pytest.raises(Exception) as e_info:
768
- m.setenv("VLLM_USE_V1", "1")
769
-
750
# Monkey-patch to extract core process pid while it's starting.
751
core_proc_pid = [None]
752
cepm_ctor = CoreEngineProcManager.__init__
@@ -841,7 +821,6 @@ def test_engine_core_proc_instantiation_cuda_empty(monkeypatch: pytest.MonkeyPat
821
mock_executor_class.side_effect = create_mock_executor
822
823
with monkeypatch.context() as m:
844
- m.setenv("VLLM_USE_V1", "1")
824
m.setenv("CUDA_VISIBLE_DEVICES", "") # No CUDA devices
825
826
from vllm.v1.engine.utils import EngineZmqAddresses
tests/v1/engine/test_llm_engine.py
+8
-12
@@ -21,12 +21,10 @@ DTYPE = "half"
21
def _vllm_model(
22
apc: bool,
23
vllm_runner: type[VllmRunner],
24
- monkeypatch: pytest.MonkeyPatch,
24
*,
25
skip_tokenizer_init: bool = False,
26
):
27
"""Set up VllmRunner instance."""
29
- monkeypatch.setenv("VLLM_USE_V1", "1")
28
return vllm_runner(
29
MODEL,
30
dtype=DTYPE,
@@ -45,16 +43,16 @@ def _vllm_model(
43
# Prefix caching
44
params=[False, True],
45
)
48
-def vllm_model(vllm_runner, request, monkeypatch):
46
+def vllm_model(vllm_runner, request):
47
"""VllmRunner test fixture parameterized by APC True/False."""
50
- with _vllm_model(request.param, vllm_runner, monkeypatch) as vllm_model:
48
+ with _vllm_model(request.param, vllm_runner) as vllm_model:
49
yield vllm_model
50
51
52
@pytest.fixture(scope="function")
55
-def vllm_model_apc(vllm_runner, monkeypatch):
53
+def vllm_model_apc(vllm_runner):
54
"""VllmRunner test fixture with APC."""
57
- with _vllm_model(True, vllm_runner, monkeypatch) as vllm_model:
55
+ with _vllm_model(True, vllm_runner) as vllm_model:
56
yield vllm_model
57
58
@@ -65,12 +63,11 @@ def vllm_model_apc(vllm_runner, monkeypatch):
63
# Prefix caching
64
params=[False, True],
65
)
68
-def vllm_model_skip_tokenizer_init(vllm_runner, request, monkeypatch):
66
+def vllm_model_skip_tokenizer_init(vllm_runner, request):
67
"""VllmRunner test fixture with APC."""
68
with _vllm_model(
69
request.param,
70
vllm_runner,
73
- monkeypatch,
71
skip_tokenizer_init=True,
72
) as vllm_model:
73
yield vllm_model
@@ -152,7 +149,7 @@ def test_parallel_sampling(vllm_model, example_prompts) -> None:
149
)
150
151
155
-def test_engine_metrics(vllm_runner, monkeypatch, example_prompts):
152
+def test_engine_metrics(vllm_runner, example_prompts):
153
max_tokens = 100
154
# Use spec decoding to test num_accepted_tokens_per_pos
155
speculative_config = {
@@ -161,7 +158,7 @@ def test_engine_metrics(vllm_runner, monkeypatch, example_prompts):
158
"prompt_lookup_min": 3,
159
"num_speculative_tokens": 5,
160
}
164
- monkeypatch.setenv("VLLM_USE_V1", "1")
161
+
162
with vllm_runner(
163
MODEL,
164
speculative_config=speculative_config,
@@ -216,8 +213,7 @@ def test_engine_metrics(vllm_runner, monkeypatch, example_prompts):
213
214
215
@pytest.mark.parametrize("model", ["meta-llama/Llama-3.2-1B-Instruct"])
219
-def test_skip_tokenizer_initialization(model: str, monkeypatch: pytest.MonkeyPatch):
220
- monkeypatch.setenv("VLLM_USE_V1", "1")
216
+def test_skip_tokenizer_initialization(model: str):
217
# This test checks if the flag skip_tokenizer_init skips the initialization
218
# of tokenizer and detokenizer. The generated output is expected to contain
219
# token ids.
tests/v1/entrypoints/llm/test_struct_output_generate.py
+1
-15
@@ -103,7 +103,6 @@ def test_guided_decoding_deprecated():
103
PARAMS_MODELS_BACKENDS_TOKENIZER_MODE,
104
)
105
def test_structured_output(
106
- monkeypatch: pytest.MonkeyPatch,
106
sample_json_schema: dict[str, Any],
107
unsupported_json_schema: dict[str, Any],
108
sample_sql_ebnf: str,
@@ -115,8 +114,6 @@ def test_structured_output(
114
model_name: str,
115
speculative_config: dict[str, Any],
116
):
118
- monkeypatch.setenv("VLLM_USE_V1", "1")
119
-
117
if current_platform.is_tpu() and speculative_config:
118
pytest.skip("TPU does not support speculative decoding")
119
@@ -620,15 +617,12 @@ Make the response as short as possible.
617
],
618
)
619
def test_structured_output_with_reasoning_matrices(
623
- monkeypatch: pytest.MonkeyPatch,
620
backend: str,
621
tokenizer_mode: TokenizerMode,
622
reasoning_parser: str,
623
model_name: str,
624
speculative_config: dict[str, Any] | None,
625
):
630
- monkeypatch.setenv("VLLM_USE_V1", "1")
631
-
626
if current_platform.is_tpu() and speculative_config:
627
pytest.skip("TPU does not support speculative decoding")
628
@@ -691,13 +685,10 @@ def test_structured_output_with_reasoning_matrices(
685
@pytest.mark.skip_global_cleanup
686
@pytest.mark.parametrize("model_name, tokenizer_mode", PARAMS_MODELS_TOKENIZER_MODE)
687
def test_structured_output_auto_mode(
694
- monkeypatch: pytest.MonkeyPatch,
688
unsupported_json_schema: dict[str, Any],
689
model_name: str,
690
tokenizer_mode: str,
691
):
699
- monkeypatch.setenv("VLLM_USE_V1", "1")
700
-
692
llm = LLM(
693
model=model_name,
694
max_model_len=1024,
@@ -739,9 +730,7 @@ def test_structured_output_auto_mode(
730
731
732
@pytest.mark.skip_global_cleanup
742
-def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
743
- monkeypatch.setenv("VLLM_USE_V1", "1")
744
-
733
+def test_guidance_no_additional_properties():
734
llm = LLM(
735
model="Qwen/Qwen2.5-1.5B-Instruct",
736
max_model_len=1024,
@@ -801,12 +790,9 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
790
791
@pytest.mark.parametrize("backend", ["guidance", "xgrammar", "outlines"])
792
def test_structured_output_batched_with_non_structured_outputs_requests(
804
- monkeypatch: pytest.MonkeyPatch,
793
sample_json_schema: dict[str, Any],
794
backend: str,
795
):
808
- monkeypatch.setenv("VLLM_USE_V1", "1")
809
-
796
# Don't use eager execution on TPUs because we want to test for no
797
# recompilation at runtime
798
enforce_eager = bool(not current_platform.is_tpu())
tests/v1/kv_connector/nixl_integration/run_tpu_disagg_accuracy_test.sh
-3
@@ -53,7 +53,6 @@ cleanup() {
53
launch_baseline() {
54
BASELINE_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME};
55
VLLM_LOGGING_LEVEL=DEBUG \
56
- VLLM_USE_V1=1 \
56
PJRT_DEVICE=TPU \
57
VLLM_WORKER_MULTIPROC_METHOD=spawn \
58
VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \
@@ -73,7 +72,6 @@ launch_pd() {
72
UCX_TLS=tcp \
73
VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
74
VLLM_LOGGING_LEVEL=DEBUG \
76
- VLLM_USE_V1=1 \
75
VLLM_NIXL_SIDE_CHANNEL_HOST=${PREFILL_HOST} \
76
VLLM_NIXL_SIDE_CHANNEL_PORT=${PREFILL_NIXL_SIDE_PORT} \
77
PJRT_DEVICE=TPU \
@@ -93,7 +91,6 @@ launch_pd() {
91
UCX_TLS=tcp \
92
VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
93
VLLM_LOGGING_LEVEL=DEBUG \
96
- VLLM_USE_V1=1 \
94
PJRT_DEVICE=TPU \
95
VLLM_WORKER_MULTIPROC_METHOD=spawn \
96
VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \
tests/v1/kv_connector/nixl_integration/run_tpu_edge_case_test.sh
-2
@@ -55,7 +55,6 @@ launch_pd() {
55
UCX_TLS=tcp \
56
VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
57
VLLM_LOGGING_LEVEL=DEBUG \
58
- VLLM_USE_V1=1 \
58
VLLM_NIXL_SIDE_CHANNEL_HOST=${PREFILL_HOST} \
59
VLLM_NIXL_SIDE_CHANNEL_PORT=${PREFILL_NIXL_SIDE_PORT} \
60
PJRT_DEVICE=TPU \
@@ -75,7 +74,6 @@ launch_pd() {
74
UCX_TLS=tcp \
75
VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
76
VLLM_LOGGING_LEVEL=DEBUG \
78
- VLLM_USE_V1=1 \
77
PJRT_DEVICE=TPU \
78
VLLM_WORKER_MULTIPROC_METHOD=spawn \
79
VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \
tests/v1/metrics/test_ray_metrics.py
-14
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import os
3
4
import pytest
5
import ray
@@ -10,15 +9,6 @@ from vllm.sampling_params import SamplingParams
9
from vllm.v1.engine.async_llm import AsyncEngineArgs, AsyncLLM
10
from vllm.v1.metrics.ray_wrappers import RayPrometheusMetric, RayPrometheusStatLogger
11
13
-
14
-@pytest.fixture(scope="function", autouse=True)
15
-def use_v1_only(monkeypatch):
16
- """
17
- The change relies on V1 APIs, so set VLLM_USE_V1=1.
18
- """
19
- monkeypatch.setenv("VLLM_USE_V1", "1")
20
-
21
-
12
MODELS = [
13
"distilbert/distilgpt2",
14
]
@@ -39,10 +29,6 @@ def test_engine_log_metrics_ray(
29
@ray.remote(num_gpus=1)
30
class EngineTestActor:
31
async def run(self):
42
- # Set environment variable inside the Ray actor since environment
43
- # variables from pytest fixtures don't propagate to Ray actors
44
- os.environ["VLLM_USE_V1"] = "1"
45
-
32
engine_args = AsyncEngineArgs(
33
model=model, dtype=dtype, disable_log_stats=False, enforce_eager=True
34
)
tests/v1/sample/test_logprobs.py
+154
-171
@@ -280,7 +280,6 @@ def test_get_logprobs_and_prompt_logprobs(
280
batch_logprobs_composition: BatchLogprobsComposition,
281
temperature: float,
282
example_prompts: list[str],
283
- monkeypatch: pytest.MonkeyPatch,
283
) -> None:
284
"""Test V1 Engine logprobs & prompt logprobs
285
@@ -308,220 +307,204 @@ def test_get_logprobs_and_prompt_logprobs(
307
temperature: "temperature" sampling parameter
308
example_prompts: example prompt fixture
309
"""
311
- with monkeypatch.context() as m:
312
- m.setenv("VLLM_USE_V1", "1")
313
- do_apc = vllm_model.llm.llm_engine.cache_config.enable_prefix_caching
314
- if do_apc and (
315
- temperature < 2.0 or batch_logprobs_composition != SAMPLE_PROMPT
316
- ):
317
- # Skip some test-cases to save time.
318
- pytest.skip()
319
- test_prompts = example_prompts
320
-
321
- max_tokens = 5
322
- hf_outputs = hf_model.generate_greedy(
323
- test_prompts,
310
+ do_apc = vllm_model.llm.llm_engine.cache_config.enable_prefix_caching
311
+ if do_apc and (temperature < 2.0 or batch_logprobs_composition != SAMPLE_PROMPT):
312
+ # Skip some test-cases to save time.
313
+ pytest.skip()
314
+ test_prompts = example_prompts
315
+
316
+ max_tokens = 5
317
+ hf_outputs = hf_model.generate_greedy(
318
+ test_prompts,
319
+ max_tokens=max_tokens,
320
+ )
321
+ hf_logprobs = hf_model.generate_greedy_logprobs(
322
+ test_prompts,
323
+ max_tokens=max_tokens,
324
+ )
325
+
326
+ # Batch has mixed sample params
327
+ # (different logprobs/prompt logprobs combos)
328
+ logprob_prompt_logprob_list = get_test_batch(batch_logprobs_composition)
329
+
330
+ # Ensure that each test prompt has a logprob config for testing
331
+ logprob_prompt_logprob_list = _repeat_logprob_config(
332
+ test_prompts, logprob_prompt_logprob_list
333
+ )
334
+ # Generate SamplingParams
335
+ vllm_sampling_params = [
336
+ SamplingParams(
337
max_tokens=max_tokens,
338
+ logprobs=num_lp,
339
+ prompt_logprobs=num_plp,
340
+ temperature=temperature,
341
+ seed=1984,
342
)
326
- hf_logprobs = hf_model.generate_greedy_logprobs(
327
- test_prompts,
343
+ for num_lp, num_plp in logprob_prompt_logprob_list
344
+ ]
345
+ for _ in range(2 if do_apc else 1):
346
+ _run_and_validate(
347
+ vllm_model=vllm_model,
348
+ test_prompts=test_prompts,
349
+ vllm_sampling_params=vllm_sampling_params,
350
+ hf_logprobs=hf_logprobs,
351
+ hf_outputs=hf_outputs,
352
+ logprob_prompt_logprob_list=logprob_prompt_logprob_list,
353
+ temperature=temperature,
354
max_tokens=max_tokens,
355
+ do_apc=do_apc,
356
)
357
331
- # Batch has mixed sample params
332
- # (different logprobs/prompt logprobs combos)
333
- logprob_prompt_logprob_list = get_test_batch(batch_logprobs_composition)
358
335
- # Ensure that each test prompt has a logprob config for testing
336
- logprob_prompt_logprob_list = _repeat_logprob_config(
337
- test_prompts, logprob_prompt_logprob_list
338
- )
339
- # Generate SamplingParams
340
- vllm_sampling_params = [
341
- SamplingParams(
342
- max_tokens=max_tokens,
343
- logprobs=num_lp,
344
- prompt_logprobs=num_plp,
345
- temperature=temperature,
346
- seed=1984,
347
- )
348
- for num_lp, num_plp in logprob_prompt_logprob_list
349
- ]
350
- for _ in range(2 if do_apc else 1):
351
- _run_and_validate(
352
- vllm_model=vllm_model,
353
- test_prompts=test_prompts,
354
- vllm_sampling_params=vllm_sampling_params,
355
- hf_logprobs=hf_logprobs,
356
- hf_outputs=hf_outputs,
357
- logprob_prompt_logprob_list=logprob_prompt_logprob_list,
358
- temperature=temperature,
359
- max_tokens=max_tokens,
360
- do_apc=do_apc,
361
- )
362
-
363
-
364
-def test_max_logprobs(monkeypatch: pytest.MonkeyPatch):
359
+def test_max_logprobs():
360
"""vLLM v1 engine should fail a request with `logprobs > max_logprobs`
361
Should also fail for `prompt_logprobs > max_logprobs`
362
APC should not matter as this test checks basic request validation.
363
"""
369
- with monkeypatch.context() as m:
370
- m.setenv("VLLM_USE_V1", "1")
371
-
372
- runner = VllmRunner(
373
- "facebook/opt-125m",
374
- max_logprobs=1,
375
- enable_prefix_caching=False,
376
- # 2 other llms alive during whole session
377
- gpu_memory_utilization=0.15,
378
- max_model_len=256,
379
- )
380
- vllm_sampling_params = SamplingParams(logprobs=1)
381
- # should pass
382
- runner.generate(["Hello world"], sampling_params=vllm_sampling_params)
364
+ runner = VllmRunner(
365
+ "facebook/opt-125m",
366
+ max_logprobs=1,
367
+ enable_prefix_caching=False,
368
+ # 2 other llms alive during whole session
369
+ gpu_memory_utilization=0.15,
370
+ max_model_len=256,
371
+ )
372
+ vllm_sampling_params = SamplingParams(logprobs=1)
373
+ # should pass
374
+ runner.generate(["Hello world"], sampling_params=vllm_sampling_params)
375
384
- bad_sampling_params = SamplingParams(logprobs=2)
385
- with pytest.raises(ValueError):
386
- runner.generate(["Hello world"], sampling_params=bad_sampling_params)
376
+ bad_sampling_params = SamplingParams(logprobs=2)
377
+ with pytest.raises(ValueError):
378
+ runner.generate(["Hello world"], sampling_params=bad_sampling_params)
379
380
389
-def test_none_logprobs(vllm_model, example_prompts, monkeypatch: pytest.MonkeyPatch):
381
+def test_none_logprobs(vllm_model, example_prompts):
382
"""Engine should return `logprobs` and `prompt_logprobs` as `None`
383
384
Args:
385
vllm_model: vLLM model fixture
386
example_prompts: list of example prompts (test fixture)
387
"""
396
- with monkeypatch.context() as m:
397
- m.setenv("VLLM_USE_V1", "1")
398
- max_tokens = 5
388
+ max_tokens = 5
389
400
- sampling_params_logprobs_none = SamplingParams(
401
- max_tokens=max_tokens,
402
- logprobs=None,
403
- prompt_logprobs=None,
404
- temperature=0.0,
405
- )
406
- results_logprobs_none = vllm_model.llm.generate(
407
- example_prompts,
408
- sampling_params=sampling_params_logprobs_none,
409
- )
390
+ sampling_params_logprobs_none = SamplingParams(
391
+ max_tokens=max_tokens,
392
+ logprobs=None,
393
+ prompt_logprobs=None,
394
+ temperature=0.0,
395
+ )
396
+ results_logprobs_none = vllm_model.llm.generate(
397
+ example_prompts,
398
+ sampling_params=sampling_params_logprobs_none,
399
+ )
400
411
- for i in range(len(results_logprobs_none)):
412
- # Check sample logprobs are None
413
- assert results_logprobs_none[i].outputs[0].logprobs is None
414
- assert results_logprobs_none[i].outputs[0].cumulative_logprob is None
415
- # Check prompt logprobs are None
416
- assert results_logprobs_none[i].prompt_logprobs is None
401
+ for i in range(len(results_logprobs_none)):
402
+ # Check sample logprobs are None
403
+ assert results_logprobs_none[i].outputs[0].logprobs is None
404
+ assert results_logprobs_none[i].outputs[0].cumulative_logprob is None
405
+ # Check prompt logprobs are None
406
+ assert results_logprobs_none[i].prompt_logprobs is None
407
408
419
-def test_zero_logprobs(vllm_model, example_prompts, monkeypatch: pytest.MonkeyPatch):
409
+def test_zero_logprobs(vllm_model, example_prompts):
410
"""Engine should return sampled token and prompt token logprobs
411
412
Args:
413
vllm_model: vLLM model fixture
414
example_prompts: list of example prompts (test fixture)
415
"""
426
- with monkeypatch.context() as m:
427
- m.setenv("VLLM_USE_V1", "1")
428
- max_tokens = 5
416
+ max_tokens = 5
417
430
- sampling_params_logprobs_zero = SamplingParams(
431
- max_tokens=max_tokens, logprobs=0, prompt_logprobs=0, temperature=0.0
432
- )
433
- results_logprobs_zero = vllm_model.llm.generate(
434
- example_prompts, sampling_params=sampling_params_logprobs_zero
435
- )
418
+ sampling_params_logprobs_zero = SamplingParams(
419
+ max_tokens=max_tokens, logprobs=0, prompt_logprobs=0, temperature=0.0
420
+ )
421
+ results_logprobs_zero = vllm_model.llm.generate(
422
+ example_prompts, sampling_params=sampling_params_logprobs_zero
423
+ )
424
437
- for i in range(len(results_logprobs_zero)):
438
- # Check that there is one sample logprob dict for each
439
- # sample token
440
- logprobs = results_logprobs_zero[i].outputs[0].logprobs
441
- prompt_logprobs = results_logprobs_zero[i].prompt_logprobs
442
- sampled_token_ids = results_logprobs_zero[i].outputs[0].token_ids
443
- prompt_token_ids = results_logprobs_zero[i].prompt_token_ids
444
- assert logprobs is not None
445
- assert len(sampled_token_ids) == len(logprobs)
446
- assert results_logprobs_zero[i].outputs[0].cumulative_logprob is not None
447
- # Check that there is one prompt logprob dict for each
448
- # prompt token
449
- assert prompt_logprobs is not None
450
- assert len(prompt_token_ids) == len(prompt_logprobs)
451
-
452
-
453
-def test_all_logprobs(example_prompts, monkeypatch: pytest.MonkeyPatch):
425
+ for i in range(len(results_logprobs_zero)):
426
+ # Check that there is one sample logprob dict for each
427
+ # sample token
428
+ logprobs = results_logprobs_zero[i].outputs[0].logprobs
429
+ prompt_logprobs = results_logprobs_zero[i].prompt_logprobs
430
+ sampled_token_ids = results_logprobs_zero[i].outputs[0].token_ids
431
+ prompt_token_ids = results_logprobs_zero[i].prompt_token_ids
432
+ assert logprobs is not None
433
+ assert len(sampled_token_ids) == len(logprobs)
434
+ assert results_logprobs_zero[i].outputs[0].cumulative_logprob is not None
435
+ # Check that there is one prompt logprob dict for each
436
+ # prompt token
437
+ assert prompt_logprobs is not None
438
+ assert len(prompt_token_ids) == len(prompt_logprobs)
439
+
440
+
441
+def test_all_logprobs(example_prompts):
442
"""Engine should return all vocabulary logprobs and prompt logprobs
443
444
Args:
445
example_prompts: list of example prompts (test fixture)
446
"""
459
- with monkeypatch.context() as m:
460
- m.setenv("VLLM_USE_V1", "1")
461
- runner = VllmRunner(
462
- "facebook/opt-125m",
463
- max_logprobs=-1,
464
- enable_prefix_caching=False,
465
- # 2 other llms alive during whole session
466
- gpu_memory_utilization=0.15,
467
- max_model_len=256,
468
- )
447
+ runner = VllmRunner(
448
+ "facebook/opt-125m",
449
+ max_logprobs=-1,
450
+ enable_prefix_caching=False,
451
+ # 2 other llms alive during whole session
452
+ gpu_memory_utilization=0.15,
453
+ max_model_len=256,
454
+ )
455
470
- sampling_params_logprobs_all = SamplingParams(
471
- max_tokens=5, logprobs=-1, prompt_logprobs=-1
472
- )
473
- results_logprobs_all = runner.llm.generate(
474
- example_prompts, sampling_params=sampling_params_logprobs_all
475
- )
476
- vocab_size = runner.llm.llm_engine.get_model_config().get_vocab_size()
456
+ sampling_params_logprobs_all = SamplingParams(
457
+ max_tokens=5, logprobs=-1, prompt_logprobs=-1
458
+ )
459
+ results_logprobs_all = runner.llm.generate(
460
+ example_prompts, sampling_params=sampling_params_logprobs_all
461
+ )
462
+ vocab_size = runner.llm.llm_engine.get_model_config().get_vocab_size()
463
478
- for i in range(len(results_logprobs_all)):
479
- logprobs = results_logprobs_all[i].outputs[0].logprobs
480
- prompt_logprobs = results_logprobs_all[i].prompt_logprobs
481
- assert logprobs is not None
482
- for logprob in logprobs:
483
- assert len(logprob) == vocab_size
484
- assert prompt_logprobs is not None
485
- assert prompt_logprobs[0] is None
486
- for prompt_logprob in prompt_logprobs[1:]:
487
- assert len(prompt_logprob) == vocab_size
464
+ for i in range(len(results_logprobs_all)):
465
+ logprobs = results_logprobs_all[i].outputs[0].logprobs
466
+ prompt_logprobs = results_logprobs_all[i].prompt_logprobs
467
+ assert logprobs is not None
468
+ for logprob in logprobs:
469
+ assert len(logprob) == vocab_size
470
+ assert prompt_logprobs is not None
471
+ assert prompt_logprobs[0] is None
472
+ for prompt_logprob in prompt_logprobs[1:]:
473
+ assert len(prompt_logprob) == vocab_size
474
475
476
@pytest.mark.parametrize("logprobs_mode", get_args(LogprobsMode))
491
-def test_logprobs_mode(logprobs_mode: LogprobsMode, monkeypatch: pytest.MonkeyPatch):
477
+def test_logprobs_mode(logprobs_mode: LogprobsMode):
478
"""Test with LLM engine with different logprobs_mode.
479
For logprobs, we should have non-positive values.
480
For logits, we should expect at least one positive values.
481
"""
482
from vllm import LLM
483
498
- with monkeypatch.context() as m:
499
- m.setenv("VLLM_USE_V1", "1")
500
-
501
- llm = LLM(
502
- "facebook/opt-125m",
503
- max_logprobs=5,
504
- enable_prefix_caching=False,
505
- # 2 other llms alive during whole session
506
- gpu_memory_utilization=0.05,
507
- max_model_len=16,
508
- logprobs_mode=logprobs_mode,
509
- )
510
- vllm_sampling_params = SamplingParams(logprobs=1)
511
- results = llm.generate(["Hello world"], sampling_params=vllm_sampling_params)
512
-
513
- total_token_with_logprobs = 0
514
- positive_values = 0
515
- for output in results[0].outputs:
516
- for logprobs in output.logprobs:
517
- for token_id in logprobs:
518
- logprob = logprobs[token_id]
519
- if logprobs_mode in ("raw_logprobs", "processed_logprobs"):
520
- assert logprob.logprob <= 0
521
- if logprob.logprob > 0:
522
- positive_values = positive_values + 1
523
- total_token_with_logprobs = total_token_with_logprobs + 1
524
- assert total_token_with_logprobs >= len(results[0].outputs)
525
- if logprobs_mode in ("raw_logits", "processed_logits"):
526
- assert positive_values > 0
527
- del llm
484
+ llm = LLM(
485
+ "facebook/opt-125m",
486
+ max_logprobs=5,
487
+ enable_prefix_caching=False,
488
+ # 2 other llms alive during whole session
489
+ gpu_memory_utilization=0.05,
490
+ max_model_len=16,
491
+ logprobs_mode=logprobs_mode,
492
+ )
493
+ vllm_sampling_params = SamplingParams(logprobs=1)
494
+ results = llm.generate(["Hello world"], sampling_params=vllm_sampling_params)
495
+
496
+ total_token_with_logprobs = 0
497
+ positive_values = 0
498
+ for output in results[0].outputs:
499
+ for logprobs in output.logprobs:
500
+ for token_id in logprobs:
501
+ logprob = logprobs[token_id]
502
+ if logprobs_mode in ("raw_logprobs", "processed_logprobs"):
503
+ assert logprob.logprob <= 0
504
+ if logprob.logprob > 0:
505
+ positive_values = positive_values + 1
506
+ total_token_with_logprobs = total_token_with_logprobs + 1
507
+ assert total_token_with_logprobs >= len(results[0].outputs)
508
+ if logprobs_mode in ("raw_logits", "processed_logits"):
509
+ assert positive_values > 0
510
+ del llm
tests/v1/sample/test_sampling_params_e2e.py
-12
@@ -1,14 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import os
3
4
import pytest
5
6
from vllm import LLM, SamplingParams
7
9
-if os.getenv("VLLM_USE_V1", "0") != "1":
10
- pytest.skip("Test package requires V1", allow_module_level=True)
11
-
8
MODEL = "meta-llama/Llama-3.2-1B"
9
PROMPT = "Hello my name is Robert and I"
10
@@ -173,14 +169,6 @@ def test_allowed_token_ids(llm):
169
_ = llm.generate(PROMPT, SamplingParams(allowed_token_ids=[10000000]))
170
171
176
-def test_priority(llm):
177
- """Check that we reject requests with priority."""
178
-
179
- # Reject all allowed token ids
180
- with pytest.raises(ValueError):
181
- _ = llm.generate(PROMPT, priority=[1])
182
-
183
-
172
def test_seed(llm):
173
"""Check that seed impacts randomness."""
174
tests/v1/spec_decode/test_max_len.py
-1
@@ -38,7 +38,6 @@ def test_eagle_max_len(
38
monkeypatch: pytest.MonkeyPatch, num_speculative_tokens: int, attn_backend: str
39
):
40
with monkeypatch.context() as m:
41
- m.setenv("VLLM_USE_V1", "1")
41
m.setenv("VLLM_ATTENTION_BACKEND", attn_backend)
42
43
if attn_backend == "TRITON_ATTN" and not current_platform.is_rocm():
tests/v1/tpu/test_basic.py
+46
-62
@@ -42,7 +42,6 @@ MAX_NUM_REQS = [16, 1024]
42
@pytest.mark.parametrize("max_num_seqs", MAX_NUM_REQS)
43
def test_basic(
44
vllm_runner: type[VllmRunner],
45
- monkeypatch: pytest.MonkeyPatch,
45
model: str,
46
max_tokens: int,
47
tensor_parallel_size: int,
@@ -55,23 +54,20 @@ def test_basic(
54
)
55
example_prompts = [prompt]
56
58
- with monkeypatch.context() as m:
59
- m.setenv("VLLM_USE_V1", "1")
57
+ with vllm_runner(
58
+ model,
59
+ # Note: max_num_batched_tokens == 1024 is needed here to
60
+ # actually test chunked prompt
61
+ max_num_batched_tokens=1024,
62
+ max_model_len=8192,
63
+ gpu_memory_utilization=0.7,
64
+ max_num_seqs=max_num_seqs,
65
+ tensor_parallel_size=tensor_parallel_size,
66
+ ) as vllm_model:
67
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
68
+ output = vllm_outputs[0][1]
69
61
- with vllm_runner(
62
- model,
63
- # Note: max_num_batched_tokens == 1024 is needed here to
64
- # actually test chunked prompt
65
- max_num_batched_tokens=1024,
66
- max_model_len=8192,
67
- gpu_memory_utilization=0.7,
68
- max_num_seqs=max_num_seqs,
69
- tensor_parallel_size=tensor_parallel_size,
70
- ) as vllm_model:
71
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
72
- output = vllm_outputs[0][1]
73
-
74
- assert "1024" in output or "0, 1" in output
70
+ assert "1024" in output or "0, 1" in output
71
72
73
@pytest.mark.skip(reason="Temporarily disabled due to timeout")
@@ -82,7 +78,6 @@ def test_basic(
78
@pytest.mark.parametrize("max_num_seqs", [16])
79
def test_phi3(
80
vllm_runner: type[VllmRunner],
85
- monkeypatch: pytest.MonkeyPatch,
81
max_tokens: int,
82
max_num_seqs: int,
83
) -> None:
@@ -99,18 +94,15 @@ def test_phi3(
94
# test head dim = 96
95
model = "microsoft/Phi-3-mini-128k-instruct"
96
102
- with monkeypatch.context() as m:
103
- m.setenv("VLLM_USE_V1", "1")
104
-
105
- with vllm_runner(
106
- model, max_num_batched_tokens=256, max_num_seqs=max_num_seqs
107
- ) as vllm_model:
108
- vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
109
- # vllm_outputs is a list of tuples whose first element is the token id
110
- # and the second element is the output (including the prompt).
111
- for output, answer in zip(vllm_outputs, answers):
112
- generated_text = output[1]
113
- assert answer in generated_text
97
+ with vllm_runner(
98
+ model, max_num_batched_tokens=256, max_num_seqs=max_num_seqs
99
+ ) as vllm_model:
100
+ vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
101
+ # vllm_outputs is a list of tuples whose first element is the token id
102
+ # and the second element is the output (including the prompt).
103
+ for output, answer in zip(vllm_outputs, answers):
104
+ generated_text = output[1]
105
+ assert answer in generated_text
106
107
108
TP_SIZE_8 = 8
@@ -123,7 +115,6 @@ TP_SIZE_8 = 8
115
)
116
def test_gemma3_27b_with_text_input_and_tp(
117
vllm_runner: type[VllmRunner],
126
- monkeypatch: pytest.MonkeyPatch,
118
) -> None:
119
model = "google/gemma-3-27b-it"
120
max_tokens = 16
@@ -140,21 +131,18 @@ def test_gemma3_27b_with_text_input_and_tp(
131
" but in rising every time we fall.",
132
]
133
143
- with monkeypatch.context() as m:
144
- m.setenv("VLLM_USE_V1", "1")
145
-
146
- with vllm_runner(
147
- model,
148
- max_num_batched_tokens=256,
149
- max_num_seqs=max_num_seqs,
150
- tensor_parallel_size=tensor_parallel_size,
151
- ) as vllm_model:
152
- vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
153
- # vllm_outputs is a list of tuples whose first element is the token id
154
- # and the second element is the output (including the prompt).
155
- for output, answer in zip(vllm_outputs, answers):
156
- generated_text = output[1]
157
- assert answer in generated_text
134
+ with vllm_runner(
135
+ model,
136
+ max_num_batched_tokens=256,
137
+ max_num_seqs=max_num_seqs,
138
+ tensor_parallel_size=tensor_parallel_size,
139
+ ) as vllm_model:
140
+ vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
141
+ # vllm_outputs is a list of tuples whose first element is the token id
142
+ # and the second element is the output (including the prompt).
143
+ for output, answer in zip(vllm_outputs, answers):
144
+ generated_text = output[1]
145
+ assert answer in generated_text
146
147
148
@pytest.mark.skipif(
@@ -162,7 +150,6 @@ def test_gemma3_27b_with_text_input_and_tp(
150
)
151
def test_w8a8_quantization(
152
vllm_runner: type[VllmRunner],
165
- monkeypatch: pytest.MonkeyPatch,
153
) -> None:
154
model = "neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8"
155
max_tokens = 5
@@ -176,18 +163,15 @@ def test_w8a8_quantization(
163
)
164
example_prompts = [prompt]
165
179
- with monkeypatch.context() as m:
180
- m.setenv("VLLM_USE_V1", "1")
181
-
182
- with vllm_runner(
183
- model,
184
- max_num_batched_tokens=64,
185
- max_model_len=4096,
186
- gpu_memory_utilization=0.7,
187
- max_num_seqs=max_num_seqs,
188
- tensor_parallel_size=tensor_parallel_size,
189
- ) as vllm_model:
190
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
191
- output = vllm_outputs[0][1]
192
-
193
- assert "1024" in output or "0, 1" in output
166
+ with vllm_runner(
167
+ model,
168
+ max_num_batched_tokens=64,
169
+ max_model_len=4096,
170
+ gpu_memory_utilization=0.7,
171
+ max_num_seqs=max_num_seqs,
172
+ tensor_parallel_size=tensor_parallel_size,
173
+ ) as vllm_model:
174
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
175
+ output = vllm_outputs[0][1]
176
+
177
+ assert "1024" in output or "0, 1" in output
tests/v1/tpu/test_perf.py
+38
-42
@@ -86,7 +86,6 @@ GPU_UTIL = 0.9
86
@pytest.mark.parametrize("params", TEST_PARAMS)
87
def test_perf(
88
vllm_runner: type[VllmRunner],
89
- monkeypatch: pytest.MonkeyPatch,
89
params: TestParams,
90
) -> None:
91
tokenizer = get_tokenizer(
@@ -107,48 +106,45 @@ def test_perf(
106
)
107
)
108
110
- with monkeypatch.context() as m:
111
- m.setenv("VLLM_USE_V1", "1")
109
+ sampling_params = SamplingParams(
110
+ max_tokens=params.decode_len, temperature=1.0, min_p=0.0
111
+ )
112
113
- sampling_params = SamplingParams(
114
- max_tokens=params.decode_len, temperature=1.0, min_p=0.0
113
+ with vllm_runner(
114
+ params.model,
115
+ max_num_batched_tokens=MAX_MODEL_LEN,
116
+ max_model_len=MAX_MODEL_LEN,
117
+ max_num_seqs=MAX_NUM_SEQS,
118
+ gpu_memory_utilization=GPU_UTIL,
119
+ enforce_eager=False,
120
+ tensor_parallel_size=1,
121
+ ) as vllm_model:
122
+ print(" -- Warmup / Compile")
123
+ for i in range(NUM_WARMUPS):
124
+ _ = vllm_model.generate(prompts, sampling_params)
125
+
126
+ print(" -- Benchmarking... ")
127
+ times = []
128
+ for i in range(NUM_RUNS):
129
+ start_time = time.time()
130
+ _ = vllm_model.generate(prompts, sampling_params)
131
+ times.append(time.time() - start_time)
132
+
133
+ avg_time = sum(times) / len(times)
134
+
135
+ print(" -- avg_time = {}".format(avg_time))
136
+ print(
137
+ " -- expected_avg_time = {} with err_tol = {}".format(
138
+ params.expected_avg_time, params.err_tol
139
+ )
140
)
116
-
117
- with vllm_runner(
118
- params.model,
119
- max_num_batched_tokens=MAX_MODEL_LEN,
120
- max_model_len=MAX_MODEL_LEN,
121
- max_num_seqs=MAX_NUM_SEQS,
122
- gpu_memory_utilization=GPU_UTIL,
123
- enforce_eager=False,
124
- tensor_parallel_size=1,
125
- ) as vllm_model:
126
- print(" -- Warmup / Compile")
127
- for i in range(NUM_WARMUPS):
128
- _ = vllm_model.generate(prompts, sampling_params)
129
-
130
- print(" -- Benchmarking... ")
131
- times = []
132
- for i in range(NUM_RUNS):
133
- start_time = time.time()
134
- _ = vllm_model.generate(prompts, sampling_params)
135
- times.append(time.time() - start_time)
136
-
137
- avg_time = sum(times) / len(times)
138
-
139
- print(" -- avg_time = {}".format(avg_time))
141
+ diff = avg_time - params.expected_avg_time
142
+ ok = diff < params.err_tol
143
+ if diff < -params.err_tol:
144
print(
141
- " -- expected_avg_time = {} with err_tol = {}".format(
142
- params.expected_avg_time, params.err_tol
143
- )
145
+ " !! WARNING !! Performance has improved by {}, "
146
+ "it may be necessary to fine-tune the "
147
+ "expected_avg_time = {}".format(-diff, params.expected_avg_time)
148
)
145
- diff = avg_time - params.expected_avg_time
146
- ok = diff < params.err_tol
147
- if diff < -params.err_tol:
148
- print(
149
- " !! WARNING !! Performance has improved by {}, "
150
- "it may be necessary to fine-tune the "
151
- "expected_avg_time = {}".format(-diff, params.expected_avg_time)
152
- )
153
-
154
- assert ok, " !! ERROR !! Regression detected"
149
+
150
+ assert ok, " !! ERROR !! Regression detected"
tests/v1/tracing/test_tracing.py
+1
-1
@@ -82,7 +82,7 @@ def test_traces(
82
):
83
with monkeypatch.context() as m:
84
m.setenv(OTEL_EXPORTER_OTLP_TRACES_INSECURE, "true")
85
- m.setenv("VLLM_USE_V1", "1")
85
+
86
sampling_params = SamplingParams(
87
temperature=0.01,
88
top_p=0.1,
vllm/v1/worker/cpu_model_runner.py
+7
-1
@@ -77,7 +77,13 @@ class CPUModelRunner(GPUModelRunner):
77
logger.info("Warming up model for the compilation...")
78
# Only generate graph for the generic shape
79
with _set_global_compilation_settings(self.vllm_config):
80
- self._dummy_run(max(16, self.max_num_reqs))
80
+ self._dummy_run(
81
+ min(
82
+ max(16, self.max_num_reqs),
83
+ self.scheduler_config.max_num_batched_tokens,
84
+ )
85
+ )
86
+
87
logger.info("Warming up done.")
88
89
def _init_device_properties(self) -> None: