A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Core] Deprecating block manager v1 and make block manager v2 default (#8704)
Removing the block manager v1. This is the initial piece of prefix-caching-centric design. In order to achieve prefix-caching-centric design, we need to simplify the code path so that we only use v2 block manager (which has much higher performance on prefix caching).
Kuntai Du committed
Oct 17, 2024 at 11:38 UTC
81ede99ca44a5b3518932a07ea4a76a719e7416e
45 files changed
+206
-2109
.buildkite/test-pipeline.yaml
+6
-12
@@ -77,8 +77,8 @@ steps:
77
- vllm/
78
- tests/basic_correctness/test_chunked_prefill
79
commands:
80
- - VLLM_ATTENTION_BACKEND=XFORMERS VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s basic_correctness/test_chunked_prefill.py
81
- - VLLM_ATTENTION_BACKEND=FLASH_ATTN VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s basic_correctness/test_chunked_prefill.py
80
+ - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
81
+ - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
82
83
- label: Core Test # 10min
84
mirror_hardwares: [amd]
@@ -88,11 +88,7 @@ steps:
88
- vllm/distributed
89
- tests/core
90
commands:
91
- - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core/test_scheduler.py
92
- - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/test_chunked_prefill_scheduler.py
93
- - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/block/e2e/test_correctness.py
94
- - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/block/e2e/test_correctness_sliding_window.py
95
- - pytest -v -s core --ignore=core/block/e2e/test_correctness.py --ignore=core/test_scheduler.py --ignore=core/test_chunked_prefill_scheduler.py --ignore=core/block/e2e/test_correctness.py --ignore=core/block/e2e/test_correctness_sliding_window.py
91
+ - pytest -v -s core
92
93
- label: Entrypoints Test # 40min
94
working_dir: "/vllm-workspace/tests"
@@ -192,8 +188,7 @@ steps:
188
- vllm/
189
- tests/prefix_caching
190
commands:
195
- - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s prefix_caching/test_prefix_caching.py
196
- - pytest -v -s prefix_caching --ignore=prefix_caching/test_prefix_caching.py
191
+ - pytest -v -s prefix_caching
192
193
- label: Samplers Test # 36min
194
source_file_dependencies:
@@ -217,8 +212,7 @@ steps:
212
- tests/spec_decode
213
commands:
214
- pytest -v -s spec_decode/e2e/test_multistep_correctness.py
220
- - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s spec_decode/e2e/test_compatibility.py
221
- - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py --ignore=spec_decode/e2e/test_compatibility.py
215
+ - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
216
217
- label: LoRA Test %N # 15min each
218
mirror_hardwares: [amd]
@@ -405,7 +399,7 @@ steps:
399
- pytest -v -s ./compile/test_basic_correctness.py
400
- pytest -v -s ./compile/test_wrapper.py
401
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
408
- - TARGET_TEST_SUITE=L4 VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest basic_correctness/ -v -s -m distributed_2_gpus
402
+ - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
403
# Avoid importing model tests that cause CUDA reinitialization error
404
- pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
405
- pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
benchmarks/benchmark_latency.py
-4
@@ -38,7 +38,6 @@ def main(args: argparse.Namespace):
38
quantization_param_path=args.quantization_param_path,
39
device=args.device,
40
ray_workers_use_nsight=args.ray_workers_use_nsight,
41
- use_v2_block_manager=args.use_v2_block_manager,
41
enable_chunked_prefill=args.enable_chunked_prefill,
42
download_dir=args.download_dir,
43
block_size=args.block_size,
@@ -221,9 +220,6 @@ if __name__ == '__main__':
220
parser.add_argument("--enable-prefix-caching",
221
action='store_true',
222
help="Enable automatic prefix caching")
224
- parser.add_argument('--use-v2-block-manager',
225
- action='store_true',
226
- default=EngineArgs.use_v2_block_manager)
223
parser.add_argument(
224
"--ray-workers-use-nsight",
225
action='store_true',
benchmarks/benchmark_prefix_caching.py
-6
@@ -33,7 +33,6 @@ from typing import List, Optional, Tuple
33
from transformers import PreTrainedTokenizerBase
34
35
from vllm import LLM, SamplingParams
36
-from vllm.engine.arg_utils import EngineArgs
36
from vllm.utils import FlexibleArgumentParser
37
38
try:
@@ -134,7 +133,6 @@ def main(args):
133
tokenizer_mode='auto',
134
trust_remote_code=True,
135
enforce_eager=True,
137
- use_v2_block_manager=args.use_v2_block_manager,
136
tensor_parallel_size=args.tensor_parallel_size,
137
enable_prefix_caching=args.enable_prefix_caching)
138
@@ -176,10 +174,6 @@ if __name__ == "__main__":
174
parser.add_argument('--enable-prefix-caching',
175
action='store_true',
176
help='enable prefix caching')
179
- parser.add_argument('--use-v2-block-manager',
180
- action='store_true',
181
- default=EngineArgs.use_v2_block_manager,
182
- help='Use BlockSpaceMangerV2')
177
parser.add_argument('--num-prompts',
178
type=int,
179
default=1,
benchmarks/benchmark_throughput.py
+1
-10
@@ -86,7 +86,6 @@ def run_vllm(
86
distributed_executor_backend: Optional[str],
87
gpu_memory_utilization: float = 0.9,
88
num_scheduler_steps: int = 1,
89
- use_v2_block_manager: bool = False,
89
download_dir: Optional[str] = None,
90
load_format: str = EngineArgs.load_format,
91
disable_async_output_proc: bool = False,
@@ -113,7 +112,6 @@ def run_vllm(
112
distributed_executor_backend=distributed_executor_backend,
113
load_format=load_format,
114
num_scheduler_steps=num_scheduler_steps,
116
- use_v2_block_manager=use_v2_block_manager,
115
disable_async_output_proc=disable_async_output_proc,
116
)
117
@@ -176,7 +174,6 @@ async def run_vllm_async(
174
distributed_executor_backend: Optional[str],
175
gpu_memory_utilization: float = 0.9,
176
num_scheduler_steps: int = 1,
179
- use_v2_block_manager: bool = False,
177
download_dir: Optional[str] = None,
178
load_format: str = EngineArgs.load_format,
179
disable_async_output_proc: bool = False,
@@ -204,7 +201,6 @@ async def run_vllm_async(
201
distributed_executor_backend=distributed_executor_backend,
202
load_format=load_format,
203
num_scheduler_steps=num_scheduler_steps,
207
- use_v2_block_manager=use_v2_block_manager,
204
disable_async_output_proc=disable_async_output_proc,
205
worker_use_ray=False,
206
disable_log_requests=True,
@@ -341,8 +337,7 @@ def main(args: argparse.Namespace):
337
args.enable_prefix_caching, args.enable_chunked_prefill,
338
args.max_num_batched_tokens, args.distributed_executor_backend,
339
args.gpu_memory_utilization, args.num_scheduler_steps,
344
- args.use_v2_block_manager, args.download_dir, args.load_format,
345
- args.disable_async_output_proc
340
+ args.download_dir, args.load_format, args.disable_async_output_proc
341
]
342
343
if args.async_engine:
@@ -471,10 +466,6 @@ if __name__ == "__main__":
466
type=int,
467
default=1,
468
help="Maximum number of forward steps per scheduler call.")
474
- parser.add_argument("--use-v2-block-manager",
475
- action='store_true',
476
- default=EngineArgs.use_v2_block_manager,
477
- help="Enable block manager v2.")
469
parser.add_argument(
470
"--enable-prefix-caching",
471
action='store_true',
benchmarks/overheads/benchmark_hashing.py
-4
@@ -16,7 +16,6 @@ def main(args):
16
enforce_eager=True,
17
enable_prefix_caching=True,
18
tensor_parallel_size=args.tensor_parallel_size,
19
- use_v2_block_manager=args.use_v2_block_manager,
19
)
20
21
sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
@@ -56,8 +55,5 @@ if __name__ == "__main__":
55
parser.add_argument('--enable-prefix-caching',
56
action='store_true',
57
help='enable prefix caching')
59
- parser.add_argument('--use-v2-block-manager',
60
- action='store_true',
61
- help='Use BlockSpaceMangerV2')
58
args = parser.parse_args()
59
main(args)
docs/source/models/spec_decode.rst
-3
@@ -30,7 +30,6 @@ The following code configures vLLM in an offline mode to use speculative decodin
30
tensor_parallel_size=1,
31
speculative_model="facebook/opt-125m",
32
num_speculative_tokens=5,
33
- use_v2_block_manager=True,
33
)
34
outputs = llm.generate(prompts, sampling_params)
35
@@ -104,7 +103,6 @@ matching n-grams in the prompt. For more information read `this thread. <https:/
103
speculative_model="[ngram]",
104
num_speculative_tokens=5,
105
ngram_prompt_lookup_max=4,
107
- use_v2_block_manager=True,
106
)
107
outputs = llm.generate(prompts, sampling_params)
108
@@ -135,7 +133,6 @@ For more information see `this blog <https://pytorch.org/blog/hitchhikers-guide-
133
tensor_parallel_size=4,
134
speculative_model="ibm-fms/llama3-70b-accelerator",
135
speculative_draft_tensor_parallel_size=1,
138
- use_v2_block_manager=True,
136
)
137
outputs = llm.generate(prompts, sampling_params)
138
examples/offline_inference_mlpspeculator.py
-2
@@ -50,8 +50,6 @@ if __name__ == "__main__":
50
llm = LLM(
51
model="meta-llama/Llama-2-13b-chat-hf",
52
speculative_model="ibm-fms/llama-13b-accelerator",
53
- # These are currently required for MLPSpeculator decoding
54
- use_v2_block_manager=True,
53
)
54
55
print("With speculation")
tests/basic_correctness/test_chunked_prefill.py
+1
-10
@@ -12,7 +12,7 @@ from contextlib import nullcontext
12
import pytest
13
14
from ..models.utils import check_logprobs_close, check_outputs_equal
15
-from ..utils import check_deprecated_block_manager_usage, multi_gpu_test
15
+from ..utils import multi_gpu_test
16
17
MODELS = [
18
"facebook/opt-125m",
@@ -20,12 +20,6 @@ MODELS = [
20
]
21
22
23
-@pytest.fixture(scope="module", autouse=True)
24
-def check_deprecated_block_manager():
25
- check_deprecated_block_manager_usage(
26
- 'tests/basic_correctness/test_chunked_prefill.py')
27
-
28
-
23
@pytest.mark.parametrize("model", MODELS)
24
@pytest.mark.parametrize("dtype", ["half"])
25
@pytest.mark.parametrize("max_tokens", [32])
@@ -197,7 +191,6 @@ def test_models_with_fp8_kv_cache(
191
@pytest.mark.parametrize("max_tokens", [16])
192
@pytest.mark.parametrize("enforce_eager", [False])
193
@pytest.mark.parametrize("chunk_size", [30, 32])
200
-@pytest.mark.parametrize("use_v2_block_manager", [False, True])
194
# NOTE: Increasing this in this suite will fail CI because we currently cannot
195
# reset distributed env properly. Use a value > 1 just when you test.
196
@pytest.mark.parametrize("tensor_parallel_size", [1])
@@ -206,7 +199,6 @@ def test_with_prefix_caching(
199
max_tokens: int,
200
enforce_eager: bool,
201
chunk_size: int,
209
- use_v2_block_manager: bool,
202
tensor_parallel_size: int,
203
) -> None:
204
"""
@@ -234,7 +226,6 @@ def test_with_prefix_caching(
226
enable_chunked_prefill=True,
227
enable_prefix_caching=enable,
228
tensor_parallel_size=tensor_parallel_size,
237
- use_v2_block_manager=use_v2_block_manager,
229
enforce_eager=enforce_eager,
230
max_num_seqs=max_num_seqs,
231
) as vllm_model:
tests/core/block/e2e/test_correctness.py
+28
-50
@@ -2,18 +2,11 @@ from itertools import cycle
2
3
import pytest
4
5
-from tests.utils import check_deprecated_block_manager_usage
5
from vllm import SamplingParams
6
7
from .conftest import get_token_ids_from_llm_generator
8
9
11
-@pytest.fixture(scope="module", autouse=True)
12
-def check_deprecated_block_manager():
13
- check_deprecated_block_manager_usage(
14
- 'tests/core/block/e2e/test_correctness.py')
15
-
16
-
10
@pytest.mark.parametrize(
11
"common_llm_kwargs",
12
[{
@@ -28,32 +21,32 @@ def check_deprecated_block_manager():
21
"num_gpu_blocks_override": 5 * (64 + 1),
22
}])
23
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
31
-@pytest.mark.parametrize("baseline_llm_kwargs", [{
32
- "use_v2_block_manager": False
33
-}])
24
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
25
@pytest.mark.parametrize("test_llm_kwargs", [{
35
- "use_v2_block_manager": True,
26
"preemption_mode": "swap"
27
}, {
38
- "use_v2_block_manager": True,
28
"preemption_mode": "recompute"
29
}])
30
@pytest.mark.parametrize("batch_size", [10])
31
@pytest.mark.parametrize("seed", [1])
43
-def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
44
- test_llm_generator, batch_size):
45
- """Verify block manager v2 produces same outputs as block manager v1, even
46
- when there is preemption.
32
+def test_block_manager_with_preemption(baseline_llm_generator,
33
+ test_llm_generator, batch_size):
34
+ """Verify block manager produces same outputs even when there is preemption.
35
36
This constructs two LLM, each with limited number of GPU blocks. The limit
37
is decided such that as the sequences in the batch grow, sequences must be
38
preempted and removed from cache.
39
40
If the output token ids are equivalent, then we have confidence that the KV
53
- cache is not corrupted in the v2 block manager.
41
+ cache is not corrupted.
42
43
NOTE: We want a significant number of generated tokens so that any incorrect
44
KV mapping has time to build up error.
45
+
46
+ NOTE(Kuntai): Though we have removed block manager v1, this test is still
47
+ useful as it asserts the behavior of block manager v2 (now it is called
48
+ SelfAttnBlockSpaceManager) is the same when swapping / preemption, so we
49
+ keep this test.
50
"""
51
output_len = 1024
52
temperature = 0.0
@@ -77,11 +70,9 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
70
temperature=temperature,
71
)
72
80
- print('Getting token ids from block manager v1')
73
baseline_token_ids = get_token_ids_from_llm_generator(
74
baseline_llm_generator, prompts, sampling_params)
75
84
- print('Getting token ids from block manager v2')
76
test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
77
prompts, sampling_params)
78
@@ -104,9 +95,6 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
95
96
# skip cuda graph creation for fast test.
97
"enforce_eager": True,
107
-
108
- # Lookahead scheduling only supported in v2 block manager.
109
- "use_v2_block_manager": True,
98
}])
99
@pytest.mark.parametrize(
100
"per_test_common_llm_kwargs",
@@ -218,26 +206,22 @@ def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
206
"max_num_seqs": 10,
207
}])
208
@pytest.mark.parametrize("baseline_llm_kwargs", [
221
- {
222
- "use_v2_block_manager": False,
223
- },
209
+ {},
210
])
211
@pytest.mark.parametrize("test_llm_kwargs", [
212
{
227
- "use_v2_block_manager": True,
213
"num_lookahead_slots": 0,
214
},
215
{
231
- "use_v2_block_manager": True,
216
"num_lookahead_slots": 5,
217
},
218
])
219
@pytest.mark.parametrize("batch_size", [4])
220
@pytest.mark.parametrize("seed", [1])
237
-def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
238
- test_llm_generator, batch_size):
239
- """Verify that chunked prefill works with BlockManagerV2, with and without
240
- lookahead scheduling.
221
+def test_chunked_prefill_block_manager(baseline_llm_generator,
222
+ test_llm_generator, batch_size):
223
+ """Verify that chunked prefill works with SelfAttnBlockSpaceManager,
224
+ with and without lookahead scheduling.
225
"""
226
output_len = 32
227
temperature = 0.0
@@ -258,11 +242,11 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
242
temperature=temperature,
243
)
244
261
- print('Getting token ids with BlockManagerV1')
245
+ print('Getting token ids with BlockManager')
246
baseline_token_ids = get_token_ids_from_llm_generator(
247
baseline_llm_generator, prompts, sampling_params)
248
265
- print('Getting token ids with BlockManagerV2')
249
+ print('Getting token ids with BlockManager, with lookahead slots.')
250
test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
251
prompts, sampling_params)
252
@@ -290,32 +274,32 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
274
"enable_prefix_caching": True,
275
}])
276
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
293
-@pytest.mark.parametrize("baseline_llm_kwargs", [{
294
- "use_v2_block_manager": False
295
-}])
277
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
278
@pytest.mark.parametrize("test_llm_kwargs", [{
297
- "use_v2_block_manager": True,
279
"preemption_mode": "swap"
280
}, {
300
- "use_v2_block_manager": True,
281
"preemption_mode": "recompute"
282
}])
283
@pytest.mark.parametrize("batch_size", [10])
284
@pytest.mark.parametrize("seed", [1])
305
-def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
285
+def test_block_manager_prefix_caching_enabled_with_preemption(
286
baseline_llm_generator, test_llm_generator, batch_size):
307
- """Verify block manager v2 produces same outputs as block manager v1, even
308
- when there is preemption.
287
+ """Verify block manager produces same outputs even when there is preemption.
288
289
This constructs two LLM, each with limited number of GPU blocks. The limit
290
is decided such that as the sequences in the batch grow, sequences must be
291
preempted and removed from cache.
292
293
If the output token ids are equivalent, then we have confidence that the KV
315
- cache is not corrupted in the v2 block manager.
294
+ cache is not corrupted.
295
296
NOTE: We want a significant number of generated tokens so that any incorrect
297
KV mapping has time to build up error.
298
+
299
+ NOTE(Kuntai): Though we have removed block manager v1, this test is still
300
+ useful as it asserts the behavior of block manager v2 (now it is called
301
+ SelfAttnBlockSpaceManager) is the same when swapping / preemption, so we
302
+ keep this test.
303
"""
304
output_len = 1024
305
temperature = 0.0
@@ -339,11 +323,11 @@ def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
323
temperature=temperature,
324
)
325
342
- print('Getting token ids from block manager v1')
326
+ print('Getting token ids from block manager')
327
baseline_token_ids = get_token_ids_from_llm_generator(
328
baseline_llm_generator, prompts, sampling_params)
329
346
- print('Getting token ids from block manager v2')
330
+ print('Getting token ids from block manager, with preemption')
331
test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
332
prompts, sampling_params)
333
@@ -366,9 +350,6 @@ def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
350
# Allow only 5 sequences of ~1024 tokens in worst case.
351
"block_size": 16,
352
"num_gpu_blocks_override": 5 * (64 + 1),
369
-
370
- # Test APC in v2 block
371
- "use_v2_block_manager": True,
353
}])
354
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
355
@pytest.mark.parametrize("baseline_llm_kwargs", [{
@@ -444,9 +425,6 @@ def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
425
"max_model_len": 48,
426
"block_size": 16,
427
"num_gpu_blocks_override": 3,
447
-
448
- # Test APC in v2 block
449
- "use_v2_block_manager": True,
428
}])
429
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
430
@pytest.mark.parametrize("baseline_llm_kwargs", [{
tests/core/block/e2e/test_correctness_sliding_window.py
+3
-16
@@ -3,7 +3,6 @@ from typing import List
3
4
import pytest
5
6
-from tests.utils import check_deprecated_block_manager_usage
6
from vllm import LLM, SamplingParams
7
8
from .conftest import get_text_from_llm_generator
@@ -13,12 +12,6 @@ MODEL = "bigcode/starcoder2-3b"
12
BLOCK_SIZE = 16
13
14
16
-@pytest.fixture(scope="module", autouse=True)
17
-def check_deprecated_block_manager():
18
- check_deprecated_block_manager_usage(
19
- 'tests/core/block/e2e/test_correctness_sliding_window.py')
20
-
21
-
15
@pytest.mark.parametrize(
16
"common_llm_kwargs",
17
[{
@@ -31,10 +24,8 @@ def check_deprecated_block_manager():
24
"num_gpu_blocks_override": 100000 // BLOCK_SIZE,
25
}])
26
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
34
-@pytest.mark.parametrize("baseline_llm_kwargs", [{
35
- "use_v2_block_manager": False
36
-}])
37
-@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
27
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
28
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
29
@pytest.mark.parametrize("batch_size", [5])
30
@pytest.mark.parametrize("seed", [1])
31
def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
@@ -55,7 +46,6 @@ def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
46
47
prompts, answer, indices = prep_prompts(batch_size)
48
58
- print('Getting token ids from block manager v1')
49
baseline_texts = get_text_from_llm_generator(baseline_llm_generator,
50
prompts,
51
sampling_params,
@@ -91,10 +81,7 @@ def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
81
"num_gpu_blocks_override": 100000 // BLOCK_SIZE,
82
}])
83
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
94
-@pytest.mark.parametrize("test_llm_kwargs", [{
95
- "use_v2_block_manager": True,
96
- "enable_chunked_prefill": True
97
-}])
84
+@pytest.mark.parametrize("test_llm_kwargs", [{"enable_chunked_prefill": True}])
85
@pytest.mark.parametrize("batch_size", [5])
86
@pytest.mark.parametrize("seed", [1])
87
def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed):
tests/core/block/test_block_manager.py
renamed
+29
-28
@@ -2,7 +2,7 @@ import pytest
2
3
from vllm.core.block.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
4
STR_NOT_IMPL_ENC_DEC_SWA)
5
-from vllm.core.block_manager_v2 import BlockSpaceManagerV2
5
+from vllm.core.block_manager import SelfAttnBlockSpaceManager
6
from vllm.core.interfaces import AllocStatus
7
from vllm.sequence import Logprob, SequenceStatus
8
from vllm.utils import chunk_list
@@ -17,7 +17,7 @@ from ..utils import (create_dummy_prompt, create_seq_group,
17
@pytest.mark.parametrize("watermark", [0.0, 0.5])
18
def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
19
num_gpu_blocks: int, watermark: float):
20
- block_manager = BlockSpaceManagerV2(
20
+ block_manager = SelfAttnBlockSpaceManager(
21
block_size=block_size,
22
num_gpu_blocks=num_gpu_blocks,
23
num_cpu_blocks=1024,
@@ -63,7 +63,7 @@ def test_can_allocate_seq_group_encoder_decoder(block_size: int,
63
num_seqs_per_group: int,
64
num_gpu_blocks: int,
65
watermark: float):
66
- block_manager = BlockSpaceManagerV2(
66
+ block_manager = SelfAttnBlockSpaceManager(
67
block_size=block_size,
68
num_gpu_blocks=num_gpu_blocks,
69
num_cpu_blocks=1024,
@@ -117,16 +117,16 @@ def test_can_allocate_encoder_decoder_fails_with_swa(block_size: int,
117
'''
118
SWA short for Sliding Window Attention.
119
120
- At time of writing block manager v2 does not support SWA.
120
+ At time of writing block manager does not support SWA.
121
122
- However even when SWA is implemented for block manager v2,
122
+ However even when SWA is implemented for block manager,
123
there will still most likely be a separate workstream required
124
to enable SWA for encoder/decoder models.
125
126
Therefore this test enforces that one of the following cases
127
hold true:
128
- 1. Block manager v2 does not support SWA at all (true at time of writing)
129
- 2. Block manager v2 fails with NotImplementError when SWA is enabled
128
+ 1. Block manager does not support SWA at all (true at time of writing)
129
+ 2. Block manager fails with NotImplementError when SWA is enabled
130
AND a SequenceGroup with an encoder sequence (i.e. in support of an
131
encoder/decoder model) is passed into can_allocate() as an argument
132
@@ -135,7 +135,7 @@ def test_can_allocate_encoder_decoder_fails_with_swa(block_size: int,
135
'''
136
137
with pytest.raises((NotImplementedError, AssertionError)) as exc_info:
138
- block_manager = BlockSpaceManagerV2(
138
+ block_manager = SelfAttnBlockSpaceManager(
139
block_size=block_size,
140
num_gpu_blocks=num_gpu_blocks,
141
num_cpu_blocks=1024,
@@ -158,7 +158,7 @@ def test_can_allocate_encoder_decoder_fails_with_swa(block_size: int,
158
block_manager.can_allocate(seq_group)
159
160
# Assert that either
161
- # 1. Block manager v2 constructor fails with assertion that sliding window
161
+ # 1. Block manager constructor fails with assertion that sliding window
162
# is not yet supported (most likely near-term outcome at time of
163
# writing), or
164
# 2. can_allocate() fails with NotImplementedError due to combination of
@@ -177,7 +177,7 @@ def test_can_allocate_encoder_decoder_fails_with_prefix_cache(
177
block_size: int, num_seqs_per_group: int, num_gpu_blocks: int,
178
watermark: float):
179
180
- block_manager = BlockSpaceManagerV2(
180
+ block_manager = SelfAttnBlockSpaceManager(
181
block_size=block_size,
182
num_gpu_blocks=num_gpu_blocks,
183
num_cpu_blocks=1024,
@@ -217,7 +217,7 @@ def test_append_slots(block_size, prompt_len, num_slots_to_append,
217
218
num_gpu_blocks = 1024
219
watermark = 0.1
220
- block_manager = BlockSpaceManagerV2(
220
+ block_manager = SelfAttnBlockSpaceManager(
221
block_size=block_size,
222
num_gpu_blocks=num_gpu_blocks,
223
num_cpu_blocks=0,
@@ -269,14 +269,15 @@ def test_swap(block_size, num_cpu_blocks, num_gpu_blocks, num_lookahead_slots,
269
"""Verify blocks number on src/desc device is correct after swapping in/out
270
sequence group (not missing or extra blocks).
271
"""
272
- block_manager = BlockSpaceManagerV2(block_size,
273
- num_cpu_blocks,
274
- num_gpu_blocks,
275
- watermark=0,
276
- enable_caching=enable_caching)
272
+ block_manager = SelfAttnBlockSpaceManager(block_size,
273
+ num_cpu_blocks,
274
+ num_gpu_blocks,
275
+ watermark=0,
276
+ enable_caching=enable_caching)
277
prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
278
prompt.status = SequenceStatus.WAITING
279
block_manager.allocate(seq_group)
280
+
281
# Emulate a forward pass by appending a single token.
282
# The block manager then knows how many unprocessed
283
# tokens will be written in the next forward pass.
@@ -321,11 +322,11 @@ def test_can_swap(block_size, num_gpu_blocks, num_lookahead_slots,
322
can be swapped in/out.
323
"""
324
num_cpu_blocks = num_gpu_blocks
324
- block_manager = BlockSpaceManagerV2(block_size,
325
- num_cpu_blocks,
326
- num_gpu_blocks,
327
- watermark=0,
328
- enable_caching=enable_caching)
325
+ block_manager = SelfAttnBlockSpaceManager(block_size,
326
+ num_cpu_blocks,
327
+ num_gpu_blocks,
328
+ watermark=0,
329
+ enable_caching=enable_caching)
330
prompt, seq_group = create_dummy_prompt(
331
"1", prompt_length=(num_gpu_blocks - 1) * block_size - 1)
332
prompt.status = SequenceStatus.WAITING
@@ -382,11 +383,11 @@ def test_swap_in_infeasible(num_lookahead_slots, enable_caching):
383
block_size = 8
384
num_cpu_blocks = 1
385
num_gpu_blocks = 1
385
- block_manager = BlockSpaceManagerV2(block_size,
386
- num_cpu_blocks,
387
- num_gpu_blocks,
388
- watermark=0,
389
- enable_caching=enable_caching)
386
+ block_manager = SelfAttnBlockSpaceManager(block_size,
387
+ num_cpu_blocks,
388
+ num_gpu_blocks,
389
+ watermark=0,
390
+ enable_caching=enable_caching)
391
prompt_length = block_size - 3
392
assert prompt_length > 0
393
prompt, seq_group = create_dummy_prompt("1", prompt_length=prompt_length)
@@ -434,7 +435,7 @@ def test_sliding_window(block_size, prompt_len, num_slots_to_append,
435
436
num_gpu_blocks = 1024
437
watermark = 0.1
437
- block_manager = BlockSpaceManagerV2(
438
+ block_manager = SelfAttnBlockSpaceManager(
439
block_size=block_size,
440
num_gpu_blocks=num_gpu_blocks,
441
num_cpu_blocks=0,
@@ -474,7 +475,7 @@ def test_sliding_window(block_size, prompt_len, num_slots_to_append,
475
seq.data.update_num_computed_tokens(prompt_len)
476
check_used(num_blocks(prompt_len))
477
477
- # this is how we compute it in BlockSpaceManagerV2.__init__
478
+ # this is how we compute it in SelfAttnBlockSpaceManager.__init__
479
sliding_blocks = (sliding_window // block_size) + 2
480
# plus one block for null block
481
sliding_blocks += 1
tests/core/test_block_manager.py
deleted
-637
@@ -1,637 +0,0 @@
1
-import time
2
-from collections import defaultdict
3
-from typing import List
4
-
5
-import pytest
6
-
7
-from vllm import SamplingParams
8
-from vllm.block import PhysicalTokenBlock
9
-from vllm.core.block.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
10
- STR_NOT_IMPL_ENC_DEC_SWA)
11
-from vllm.core.block_manager_v1 import (BlockSpaceManagerV1,
12
- UncachedBlockAllocator)
13
-from vllm.core.interfaces import AllocStatus
14
-from vllm.sequence import Logprob, Sequence, SequenceGroup, SequenceStatus
15
-from vllm.utils import Device
16
-
17
-from .utils import create_dummy_prompt, create_dummy_prompt_encoder_decoder
18
-
19
-
20
-def test_block_allocator_allocate():
21
- block_size = 4
22
- num_cpu_blocks = 4
23
- cpu_allocator = UncachedBlockAllocator(Device.CPU, block_size,
24
- num_cpu_blocks)
25
-
26
- # Allocate all available cpu blocks.
27
- num_free = num_cpu_blocks
28
- assert cpu_allocator.get_num_free_blocks() == num_free
29
- for _ in range(num_cpu_blocks):
30
- block = cpu_allocator.allocate()
31
- num_free -= 1
32
-
33
- assert block not in cpu_allocator.free_blocks
34
- assert cpu_allocator.get_num_free_blocks() == num_free
35
-
36
- with pytest.raises(ValueError):
37
- cpu_allocator.allocate()
38
-
39
-
40
-def test_block_allocator_free():
41
- block_size = 4
42
- num_cpu_blocks = 4
43
- cpu_allocator = UncachedBlockAllocator(Device.CPU, block_size,
44
- num_cpu_blocks)
45
-
46
- # Allocate all available cpu blocks.
47
- blocks: List[PhysicalTokenBlock] = []
48
- for _ in range(num_cpu_blocks):
49
- block = cpu_allocator.allocate()
50
- blocks.append(block)
51
- assert block not in cpu_allocator.free_blocks
52
-
53
- # Free all allocated cpu blocks.
54
- num_free = 0
55
- assert cpu_allocator.get_num_free_blocks() == num_free
56
- for block in blocks:
57
- cpu_allocator.free(block)
58
- num_free += 1
59
- assert block in cpu_allocator.free_blocks
60
- assert cpu_allocator.get_num_free_blocks() == num_free
61
-
62
- with pytest.raises(ValueError):
63
- cpu_allocator.free(block)
64
-
65
-
66
-def test_allocate():
67
- block_size = 4
68
- num_cpu_blocks = 4
69
- num_gpu_blocks = 4
70
- block_manager = BlockSpaceManagerV1(block_size,
71
- num_cpu_blocks,
72
- num_gpu_blocks,
73
- watermark=0)
74
-
75
- # Allocate same sequence group to all available gpu blocks.
76
- for i in range(num_gpu_blocks):
77
- _, seq_group = create_dummy_prompt(str(i), block_size)
78
- assert block_manager.can_allocate(seq_group) == AllocStatus.OK
79
- block_manager.allocate(seq_group)
80
- assert block_manager.can_allocate(seq_group) != AllocStatus.OK
81
-
82
- # Allocate same sequence group to all available gpu blocks.
83
- # Use watermark to reserve one gpu block.
84
- block_manager = BlockSpaceManagerV1(block_size,
85
- num_cpu_blocks,
86
- num_gpu_blocks,
87
- watermark=1 / num_gpu_blocks)
88
- for i in range(num_gpu_blocks - 1):
89
- _, seq_group = create_dummy_prompt(str(i), block_size)
90
- assert block_manager.can_allocate(seq_group) == AllocStatus.OK
91
- block_manager.allocate(seq_group)
92
- assert block_manager.can_allocate(seq_group) != AllocStatus.OK
93
-
94
-
95
-def test_allocate_encoder_decoder():
96
- block_size = 4
97
- num_cpu_blocks = 4
98
- num_gpu_blocks = 4
99
- block_req_per_seq_group = 2
100
- block_manager = BlockSpaceManagerV1(block_size,
101
- num_cpu_blocks,
102
- num_gpu_blocks,
103
- watermark=0)
104
-
105
- # Allocate same sequence group to all available gpu blocks.
106
- for i in range(num_gpu_blocks // block_req_per_seq_group):
107
- _, _, seq_group = create_dummy_prompt_encoder_decoder(
108
- str(i),
109
- decoder_prompt_length=block_size,
110
- encoder_prompt_length=block_size)
111
- assert block_manager.can_allocate(seq_group) == AllocStatus.OK
112
- block_manager.allocate(seq_group)
113
- assert block_manager.can_allocate(seq_group) != AllocStatus.OK
114
-
115
- # Allocate same sequence group to all available gpu blocks.
116
- # Use watermark to reserve one gpu block.
117
- block_manager = BlockSpaceManagerV1(block_size,
118
- num_cpu_blocks,
119
- num_gpu_blocks,
120
- watermark=1 / num_gpu_blocks)
121
- for i in range((num_gpu_blocks - 1) // block_req_per_seq_group):
122
- _, _, seq_group = create_dummy_prompt_encoder_decoder(
123
- str(i),
124
- decoder_prompt_length=block_size,
125
- encoder_prompt_length=block_size)
126
- assert block_manager.can_allocate(seq_group) == AllocStatus.OK
127
- block_manager.allocate(seq_group)
128
- assert block_manager.can_allocate(seq_group) != AllocStatus.OK
129
-
130
-
131
-def test_allocate_encoder_decoder_fails_with_swa():
132
- # SWA short for sliding window attention
133
-
134
- block_size = 4
135
- num_cpu_blocks = 4
136
- num_gpu_blocks = 4
137
- block_manager = BlockSpaceManagerV1(block_size,
138
- num_cpu_blocks,
139
- num_gpu_blocks,
140
- watermark=0,
141
- sliding_window=5) # swa
142
-
143
- # Allocate same sequence group to all available gpu blocks.
144
- _, _, seq_group = create_dummy_prompt_encoder_decoder(
145
- "0",
146
- decoder_prompt_length=block_size,
147
- encoder_prompt_length=block_size)
148
-
149
- # Assert that can_allocate() fails due to SWA
150
- with pytest.raises(NotImplementedError) as exc_info:
151
- block_manager.can_allocate(seq_group)
152
-
153
- assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
154
-
155
- # Assert that allocate() fails due to SWA
156
- with pytest.raises(NotImplementedError) as exc_info:
157
- block_manager.allocate(seq_group)
158
-
159
- assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
160
-
161
-
162
-def test_allocate_encoder_decoder_fails_with_prefix_caching():
163
- block_size = 4
164
- num_cpu_blocks = 4
165
- num_gpu_blocks = 4
166
- block_manager = BlockSpaceManagerV1(block_size,
167
- num_cpu_blocks,
168
- num_gpu_blocks,
169
- watermark=0,
170
- enable_caching=True) # Prefix cache
171
-
172
- # Allocate same sequence group to all available gpu blocks.
173
- _, _, seq_group = create_dummy_prompt_encoder_decoder(
174
- "0",
175
- decoder_prompt_length=block_size,
176
- encoder_prompt_length=block_size)
177
-
178
- # Assert that can_allocate() fails due to prefix caching
179
- with pytest.raises(NotImplementedError) as exc_info:
180
- block_manager.can_allocate(seq_group)
181
-
182
- assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
183
-
184
- # Assert that allocate() fails due to prefix caching
185
- with pytest.raises(NotImplementedError) as exc_info:
186
- block_manager.allocate(seq_group)
187
-
188
- assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
189
-
190
-
191
-def test_append_slot_single_seq():
192
- block_size = 4
193
- num_cpu_blocks = 4
194
- num_gpu_blocks = 4
195
- block_manager = BlockSpaceManagerV1(block_size,
196
- num_cpu_blocks,
197
- num_gpu_blocks,
198
- watermark=0)
199
-
200
- # Allocate single seq to gpu block.
201
- prompt, seq_group = create_dummy_prompt("1", block_size)
202
- block_manager.allocate(seq_group)
203
-
204
- # Nothing to append. Sequence has no new logical blocks.
205
- assert block_manager.can_append_slots(seq_group)
206
- before_blocks = block_manager.get_num_free_gpu_blocks()
207
- assert not block_manager.append_slots(prompt)
208
- after_blocks = block_manager.get_num_free_gpu_blocks()
209
- assert before_blocks == after_blocks
210
-
211
- # Add block_size number of new tokens and append slot.
212
- for i in range(block_size):
213
- token_id = i + 5
214
- prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
215
-
216
- assert block_manager.can_append_slots(seq_group)
217
- before_blocks = block_manager.get_num_free_gpu_blocks()
218
- assert not block_manager.append_slots(prompt)
219
- after_blocks = block_manager.get_num_free_gpu_blocks()
220
- assert before_blocks - after_blocks == 1
221
-
222
-
223
-def test_append_slot_cow():
224
- block_size = 4
225
- num_cpu_blocks = 4
226
- num_gpu_blocks = 4
227
- block_manager = BlockSpaceManagerV1(block_size=block_size,
228
- num_cpu_blocks=num_cpu_blocks,
229
- num_gpu_blocks=num_gpu_blocks,
230
- watermark=0)
231
-
232
- # Allocate prompt to gpu block. There is one slot left in the block.
233
- prompt = Sequence(seq_id=1,
234
- inputs={
235
- "prompt": "one two three",
236
- "prompt_token_ids": [1, 2, 3],
237
- },
238
- block_size=block_size)
239
-
240
- # Fork the sequence, such that a COW will be required when we append a new
241
- # token id.
242
- child = prompt.fork(new_seq_id=2)
243
-
244
- # Allocate space for the sequence group.
245
- seq_group = SequenceGroup(request_id="1",
246
- seqs=[prompt, child],
247
- arrival_time=time.time(),
248
- sampling_params=SamplingParams())
249
- block_manager.allocate(seq_group)
250
-
251
- # Fork and append a new token id. We expect a COW to be scheduled.
252
- token_id = 4
253
- child.append_token_id(token_id, {token_id: Logprob(0.0)})
254
- block_manager.fork(prompt, child)
255
-
256
- assert block_manager.can_append_slots(seq_group)
257
- before_blocks = block_manager.get_num_free_gpu_blocks()
258
-
259
- cows = block_manager.append_slots(child)
260
- assert cows
261
- dict_cows = defaultdict(list)
262
- for src_block, dst_block in cows:
263
- dict_cows[src_block].append(dst_block)
264
- for src_block, dst_blocks in dict_cows.items():
265
- assert src_block not in dst_blocks
266
-
267
- after_blocks = block_manager.get_num_free_gpu_blocks()
268
- assert before_blocks - after_blocks == 1
269
-
270
-
271
-def test_fork():
272
- block_size = 4
273
- num_cpu_blocks = 4
274
- num_gpu_blocks = 4
275
- block_manager = BlockSpaceManagerV1(block_size,
276
- num_cpu_blocks,
277
- num_gpu_blocks,
278
- watermark=0)
279
-
280
- prompt, seq_group = create_dummy_prompt("1",
281
- block_size - 1,
282
- block_size=block_size)
283
- block_manager.allocate(seq_group)
284
-
285
- # Fork prompt and copy block tables.
286
- child = prompt.fork(2)
287
- block_manager.fork(prompt, child)
288
- assert block_manager.get_block_table(
289
- prompt) == block_manager.get_block_table(child)
290
- token_id = 4
291
- # Append token to child. Block is shared so copy on write occurs.
292
- child.append_token_id(token_id, {token_id: Logprob(0.0)})
293
- block_manager.append_slots(child)
294
- assert block_manager.get_block_table(
295
- prompt) != block_manager.get_block_table(child)
296
-
297
-
298
-def test_swap():
299
- block_size = 4
300
- num_cpu_blocks = 4
301
- num_gpu_blocks = 4
302
- block_manager = BlockSpaceManagerV1(block_size,
303
- num_cpu_blocks,
304
- num_gpu_blocks,
305
- watermark=0)
306
-
307
- prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
308
- prompt.status = SequenceStatus.WAITING
309
- block_manager.allocate(seq_group)
310
-
311
- # Emulate a forward pass by appending a single token.
312
- # The block manager then knows how many unprocessed
313
- # tokens will be written in the next forward pass.
314
- token_id = 0
315
- prompt.status = SequenceStatus.RUNNING
316
- prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
317
-
318
- # Swap seq group from GPU -> CPU.
319
- gpu_blocks = block_manager.get_block_table(prompt)
320
- assert block_manager.can_swap_out(seq_group)
321
- before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
322
- before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
323
- mapping = block_manager.swap_out(seq_group)
324
- assert [x[0] for x in mapping] == gpu_blocks
325
- after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
326
- after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
327
- assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
328
- assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
329
- prompt.status = SequenceStatus.SWAPPED
330
-
331
- # Swap seq group from CPU -> GPU.
332
- cpu_blocks = block_manager.get_block_table(prompt)
333
- assert block_manager.can_swap_in(seq_group) == AllocStatus.OK
334
- before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
335
- before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
336
- mapping = block_manager.swap_in(seq_group)
337
- assert [x[0] for x in mapping] == cpu_blocks
338
- after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
339
- after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
340
- assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
341
- assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
342
-
343
-
344
-def test_swap_encoder_decoder():
345
- block_size = 4
346
- num_cpu_blocks = 4
347
- num_gpu_blocks = 4
348
- block_manager = BlockSpaceManagerV1(block_size,
349
- num_cpu_blocks,
350
- num_gpu_blocks,
351
- watermark=0)
352
-
353
- decoder_prompt, encoder_prompt, seq_group = \
354
- create_dummy_prompt_encoder_decoder(
355
- "1",
356
- decoder_prompt_length=block_size,
357
- encoder_prompt_length=block_size)
358
- decoder_prompt.status = SequenceStatus.WAITING
359
- encoder_prompt.status = SequenceStatus.WAITING
360
- block_manager.allocate(seq_group)
361
-
362
- # Emulate a forward pass by appending a single token.
363
- # The block manager then knows how many unprocessed
364
- # tokens will be written in the next forward pass.
365
- token_id = 0
366
- decoder_prompt.status = SequenceStatus.RUNNING
367
- decoder_prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
368
-
369
- # Swap encoder/decoder seq group from GPU -> CPU.
370
- decoder_gpu_blocks = block_manager.get_block_table(decoder_prompt)
371
- cross_gpu_blocks = block_manager.get_cross_block_table(seq_group)
372
- gpu_blocks = decoder_gpu_blocks + cross_gpu_blocks
373
- assert block_manager.can_swap_out(seq_group)
374
- before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
375
- before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
376
- mapping = block_manager.swap_out(seq_group)
377
- assert [x[0] for x in mapping] == gpu_blocks
378
- #assert list(mapping.keys()) == gpu_blocks
379
- after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
380
- after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
381
- assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
382
- assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
383
- decoder_prompt.status = SequenceStatus.SWAPPED
384
-
385
- # Swap encoder/decoder seq group from CPU -> GPU.
386
- decoder_cpu_blocks = block_manager.get_block_table(decoder_prompt)
387
- cross_cpu_blocks = block_manager.get_cross_block_table(seq_group)
388
- cpu_blocks = decoder_cpu_blocks + cross_cpu_blocks
389
- assert block_manager.can_swap_in(seq_group) == AllocStatus.OK
390
- before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
391
- before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
392
- mapping = block_manager.swap_in(seq_group)
393
- assert [x[0] for x in mapping] == cpu_blocks
394
- after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
395
- after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
396
- assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
397
- assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
398
-
399
-
400
-def test_free():
401
- block_size = 4
402
- num_cpu_blocks = 4
403
- num_gpu_blocks = 4
404
- block_manager = BlockSpaceManagerV1(block_size,
405
- num_cpu_blocks,
406
- num_gpu_blocks,
407
- watermark=0)
408
-
409
- prompt, seq_group = create_dummy_prompt("1", block_size)
410
- block_manager.allocate(seq_group)
411
-
412
- # Free allocated seq.
413
- prompt_blocks = len(block_manager.get_block_table(prompt))
414
- before_blocks = block_manager.get_num_free_gpu_blocks()
415
- block_manager.free(prompt)
416
- after_blocks = block_manager.get_num_free_gpu_blocks()
417
- assert after_blocks == before_blocks + prompt_blocks
418
-
419
- # Block table for freed seq is deleted.
420
- with pytest.raises(KeyError):
421
- block_manager.get_block_table(prompt)
422
-
423
-
424
-def test_free_encoder_decoder():
425
- block_size = 4
426
- num_cpu_blocks = 4
427
- num_gpu_blocks = 4
428
- block_manager = BlockSpaceManagerV1(block_size,
429
- num_cpu_blocks,
430
- num_gpu_blocks,
431
- watermark=0)
432
-
433
- decoder_prompt, encoder_prompt, seq_group = \
434
- create_dummy_prompt_encoder_decoder(
435
- "1",
436
- decoder_prompt_length=block_size,
437
- encoder_prompt_length=block_size)
438
- block_manager.allocate(seq_group)
439
-
440
- # Free allocated seq.
441
- decoder_prompt_blocks = len(block_manager.get_block_table(decoder_prompt))
442
- encoder_prompt_blocks = len(block_manager.get_cross_block_table(seq_group))
443
- prompt_blocks = decoder_prompt_blocks + encoder_prompt_blocks
444
- before_blocks = block_manager.get_num_free_gpu_blocks()
445
- block_manager.free(decoder_prompt)
446
- block_manager.free_cross(seq_group)
447
- after_blocks = block_manager.get_num_free_gpu_blocks()
448
- assert after_blocks == before_blocks + prompt_blocks
449
-
450
- # Block table for freed encoder & decoder seq's are deleted.
451
- with pytest.raises(KeyError):
452
- block_manager.get_block_table(decoder_prompt)
453
-
454
- # Block table for freed encoder & decoder seq's are deleted.
455
- with pytest.raises(KeyError):
456
- block_manager.get_block_table(encoder_prompt)
457
-
458
-
459
-def test_reset():
460
- block_size = 4
461
- num_cpu_blocks = 4
462
- num_gpu_blocks = 4
463
- block_manager = BlockSpaceManagerV1(block_size,
464
- num_cpu_blocks,
465
- num_gpu_blocks,
466
- watermark=0)
467
-
468
- # Allocate same seq group on all available gpu blocks.
469
- original_blocks = block_manager.get_num_free_gpu_blocks()
470
- for i in range(num_gpu_blocks):
471
- _, seq_group = create_dummy_prompt(str(i), block_size)
472
- block_manager.allocate(seq_group)
473
- assert block_manager.get_num_free_gpu_blocks() == 0
474
-
475
- # Resetting block manager frees all allocated blocks.
476
- block_manager.reset()
477
- assert block_manager.get_num_free_gpu_blocks() == original_blocks
478
-
479
-
480
-def test_reset_encoder_decoder():
481
- block_size = 4
482
- num_cpu_blocks = 4
483
- num_gpu_blocks = 4
484
- block_req_per_seq_group = 2
485
- block_manager = BlockSpaceManagerV1(block_size,
486
- num_cpu_blocks,
487
- num_gpu_blocks,
488
- watermark=0)
489
-
490
- # Allocate same seq group on all available gpu blocks.
491
- original_blocks = block_manager.get_num_free_gpu_blocks()
492
- for i in range(num_gpu_blocks // block_req_per_seq_group):
493
- _, _, seq_group = create_dummy_prompt_encoder_decoder(
494
- f"{i}",
495
- decoder_prompt_length=block_size,
496
- encoder_prompt_length=block_size)
497
- block_manager.allocate(seq_group)
498
- assert block_manager.get_num_free_gpu_blocks() == 0
499
-
500
- # Resetting block manager frees all allocated blocks.
501
- block_manager.reset()
502
- assert block_manager.get_num_free_gpu_blocks() == original_blocks
503
-
504
-
505
-def test_sliding_window_multi_seq():
506
- """
507
- Tests that memory allocation and deallocation is handled
508
- correctly with multiple sequences that exceed the sliding
509
- window's capacity.
510
- """
511
- block_size = 1
512
- num_cpu_blocks = 8
513
- num_gpu_blocks = 8
514
- sliding_window = 2
515
- block_manager = BlockSpaceManagerV1(block_size,
516
- num_cpu_blocks,
517
- num_gpu_blocks,
518
- sliding_window=sliding_window,
519
- watermark=0)
520
-
521
- assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
522
-
523
- parent = Sequence(seq_id=1,
524
- inputs={
525
- "prompt": "one two three",
526
- "prompt_token_ids": [0, 1, 2],
527
- },
528
- block_size=block_size)
529
- seq_group = SequenceGroup(request_id="1",
530
- seqs=[parent],
531
- arrival_time=time.time(),
532
- sampling_params=SamplingParams(),
533
- lora_request=None)
534
- block_manager.allocate(seq_group)
535
-
536
- # assert the number of blocks allocated is correct
537
- # the parent seq has len 3, but since sliding_window is 2,
538
- # we will use at most 2 blocks
539
- assert block_manager.get_num_free_gpu_blocks(
540
- ) == num_gpu_blocks - sliding_window
541
-
542
- # Fork prompt and copy block tables.
543
- child = parent.fork(2)
544
- block_manager.fork(parent, child)
545
-
546
- # assert the number of blocks allocated is correct
547
- # forking does not increase memory consumption
548
- assert block_manager.get_num_free_gpu_blocks(
549
- ) == num_gpu_blocks - sliding_window
550
-
551
- # assert both parent and child share all blocks
552
- assert block_manager.get_block_table(
553
- parent) == block_manager.get_block_table(child)
554
-
555
- token_id = 4
556
- # Append token to child. Block is shared so copy on write occurs.
557
- child.append_token_id(token_id, {token_id: Logprob(0.0)})
558
- block_manager.append_slots(child)
559
-
560
- # assert the number of blocks allocated is correct
561
- # we will use now one block more. Each seq will use 2 blocks,
562
- # but only one can be shared
563
- assert block_manager.get_num_free_gpu_blocks(
564
- ) == num_gpu_blocks - sliding_window - 1
565
-
566
- token_id = 5
567
- parent.append_token_id(token_id, {token_id: Logprob(0.0)})
568
- block_manager.append_slots(parent)
569
-
570
- # assert the number of blocks allocated is correct
571
- # no change, because both sequences are still just sharing one block
572
- assert block_manager.get_num_free_gpu_blocks(
573
- ) == num_gpu_blocks - sliding_window - 1
574
-
575
- block_table_parent = block_manager.get_block_table(parent)
576
- block_table_child = block_manager.get_block_table(child)
577
-
578
- assert block_table_parent != block_table_child
579
-
580
- # assert both blocks are sharing the second-last block
581
- assert block_table_parent[-2] == block_table_child[-2]
582
-
583
- # now let's clean up...
584
- block_manager.free(parent)
585
-
586
- # assert the number of blocks allocated is correct
587
- # We have freed one seq, reducing the ref count of two blocks by one.
588
- # One of the two was only used by the parent seq, so this is now free.
589
- # The child seq still consumes sliding_window blocks
590
- assert block_manager.get_num_free_gpu_blocks(
591
- ) == num_gpu_blocks - sliding_window
592
-
593
- # free all blocks
594
- block_manager.free(child)
595
-
596
- # assert all blocks are free now
597
- assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
598
-
599
-
600
-def test_mark_blocks_as_computed_with_prefix_cache_and_chunked_prefill():
601
- """When prefix cache and chunked prefill are enabled, the block manager
602
- should only mark a chunk of blocks as computed instead of all blocks.
603
- """
604
-
605
- block_size = 4
606
- num_cpu_blocks = 0
607
- num_gpu_blocks = 16
608
- block_manager = BlockSpaceManagerV1(block_size,
609
- num_gpu_blocks,
610
- num_cpu_blocks,
611
- watermark=0,
612
- enable_caching=True)
613
-
614
- # Set prompt size to have num_gpu_blocks - 1 full blocks.
615
- prompt_length = block_size * num_gpu_blocks - 1
616
-
617
- # Allocate (reserve) all blocks.
618
- _, seq_group = create_dummy_prompt("0",
619
- prompt_length,
620
- block_size=block_size)
621
- block_manager.allocate(seq_group)
622
- assert seq_group.seqs[0].n_blocks == num_gpu_blocks
623
-
624
- # 1st chunk: Compute 2 and half blocks. Should mark 2 blocks as computed.
625
- token_chunk_size = int(block_size * 2.5)
626
- block_manager.mark_blocks_as_computed(seq_group, token_chunk_size)
627
- computed_blocks = block_manager.get_all_computed_blocks(seq_group.seqs[0])
628
- assert len(computed_blocks) == 2
629
-
630
- # Actual computed tokens.
631
- seq_group.seqs[0].data.update_num_computed_tokens(token_chunk_size)
632
-
633
- # 2nd chunk: Complete 3rd block and additional 4 blocks.
634
- token_chunk_size = int(block_size * 4.5)
635
- block_manager.mark_blocks_as_computed(seq_group, token_chunk_size)
636
- computed_blocks = block_manager.get_all_computed_blocks(seq_group.seqs[0])
637
- assert len(computed_blocks) == 7
tests/core/test_chunked_prefill_scheduler.py
+24
-44
@@ -8,7 +8,6 @@ from vllm.core.interfaces import AllocStatus
8
from vllm.core.scheduler import Scheduler
9
from vllm.sequence import Logprob, SequenceGroup
10
11
-from ..utils import check_deprecated_block_manager_usage
11
from .utils import create_dummy_prompt
12
13
@@ -28,25 +27,16 @@ def schedule_and_update_computed_tokens(scheduler):
27
return metas, out
28
29
31
-@pytest.fixture(scope="module", autouse=True)
32
-def check_deprecated_block_manager():
33
- check_deprecated_block_manager_usage(
34
- 'tests/core/test_chunked_prefill_scheduler.py')
35
-
36
-
37
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
38
-def test_simple(use_v2_block_manager: bool):
30
+def test_simple():
31
"""Verify basic scheduling works."""
32
block_size = 4
33
num_seq_group = 4
34
max_model_len = 16
35
max_num_batched_tokens = 64
44
- scheduler_config = SchedulerConfig(
45
- max_num_batched_tokens,
46
- num_seq_group,
47
- max_model_len,
48
- enable_chunked_prefill=True,
49
- use_v2_block_manager=use_v2_block_manager)
36
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
37
+ num_seq_group,
38
+ max_model_len,
39
+ enable_chunked_prefill=True)
40
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
41
cache_config.num_cpu_blocks = 8
42
cache_config.num_gpu_blocks = 8
@@ -81,8 +71,7 @@ def test_simple(use_v2_block_manager: bool):
71
assert len(seq_group_meta) == num_seq_group
72
73
84
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
85
-def test_chunk(use_v2_block_manager: bool):
74
+def test_chunk():
75
"""Verify prefills are chunked properly."""
76
block_size = 4
77
max_seqs = 60
@@ -93,7 +82,7 @@ def test_chunk(use_v2_block_manager: bool):
82
max_seqs,
83
max_model_len,
84
enable_chunked_prefill=True,
96
- use_v2_block_manager=use_v2_block_manager)
85
+ )
86
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
87
cache_config.num_cpu_blocks = 32
88
cache_config.num_gpu_blocks = 32
@@ -131,8 +120,7 @@ def test_chunk(use_v2_block_manager: bool):
120
assert out.num_batched_tokens == 57
121
122
134
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
135
-def test_complex(use_v2_block_manager: bool):
123
+def test_complex():
124
block_size = 4
125
max_seqs = 60
126
max_model_len = 80
@@ -142,7 +130,7 @@ def test_complex(use_v2_block_manager: bool):
130
max_seqs,
131
max_model_len,
132
enable_chunked_prefill=True,
145
- use_v2_block_manager=use_v2_block_manager)
133
+ )
134
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
135
cache_config.num_cpu_blocks = 64
136
cache_config.num_gpu_blocks = 64
@@ -201,8 +189,7 @@ def test_complex(use_v2_block_manager: bool):
189
assert running[2].is_prefill()
190
191
204
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
205
-def test_maximal_decoding(use_v2_block_manager: bool):
192
+def test_maximal_decoding():
193
"""Verify decoding requests are prioritized."""
194
block_size = 4
195
max_seqs = 2
@@ -213,7 +200,7 @@ def test_maximal_decoding(use_v2_block_manager: bool):
200
max_seqs,
201
max_model_len,
202
enable_chunked_prefill=True,
216
- use_v2_block_manager=use_v2_block_manager)
203
+ )
204
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
205
cache_config.num_cpu_blocks = 8
206
cache_config.num_gpu_blocks = 8
@@ -295,8 +282,7 @@ def test_maximal_decoding(use_v2_block_manager: bool):
282
assert out.num_batched_tokens == 2
283
284
298
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
299
-def test_prompt_limit(use_v2_block_manager: bool):
285
+def test_prompt_limit():
286
"""Verify max_num_batched_tokens < max_model_len is possible."""
287
block_size = 4
288
max_seqs = 32
@@ -307,7 +293,7 @@ def test_prompt_limit(use_v2_block_manager: bool):
293
max_seqs,
294
max_model_len,
295
enable_chunked_prefill=True,
310
- use_v2_block_manager=use_v2_block_manager)
296
+ )
297
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
298
cache_config.num_cpu_blocks = 16
299
cache_config.num_gpu_blocks = 16
@@ -330,8 +316,7 @@ def test_prompt_limit(use_v2_block_manager: bool):
316
assert out.num_batched_tokens == 32
317
318
333
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
334
-def test_prompt_limit_exceed(use_v2_block_manager: bool):
319
+def test_prompt_limit_exceed():
320
block_size = 4
321
max_seqs = 64
322
max_model_len = 32
@@ -356,8 +341,7 @@ def test_prompt_limit_exceed(use_v2_block_manager: bool):
341
assert out.ignored_seq_groups[0] == seq_group
342
343
359
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
360
-def test_swap(use_v2_block_manager: bool):
344
+def test_swap():
345
"""Verify swapping works with chunked prefill requests"""
346
block_size = 4
347
max_seqs = 30
@@ -368,7 +352,7 @@ def test_swap(use_v2_block_manager: bool):
352
max_seqs,
353
max_model_len,
354
enable_chunked_prefill=True,
371
- use_v2_block_manager=use_v2_block_manager)
355
+ )
356
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
357
cache_config.num_cpu_blocks = 16
358
cache_config.num_gpu_blocks = 16
@@ -414,8 +398,7 @@ def test_swap(use_v2_block_manager: bool):
398
assert out.blocks_to_swap_out == []
399
400
417
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
418
-def test_running_prefill_prioritized_over_swap(use_v2_block_manager: bool):
401
+def test_running_prefill_prioritized_over_swap():
402
block_size = 4
403
max_seqs = 30
404
max_model_len = 200
@@ -425,7 +408,7 @@ def test_running_prefill_prioritized_over_swap(use_v2_block_manager: bool):
408
max_seqs,
409
max_model_len,
410
enable_chunked_prefill=True,
428
- use_v2_block_manager=use_v2_block_manager)
411
+ )
412
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
413
cache_config.num_cpu_blocks = 32
414
cache_config.num_gpu_blocks = 32
@@ -508,8 +491,7 @@ def test_running_prefill_prioritized_over_swap(use_v2_block_manager: bool):
491
assert out.blocks_to_swap_out == []
492
493
511
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
512
-def test_chunked_prefill_preempt(use_v2_block_manager: bool):
494
+def test_chunked_prefill_preempt():
495
"""Verify preempt works with chunked prefill requests"""
496
block_size = 4
497
max_seqs = 30
@@ -520,7 +502,7 @@ def test_chunked_prefill_preempt(use_v2_block_manager: bool):
502
max_seqs,
503
max_model_len,
504
enable_chunked_prefill=True,
523
- use_v2_block_manager=use_v2_block_manager)
505
+ )
506
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
507
cache_config.num_cpu_blocks = 16
508
cache_config.num_gpu_blocks = 16
@@ -575,8 +557,7 @@ def test_chunked_prefill_preempt(use_v2_block_manager: bool):
557
assert out.num_batched_tokens == max_num_batched_tokens
558
559
578
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
579
-def test_chunked_prefill_max_seqs(use_v2_block_manager: bool):
560
+def test_chunked_prefill_max_seqs():
561
block_size = 4
562
max_seqs = 2
563
max_model_len = 80
@@ -586,7 +567,7 @@ def test_chunked_prefill_max_seqs(use_v2_block_manager: bool):
567
max_seqs,
568
max_model_len,
569
enable_chunked_prefill=True,
589
- use_v2_block_manager=use_v2_block_manager)
570
+ )
571
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
572
cache_config.num_cpu_blocks = 128
573
cache_config.num_gpu_blocks = 128
@@ -629,8 +610,7 @@ def test_chunked_prefill_max_seqs(use_v2_block_manager: bool):
610
assert not running[1].is_prefill()
611
612
632
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
633
-def test_perfix_caching(use_v2_block_manager: bool):
613
+def test_perfix_caching():
614
"""Verify allocating full blocks when prefix caching is enabled."""
615
block_size = 4
616
max_seqs = 10
@@ -641,7 +621,7 @@ def test_perfix_caching(use_v2_block_manager: bool):
621
max_seqs,
622
max_model_len,
623
enable_chunked_prefill=True,
644
- use_v2_block_manager=use_v2_block_manager)
624
+ )
625
cache_config = CacheConfig(block_size,
626
1.0,
627
1,
tests/core/test_num_computed_tokens_update.py
-1
@@ -31,7 +31,6 @@ def test_num_computed_tokens_update(num_scheduler_steps: int,
31
# Make a vllm engine
32
runner = VllmRunner(model_name=MODEL,
33
gpu_memory_utilization=0.7,
34
- use_v2_block_manager=True,
34
num_scheduler_steps=num_scheduler_steps,
35
enable_chunked_prefill=enable_chunked_prefill,
36
enforce_eager=enforce_eager)
tests/core/test_scheduler.py
+55
-95
@@ -3,7 +3,7 @@ from collections import deque
3
from typing import List, Set, Tuple
4
from unittest.mock import MagicMock
5
6
-import pytest
6
+import pytest # noqa
7
from torch import Use # noqa
8
9
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
@@ -12,23 +12,18 @@ from vllm.core.scheduler import Scheduler, SchedulingBudget
12
from vllm.lora.request import LoRARequest
13
from vllm.sequence import SequenceGroup, SequenceStatus
14
15
-from ..utils import check_deprecated_block_manager_usage
15
from .utils import (append_new_token, append_new_token_seq_group,
16
create_dummy_prompt, get_sequence_groups,
17
schedule_and_update_computed_tokens)
18
19
21
-@pytest.fixture(scope="module", autouse=True)
22
-def check_deprecated_block_manager():
23
- check_deprecated_block_manager_usage(
24
- "tests/core/test_chunked_prefill_scheduler.py")
25
-
26
-
27
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
28
-def test_scheduler_add_seq_group(use_v2_block_manager: bool):
20
+def test_scheduler_add_seq_group():
21
block_size = 4
22
scheduler_config = SchedulerConfig(
31
- 100, 64, 1, use_v2_block_manager=use_v2_block_manager)
23
+ 100,
24
+ 64,
25
+ 1,
26
+ )
27
cache_config = CacheConfig(block_size, 1.0, 1, cache_dtype="auto")
28
cache_config.num_cpu_blocks = 4
29
cache_config.num_gpu_blocks = 4
@@ -44,11 +39,13 @@ def test_scheduler_add_seq_group(use_v2_block_manager: bool):
39
assert scheduler.get_num_unfinished_seq_groups() == i + 1
40
41
47
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
48
-def test_scheduler_abort_seq_group(use_v2_block_manager: bool):
42
+def test_scheduler_abort_seq_group():
43
block_size = 4
44
scheduler_config = SchedulerConfig(
51
- 100, 64, 1, use_v2_block_manager=use_v2_block_manager)
45
+ 100,
46
+ 64,
47
+ 1,
48
+ )
49
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
50
cache_config.num_cpu_blocks = 4
51
cache_config.num_gpu_blocks = 4
@@ -68,8 +65,7 @@ def test_scheduler_abort_seq_group(use_v2_block_manager: bool):
65
assert scheduler.get_num_unfinished_seq_groups() == 0
66
67
71
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
72
-def test_scheduler_schedule_simple(use_v2_block_manager: bool):
68
+def test_scheduler_schedule_simple():
69
block_size = 4
70
num_seq_group = 4
71
max_model_len = 16
@@ -77,7 +73,7 @@ def test_scheduler_schedule_simple(use_v2_block_manager: bool):
73
64,
74
num_seq_group,
75
max_model_len,
80
- use_v2_block_manager=use_v2_block_manager)
76
+ )
77
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
78
cache_config.num_cpu_blocks = 8
79
cache_config.num_gpu_blocks = 8
@@ -112,8 +108,7 @@ def test_scheduler_schedule_simple(use_v2_block_manager: bool):
108
append_new_token(out, 1)
109
110
115
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
116
-def test_scheduler_prefill_prioritized(use_v2_block_manager: bool):
111
+def test_scheduler_prefill_prioritized():
112
"""Verify running batched tokens are not applied to prefill requests."""
113
block_size = 4
114
max_model_len = 30
@@ -122,7 +117,7 @@ def test_scheduler_prefill_prioritized(use_v2_block_manager: bool):
117
max_batched_num_tokens,
118
2,
119
max_model_len,
125
- use_v2_block_manager=use_v2_block_manager)
120
+ )
121
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
122
cache_config.num_cpu_blocks = 16
123
cache_config.num_gpu_blocks = 16
@@ -146,12 +141,14 @@ def test_scheduler_prefill_prioritized(use_v2_block_manager: bool):
141
assert get_sequence_groups(out) == [seq_group_b]
142
143
149
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
150
-def test_scheduler_schedule_preempt_abort(use_v2_block_manager: bool):
144
+def test_scheduler_schedule_preempt_abort():
145
block_size = 4
146
max_model_len = 16
147
scheduler_config = SchedulerConfig(
154
- 64, 2, max_model_len, use_v2_block_manager=use_v2_block_manager)
148
+ 64,
149
+ 2,
150
+ max_model_len,
151
+ )
152
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
153
cache_config.num_cpu_blocks = 2
154
cache_config.num_gpu_blocks = 2
@@ -201,8 +198,7 @@ def test_scheduler_schedule_preempt_abort(use_v2_block_manager: bool):
198
assert scheduler.get_num_unfinished_seq_groups() == 1
199
200
204
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
205
-def test_scheduler_max_seqs(use_v2_block_manager: bool):
201
+def test_scheduler_max_seqs():
202
block_size = 4
203
num_seq_group = 4
204
max_seq_group = 2
@@ -211,7 +207,7 @@ def test_scheduler_max_seqs(use_v2_block_manager: bool):
207
64,
208
max_seq_group,
209
max_model_len,
214
- use_v2_block_manager=use_v2_block_manager)
210
+ )
211
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
212
cache_config.num_cpu_blocks = 8
213
cache_config.num_gpu_blocks = 8
@@ -249,15 +245,14 @@ def test_scheduler_max_seqs(use_v2_block_manager: bool):
245
assert set(get_sequence_groups(out)) == set([all_seq_groups[1]])
246
247
252
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
253
-def test_scheduler_delay_factor(use_v2_block_manager: bool):
248
+def test_scheduler_delay_factor():
249
block_size = 4
250
scheduler_config = SchedulerConfig(
251
100,
252
64,
253
16,
254
delay_factor=0.5,
260
- use_v2_block_manager=use_v2_block_manager)
255
+ )
256
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
257
cache_config.num_cpu_blocks = 8
258
cache_config.num_gpu_blocks = 8
@@ -294,12 +289,10 @@ def test_scheduler_delay_factor(use_v2_block_manager: bool):
289
append_new_token(out, 1)
290
291
297
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
298
-def test_swapped_out_prioritized(use_v2_block_manager: bool):
292
+def test_swapped_out_prioritized():
293
block_size = 4
294
scheduler = initialize_scheduler(max_num_seqs=6,
295
block_size=block_size,
302
- use_v2_block_manager=use_v2_block_manager,
296
num_cpu_blocks=64,
297
num_gpu_blocks=64)
298
# best_of=2 * 3 == 6 sequences.
@@ -351,7 +344,6 @@ def initialize_scheduler(
344
max_token_budget=1000,
345
max_model_len=1000,
346
lora_config=None,
354
- use_v2_block_manager=False,
347
block_size=4,
348
num_cpu_blocks=8,
349
num_gpu_blocks=8,
@@ -361,7 +353,7 @@ def initialize_scheduler(
353
max_token_budget,
354
max_num_seqs,
355
max_model_len,
364
- use_v2_block_manager=use_v2_block_manager)
356
+ )
357
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
358
cache_config.num_cpu_blocks = num_cpu_blocks
359
cache_config.num_gpu_blocks = num_gpu_blocks
@@ -386,15 +378,12 @@ def add_token_budget(budget: SchedulingBudget,
378
budget.add_num_seqs(mock_seq_group.request_id, num_curr_seqs)
379
380
389
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
390
-def test_prefill_schedule_max_prompt_len(use_v2_block_manager: bool):
381
+def test_prefill_schedule_max_prompt_len():
382
"""
383
Test prompt longer than max_prompt_len is aborted.
384
"""
385
block_size = 4
395
- scheduler = initialize_scheduler(max_model_len=30,
396
- use_v2_block_manager=use_v2_block_manager,
397
- block_size=block_size)
386
+ scheduler = initialize_scheduler(max_model_len=30, block_size=block_size)
387
_, seq_group = create_dummy_prompt("0",
388
prompt_length=60,
389
block_size=block_size)
@@ -409,14 +398,12 @@ def test_prefill_schedule_max_prompt_len(use_v2_block_manager: bool):
398
assert len(remaining_waiting) == 0
399
400
412
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
413
-def test_prefill_schedule_token_budget(use_v2_block_manager: bool):
401
+def test_prefill_schedule_token_budget():
402
"""
403
Test token budget respected.
404
"""
405
block_size = 4
418
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
419
- block_size=block_size,
406
+ scheduler = initialize_scheduler(block_size=block_size,
407
num_cpu_blocks=64,
408
num_gpu_blocks=64)
409
budget = create_token_budget(token_budget=0)
@@ -446,8 +433,7 @@ def test_prefill_schedule_token_budget(use_v2_block_manager: bool):
433
assert len(remaining_waiting) == 1
434
435
# Test when current_batched_tokens respected.
449
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
450
- block_size=block_size,
436
+ scheduler = initialize_scheduler(block_size=block_size,
437
num_cpu_blocks=16,
438
num_gpu_blocks=16)
439
budget = create_token_budget(token_budget=60)
@@ -474,14 +460,12 @@ def test_prefill_schedule_token_budget(use_v2_block_manager: bool):
460
assert len(remaining_waiting) == 0
461
462
477
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
478
-def test_prefill_schedule_max_seqs(use_v2_block_manager: bool):
463
+def test_prefill_schedule_max_seqs():
464
"""
465
Test max seq respected.
466
"""
467
block_size = 4
483
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
484
- block_size=block_size,
468
+ scheduler = initialize_scheduler(block_size=block_size,
469
num_cpu_blocks=64,
470
num_gpu_blocks=64)
471
budget = create_token_budget(max_num_seqs=2)
@@ -515,15 +499,13 @@ def test_prefill_schedule_max_seqs(use_v2_block_manager: bool):
499
assert len(remaining_waiting) == 1
500
501
518
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
519
-def test_prefill_schedule_max_lora(use_v2_block_manager: bool):
502
+def test_prefill_schedule_max_lora():
503
"""
504
Test max lora is respected and prioritized.
505
"""
506
block_size = 4
507
lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
508
scheduler = initialize_scheduler(lora_config=lora_config,
526
- use_v2_block_manager=use_v2_block_manager,
509
block_size=block_size,
510
num_cpu_blocks=64,
511
num_gpu_blocks=64)
@@ -570,14 +552,12 @@ def test_prefill_schedule_max_lora(use_v2_block_manager: bool):
552
assert budget.num_batched_tokens == 60
553
554
573
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
574
-def test_prefill_schedule_no_block_manager_capacity(use_v2_block_manager):
555
+def test_prefill_schedule_no_block_manager_capacity():
556
"""
557
Test sequence cannot be scheduled due to block manager has no capacity.
558
"""
559
block_size = 4
579
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
580
- block_size=block_size,
560
+ scheduler = initialize_scheduler(block_size=block_size,
561
num_gpu_blocks=128,
562
num_cpu_blocks=128)
563
budget = create_token_budget()
@@ -614,14 +594,12 @@ def test_prefill_schedule_no_block_manager_capacity(use_v2_block_manager):
594
assert len(remaining_waiting) == 0
595
596
617
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
618
-def test_decode_schedule_preempted(use_v2_block_manager: bool):
597
+def test_decode_schedule_preempted():
598
"""
599
Test decodes cannot be scheduled and preempted.
600
"""
601
block_size = 4
623
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
624
- block_size=block_size,
602
+ scheduler = initialize_scheduler(block_size=block_size,
603
num_cpu_blocks=64,
604
num_gpu_blocks=64)
605
curr_loras = None
@@ -660,14 +638,12 @@ def test_decode_schedule_preempted(use_v2_block_manager: bool):
638
assert output.blocks_to_copy == []
639
640
663
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
664
-def test_decode_swap_beam_search(use_v2_block_manager: bool):
641
+def test_decode_swap_beam_search():
642
"""
643
Test best_of > 1 swap out blocks
644
"""
645
block_size = 4
669
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
670
- block_size=block_size,
646
+ scheduler = initialize_scheduler(block_size=block_size,
647
num_gpu_blocks=64,
648
num_cpu_blocks=64)
649
curr_loras = None
@@ -716,14 +692,12 @@ def test_decode_swap_beam_search(use_v2_block_manager: bool):
692
assert output.blocks_to_copy == []
693
694
719
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
720
-def test_schedule_decode_blocks_to_copy_update(use_v2_block_manager: bool):
695
+def test_schedule_decode_blocks_to_copy_update():
696
"""
697
Verify blocks_to_copy is updated.
698
"""
699
block_size = 4
725
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
726
- block_size=4,
700
+ scheduler = initialize_scheduler(block_size=4,
701
num_cpu_blocks=16,
702
num_gpu_blocks=16)
703
_, seq_group = create_dummy_prompt("1",
@@ -754,11 +728,9 @@ def test_schedule_decode_blocks_to_copy_update(use_v2_block_manager: bool):
728
assert output.blocks_to_copy == [(2, 3)]
729
730
757
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
758
-def test_schedule_swapped_simple(use_v2_block_manager: bool):
731
+def test_schedule_swapped_simple():
732
block_size = 4
760
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
761
- block_size=block_size)
733
+ scheduler = initialize_scheduler(block_size=block_size)
734
curr_loras = None
735
blocks_to_swap_out: List[Tuple[int, int]] = []
736
_, seq_group = create_dummy_prompt("1",
@@ -785,11 +757,9 @@ def test_schedule_swapped_simple(use_v2_block_manager: bool):
757
assert blocks_to_swap_out == blocks_to_swap_in_reverse
758
759
788
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
789
-def test_schedule_swapped_max_token_budget(use_v2_block_manager: bool):
760
+def test_schedule_swapped_max_token_budget():
761
block_size = 4
791
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
792
- block_size=block_size,
762
+ scheduler = initialize_scheduler(block_size=block_size,
763
num_cpu_blocks=32,
764
num_gpu_blocks=32)
765
curr_loras = None
@@ -822,11 +792,9 @@ def test_schedule_swapped_max_token_budget(use_v2_block_manager: bool):
792
assert len(output.prefill_seq_groups) == 0
793
794
825
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
826
-def test_schedule_swapped_max_seqs(use_v2_block_manager: bool):
795
+def test_schedule_swapped_max_seqs():
796
block_size = 4
828
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
829
- block_size=block_size,
797
+ scheduler = initialize_scheduler(block_size=block_size,
798
num_cpu_blocks=64,
799
num_gpu_blocks=64)
800
curr_loras = None
@@ -859,12 +827,10 @@ def test_schedule_swapped_max_seqs(use_v2_block_manager: bool):
827
assert len(output.prefill_seq_groups) == 0
828
829
862
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
863
-def test_schedule_swapped_max_loras(use_v2_block_manager: bool):
830
+def test_schedule_swapped_max_loras():
831
block_size = 4
832
lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
833
scheduler = initialize_scheduler(lora_config=lora_config,
867
- use_v2_block_manager=use_v2_block_manager,
834
block_size=block_size,
835
num_cpu_blocks=32,
836
num_gpu_blocks=32)
@@ -894,11 +860,9 @@ def test_schedule_swapped_max_loras(use_v2_block_manager: bool):
860
assert len(curr_loras) == 1
861
862
897
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
898
-def test_schedule_swapped_cannot_swap_in(use_v2_block_manager: bool):
863
+def test_schedule_swapped_cannot_swap_in():
864
block_size = 4
900
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
901
- block_size=block_size,
865
+ scheduler = initialize_scheduler(block_size=block_size,
866
num_cpu_blocks=32,
867
num_gpu_blocks=32)
868
curr_loras = None
@@ -927,11 +891,9 @@ def test_schedule_swapped_cannot_swap_in(use_v2_block_manager: bool):
891
assert len(output.prefill_seq_groups) == 0
892
893
930
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
931
-def test_infeasible_swap(use_v2_block_manager: bool):
894
+def test_infeasible_swap():
895
block_size = 4
933
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
934
- block_size=block_size,
896
+ scheduler = initialize_scheduler(block_size=block_size,
897
num_cpu_blocks=32,
898
num_gpu_blocks=32)
899
curr_loras = None
@@ -961,11 +923,9 @@ def test_infeasible_swap(use_v2_block_manager: bool):
923
assert len(output.prefill_seq_groups) == 0
924
925
964
-@pytest.mark.parametrize('use_v2_block_manager', [True, False])
965
-def test_schedule_swapped_blocks_to_copy(use_v2_block_manager: bool):
926
+def test_schedule_swapped_blocks_to_copy():
927
block_size = 4
967
- scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
968
- block_size=block_size,
928
+ scheduler = initialize_scheduler(block_size=block_size,
929
num_cpu_blocks=32,
930
num_gpu_blocks=32)
931
curr_loras = None
tests/metrics/test_metrics.py
+8
-8
@@ -185,13 +185,14 @@ def test_metric_spec_decode(
185
) -> None:
186
k = 5
187
188
- with vllm_runner(model,
189
- dtype=dtype,
190
- disable_log_stats=False,
191
- gpu_memory_utilization=0.4,
192
- speculative_model=model,
193
- num_speculative_tokens=k,
194
- use_v2_block_manager=True) as vllm_model:
188
+ with vllm_runner(
189
+ model,
190
+ dtype=dtype,
191
+ disable_log_stats=False,
192
+ gpu_memory_utilization=0.4,
193
+ speculative_model=model,
194
+ num_speculative_tokens=k,
195
+ ) as vllm_model:
196
197
# Force log interval to be 0 to catch all metrics.
198
stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
@@ -242,7 +243,6 @@ def test_metric_spec_decode_interval(
243
gpu_memory_utilization=0.4,
244
speculative_model=model,
245
num_speculative_tokens=k,
245
- use_v2_block_manager=True,
246
enforce_eager=True)
247
248
engine = LLMEngine.from_engine_args(engine_args)
tests/multi_step/test_correctness_async_llm.py
-1
@@ -17,7 +17,6 @@ NUM_PROMPTS = [10]
17
18
DEFAULT_SERVER_ARGS: List[str] = [
19
"--disable-log-requests",
20
- "--use-v2-block-manager",
20
"--worker-use-ray",
21
"--gpu-memory-utilization",
22
"0.85",
tests/multi_step/test_correctness_llm.py
-4
@@ -76,7 +76,6 @@ def test_multi_step_llm(
76
enforce_eager=enforce_eager,
77
gpu_memory_utilization=0.7,
78
tensor_parallel_size=tp_size,
79
- use_v2_block_manager=True,
79
enable_chunked_prefill=enable_chunked_prefill,
80
num_scheduler_steps=num_scheduler_steps,
81
) as vllm_model:
@@ -169,7 +168,6 @@ def test_multi_step_llm_w_prompt_logprobs(
168
enforce_eager=enforce_eager,
169
gpu_memory_utilization=0.7,
170
tensor_parallel_size=tp_size,
172
- use_v2_block_manager=True,
171
num_scheduler_steps=num_scheduler_steps,
172
) as vllm_model:
173
vllm_outputs = vllm_model.generate_greedy_logprobs(
@@ -305,7 +303,6 @@ def test_multi_step_llm_chunked_prefill_prefix_cache(
303
enforce_eager=enforce_eager,
304
gpu_memory_utilization=0.7,
305
tensor_parallel_size=tp_size,
308
- use_v2_block_manager=True,
306
num_scheduler_steps=num_scheduler_steps,
307
max_model_len=48,
308
max_num_batched_tokens=48,
@@ -324,7 +321,6 @@ def test_multi_step_llm_chunked_prefill_prefix_cache(
321
enforce_eager=enforce_eager,
322
gpu_memory_utilization=0.7,
323
tensor_parallel_size=tp_size,
327
- use_v2_block_manager=True,
324
enable_chunked_prefill=True,
325
enable_prefix_caching=True,
326
num_scheduler_steps=num_scheduler_steps,
tests/prefix_caching/test_prefix_caching.py
-89
@@ -2,15 +2,9 @@
2
3
Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4
"""
5
-from typing import List
6
-
5
import pytest
6
7
from tests.kernels.utils import override_backend_env_variable
10
-from tests.utils import check_deprecated_block_manager_usage
11
-from vllm.block import PhysicalTokenBlock
12
-from vllm.core.block_manager_v1 import CachedBlockAllocator
13
-from vllm.utils import Device
8
9
from ..models.utils import check_outputs_equal
10
@@ -19,92 +13,11 @@ MODELS = [
13
]
14
15
22
-@pytest.fixture(scope="module", autouse=True)
23
-def check_deprecated_block_manager():
24
- check_deprecated_block_manager_usage(
25
- 'tests/prefix_caching/test_prefix_caching.py')
26
-
27
-
28
-@pytest.mark.parametrize("block_size", [16])
29
-@pytest.mark.parametrize("num_blocks", [16])
30
-def test_block_allocator(
31
- block_size: int,
32
- num_blocks: int,
33
-):
34
- block_hash = 1
35
- block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
36
-
37
- # Allocate two PysicalTokenBlocks with the same hash and check
38
- # that they are the same PhysicalTokenBlock
39
- first_block = block_allocator.allocate(block_hash, 0)
40
- second_block = block_allocator.allocate(block_hash, 0)
41
- assert (first_block == second_block)
42
- assert (second_block.ref_count == 2)
43
-
44
- # Check metric: 1 hit of 2 queries
45
- assert block_allocator.get_prefix_cache_hit_rate() == 0.5
46
-
47
- # Free the first_block and confirm that the ref_count is correctly
48
- # decremented on the second block
49
- block_allocator.free(first_block)
50
- assert (second_block.ref_count == 1)
51
-
52
- # Free the second block
53
- block_allocator.free(second_block)
54
-
55
- # Reallocate the first block and confirm that, even after the block
56
- # had its ref_count go to 0, we still get the same block back
57
- first_block = block_allocator.allocate(block_hash, 0)
58
- assert (first_block == second_block)
59
- assert (first_block.block_hash == block_hash)
60
-
61
- # Allocate one more time to get 3/4 hit rate for easy checking
62
- block_allocator.allocate(block_hash, 0)
63
- assert block_allocator.get_prefix_cache_hit_rate() == 0.75
64
-
65
-
66
-@pytest.mark.parametrize("num_blocks", [16])
67
-def test_eviction(num_blocks: int, ):
68
- block_size = 16
69
- block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
70
- blocks: List[PhysicalTokenBlock] = []
71
-
72
- for i in range(num_blocks):
73
- # use i as the block_hash
74
- blocks.append(block_allocator.allocate(i, 0))
75
-
76
- #Free all blocks
77
- for block in blocks:
78
- block_allocator.free(block)
79
-
80
- # Allocate a new block and confirm that it's the first block freed.
81
- # I.E The Least Recently Used block
82
- new_block_hash = block_size
83
- new_block = block_allocator.allocate(new_block_hash, 0)
84
- assert (new_block == blocks[0])
85
- assert (new_block.block_hash == new_block_hash)
86
-
87
- # Reallocate the second in blocks to remove it from the free list
88
- realloc_block_hash = 1
89
- realloc_block = block_allocator.allocate(realloc_block_hash, 0)
90
- assert (realloc_block == blocks[realloc_block_hash])
91
- assert (realloc_block.block_hash == realloc_block_hash)
92
-
93
- # Allocate a new block and confirm that it's not the realloc_block,
94
- # since the realloc_block shouldn't be in the free list
95
- new_block_hash = block_size + 1
96
- new_block = block_allocator.allocate(new_block_hash, 0)
97
- assert (realloc_block != new_block)
98
- assert (new_block.block_hash == new_block_hash)
99
- assert (new_block.block_number == 2)
100
-
101
-
16
@pytest.mark.parametrize("model", MODELS)
17
@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
18
@pytest.mark.parametrize("dtype", ["half"])
19
@pytest.mark.parametrize("max_tokens", [5])
20
@pytest.mark.parametrize("cached_position", [0, 1])
107
-@pytest.mark.parametrize("use_v2_block_manager", [False, True])
21
def test_mixed_requests(
22
hf_runner,
23
vllm_runner,
@@ -114,7 +27,6 @@ def test_mixed_requests(
27
dtype: str,
28
max_tokens: int,
29
cached_position: int,
117
- use_v2_block_manager: bool,
30
monkeypatch,
31
) -> None:
32
"""
@@ -132,7 +44,6 @@ def test_mixed_requests(
44
model,
45
dtype=dtype,
46
enable_prefix_caching=True,
135
- use_v2_block_manager=use_v2_block_manager,
47
) as vllm_model:
48
# Run the first prompt so the cache is populated
49
vllm_outputs = vllm_model.generate_greedy([cached_prompt], max_tokens)
tests/spec_decode/e2e/test_compatibility.py
+10
-58
@@ -1,27 +1,15 @@
1
import pytest
2
3
-from tests.utils import check_deprecated_block_manager_usage
3
from vllm import SamplingParams
4
5
from .conftest import get_output_from_llm_generator
6
7
9
-@pytest.fixture(scope="module", autouse=True)
10
-def check_deprecated_block_manager():
11
- check_deprecated_block_manager_usage(
12
- 'tests/spec_decode/e2e/test_compatibility.py')
13
-
14
-
15
-@pytest.mark.parametrize(
16
- "common_llm_kwargs",
17
- [{
18
- "model": "JackFram/llama-68m",
19
- "speculative_model": "JackFram/llama-68m",
20
- "num_speculative_tokens": 5,
21
-
22
- # Required for spec decode.
23
- "use_v2_block_manager": True
24
- }])
8
+@pytest.mark.parametrize("common_llm_kwargs", [{
9
+ "model": "JackFram/llama-68m",
10
+ "speculative_model": "JackFram/llama-68m",
11
+ "num_speculative_tokens": 5,
12
+}])
13
@pytest.mark.parametrize("per_test_common_llm_kwargs", [
14
{
15
"enable_chunked_prefill": True,
@@ -51,16 +39,11 @@ def test_spec_decode_xfail_chunked_prefill(test_llm_generator):
39
sampling_params)
40
41
54
-@pytest.mark.parametrize(
55
- "common_llm_kwargs",
56
- [{
57
- "model": "meta-llama/Llama-2-7b-chat-hf",
58
- "speculative_model": "JackFram/llama-68m",
59
- "num_speculative_tokens": 5,
60
-
61
- # Required for spec decode.
62
- "use_v2_block_manager": True
63
- }])
42
+@pytest.mark.parametrize("common_llm_kwargs", [{
43
+ "model": "meta-llama/Llama-2-7b-chat-hf",
44
+ "speculative_model": "JackFram/llama-68m",
45
+ "num_speculative_tokens": 5,
46
+}])
47
@pytest.mark.parametrize(
48
"per_test_common_llm_kwargs",
49
[
@@ -101,34 +84,3 @@ def test_spec_decode_xfail_spec_max_model_len(test_llm_generator):
84
with pytest.raises(ValueError, match="cannot be larger than"):
85
get_output_from_llm_generator(test_llm_generator, prompts,
86
sampling_params)
104
-
105
-
106
-@pytest.mark.parametrize("common_llm_kwargs", [{
107
- "model": "JackFram/llama-68m",
108
- "speculative_model": "JackFram/llama-68m",
109
- "num_speculative_tokens": 5,
110
- "use_v2_block_manager": False,
111
-}])
112
-@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
113
-@pytest.mark.parametrize("test_llm_kwargs", [{}])
114
-@pytest.mark.parametrize("seed", [1])
115
-def test_spec_decode_xfail_block_manager_v1(test_llm_generator):
116
- """Verify that speculative decoding with block manager v1 fails.
117
- """
118
- output_len = 128
119
- temperature = 0.0
120
-
121
- prompts = [
122
- "Hello, my name is",
123
- ]
124
-
125
- sampling_params = SamplingParams(
126
- max_tokens=output_len,
127
- ignore_eos=True,
128
- temperature=temperature,
129
- )
130
-
131
- with pytest.raises(ValueError,
132
- match="Speculative decoding requires usage of the V2"):
133
- get_output_from_llm_generator(test_llm_generator, prompts,
134
- sampling_params)
tests/spec_decode/e2e/test_eagle_correctness.py
-18
@@ -43,9 +43,6 @@ PRECISION = "float32"
43
# Skip cuda graph recording for fast test.
44
"enforce_eager": True,
45
46
- # Required for spec decode.
47
- "use_v2_block_manager": True,
48
-
46
# Print spec metrics.
47
"disable_log_stats": False,
48
@@ -86,9 +83,6 @@ def test_eagle_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
83
# Skip cuda graph recording for fast test.
84
"enforce_eager": True,
85
89
- # Required for spec decode.
90
- "use_v2_block_manager": True,
91
-
86
# Print spec metrics.
87
"disable_log_stats": False,
88
@@ -143,9 +137,6 @@ def test_eagle_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
137
[{
138
"enforce_eager": False,
139
146
- # Required for spec decode.
147
- "use_v2_block_manager": True,
148
-
140
# Print spec metrics.
141
"disable_log_stats": False,
142
@@ -191,9 +182,6 @@ def test_eagle_e2e_greedy_correctness_cuda_graph(
182
# Skip cuda graph recording for fast test.
183
"enforce_eager": True,
184
194
- # Required for spec decode.
195
- "use_v2_block_manager": True,
196
-
185
# Precision
186
"dtype": PRECISION,
187
@@ -235,9 +223,6 @@ def test_eagle_e2e_greedy_correctness_with_preemption(
223
# Skip cuda graph recording for fast test.
224
"enforce_eager": True,
225
238
- # Required for spec decode.
239
- "use_v2_block_manager": True,
240
-
226
# Precision
227
"dtype": PRECISION,
228
@@ -283,9 +268,6 @@ def test_eagle_different_k(vllm_runner, common_llm_kwargs,
268
# Skip cuda graph recording for fast test.
269
"enforce_eager": True,
270
286
- # Required for spec decode.
287
- "use_v2_block_manager": True,
288
-
271
# Precision
272
"dtype": PRECISION,
273
tests/spec_decode/e2e/test_integration.py
-8
@@ -12,8 +12,6 @@ MAIN_MODEL = "JackFram/llama-68m"
12
@pytest.mark.parametrize(
13
"common_llm_kwargs",
14
[{
15
- # Required for spec decode.
16
- "use_v2_block_manager": True,
15
16
# Verify equality when cuda graphs allowed.
17
"enforce_eager": False,
@@ -57,9 +55,6 @@ def test_spec_decode_cuda_graph(vllm_runner, common_llm_kwargs,
55
56
# Skip cuda graph recording for fast test.
57
"enforce_eager": True,
60
-
61
- # Required for spec decode.
62
- "use_v2_block_manager": True,
58
}])
59
@pytest.mark.parametrize("per_test_common_llm_kwargs", [
60
{
@@ -111,9 +106,6 @@ def test_speculative_model_quantization_config(vllm_runner, common_llm_kwargs,
106
107
# Skip cuda graph recording for fast test.
108
"enforce_eager": True,
114
-
115
- # Required for spec decode.
116
- "use_v2_block_manager": True,
109
"speculative_model": "JackFram/llama-68m",
110
"num_speculative_tokens": 3,
111
}])
tests/spec_decode/e2e/test_integration_dist_tp2.py
-6
@@ -17,9 +17,6 @@ from .conftest import run_equality_correctness_test_tp
17
[[
18
# Skip cuda graph recording for fast test.
19
"--enforce-eager",
20
-
21
- # Required for spec decode.
22
- "--use-v2-block-manager",
20
"--tensor-parallel-size",
21
"2"
22
]])
@@ -74,9 +71,6 @@ def test_target_model_tp_gt_1(common_llm_kwargs, per_test_common_llm_kwargs,
71
[[
72
# Skip cuda graph recording for fast test.
73
"--enforce-eager",
77
-
78
- # Required for spec decode.
79
- "--use_v2_block_manager",
74
"--tensor_parallel_size",
75
"2",
76
tests/spec_decode/e2e/test_integration_dist_tp4.py
-6
@@ -19,9 +19,6 @@ SPEC_MODEL = "JackFram/llama-68m"
19
[[
20
# Skip cuda graph recording for fast test.
21
"--enforce_eager",
22
-
23
- # Required for spec decode.
24
- "--use-v2-block-manager",
22
"--tensor-parallel-size",
23
"4",
24
]])
@@ -71,9 +68,6 @@ def test_draft_model_tp_lt_target_model_tp4(common_llm_kwargs,
68
69
# Skip cuda graph recording for fast test.
70
"--enforce-eager",
74
-
75
- # Required for spec decode.
76
- "--use-v2-block-manager",
71
"--tensor-parallel-size",
72
"4",
73
]])
tests/spec_decode/e2e/test_logprobs.py
-14
@@ -14,9 +14,6 @@ from .conftest import run_equality_correctness_test
14
15
# Skip cuda graph recording for fast test.
16
"enforce_eager": True,
17
-
18
- # Required for spec decode.
19
- "use_v2_block_manager": True,
17
}])
18
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
19
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -67,9 +64,6 @@ def test_logprobs_equality(vllm_runner, common_llm_kwargs,
64
65
# Skip cuda graph recording for fast test.
66
"enforce_eager": True,
70
-
71
- # Required for spec decode.
72
- "use_v2_block_manager": True
67
}])
68
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
69
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -119,9 +113,6 @@ def test_logprobs_different_k(vllm_runner, common_llm_kwargs,
113
114
# Skip cuda graph recording for fast test.
115
"enforce_eager": True,
122
-
123
- # Required for spec decode.
124
- "use_v2_block_manager": True
116
}])
117
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
118
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -173,9 +164,6 @@ def test_logprobs_when_skip_speculation(vllm_runner, common_llm_kwargs,
164
165
# Skip cuda graph recording for fast test.
166
"enforce_eager": True,
176
-
177
- # Required for spec decode.
178
- "use_v2_block_manager": True
167
}])
168
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
169
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -251,8 +239,6 @@ def test_logprobs_temp_1(vllm_runner, common_llm_kwargs,
239
"model_name": "JackFram/llama-160m",
240
# Skip cuda graph recording for fast test.
241
"enforce_eager": True,
254
- # Required for spec decode.
255
- "use_v2_block_manager": True,
242
}])
243
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
244
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
tests/spec_decode/e2e/test_medusa_correctness.py
-21
@@ -45,9 +45,6 @@ PRECISION = "float32"
45
# Skip cuda graph recording for fast test.
46
"enforce_eager": True,
47
48
- # Required for spec decode.
49
- "use_v2_block_manager": True,
50
-
48
# Print spec metrics.
49
"disable_log_stats": False,
50
@@ -93,9 +90,6 @@ def test_medusa_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
90
# Skip cuda graph recording for fast test.
91
"enforce_eager": True,
92
96
- # Required for spec decode.
97
- "use_v2_block_manager": True,
98
-
93
# Print spec metrics.
94
"disable_log_stats": False,
95
@@ -151,9 +145,6 @@ def test_medusa_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
145
[{
146
"enforce_eager": False,
147
154
- # Required for spec decode.
155
- "use_v2_block_manager": True,
156
-
148
# Print spec metrics.
149
"disable_log_stats": False,
150
@@ -204,9 +195,6 @@ def test_medusa_e2e_greedy_correctness_cuda_graph(
195
# Skip cuda graph recording for fast test.
196
"enforce_eager": True,
197
207
- # Required for spec decode.
208
- "use_v2_block_manager": True,
209
-
198
# Precision
199
"dtype": PRECISION,
200
@@ -253,9 +241,6 @@ def test_medusa_e2e_greedy_correctness_with_preemption(
241
# Skip cuda graph recording for fast test.
242
"enforce_eager": True,
243
256
- # Required for spec decode.
257
- "use_v2_block_manager": True,
258
-
244
# Precision
245
"dtype": PRECISION,
246
@@ -306,9 +291,6 @@ def test_medusa_different_k(vllm_runner, common_llm_kwargs,
291
# Skip cuda graph recording for fast test.
292
"enforce_eager": True,
293
309
- # Required for spec decode.
310
- "use_v2_block_manager": True,
311
-
294
# Precision
295
"dtype": PRECISION,
296
@@ -356,9 +338,6 @@ def test_medusa_disable_queue(vllm_runner, common_llm_kwargs,
338
# Skip cuda graph recording for fast test.
339
"enforce_eager": True,
340
359
- # Required for spec decode.
360
- "use_v2_block_manager": True,
361
-
341
# Precision
342
"dtype": PRECISION,
343
tests/spec_decode/e2e/test_mlp_correctness.py
-27
@@ -47,9 +47,6 @@ PRECISION = "float32"
47
# Skip cuda graph recording for fast test.
48
"enforce_eager": True,
49
50
- # Required for spec decode.
51
- "use_v2_block_manager": True,
52
-
50
# Print spec metrics.
51
"disable_log_stats": False,
52
@@ -94,9 +91,6 @@ def test_mlp_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
91
# Skip cuda graph recording for fast test.
92
"enforce_eager": True,
93
97
- # Required for spec decode.
98
- "use_v2_block_manager": True,
99
-
94
# Print spec metrics.
95
"disable_log_stats": False,
96
@@ -149,9 +143,6 @@ def test_mlp_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
143
# Skip cuda graph recording for fast test.
144
"enforce_eager": True,
145
152
- # Required for spec decode.
153
- "use_v2_block_manager": True,
154
-
146
# Print spec metrics.
147
"disable_log_stats": False,
148
@@ -195,9 +186,6 @@ def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
186
# Skip cuda graph recording for fast test.
187
"enforce_eager": True,
188
198
- # Required for spec decode.
199
- "use_v2_block_manager": True,
200
-
189
# Print spec metrics.
190
"disable_log_stats": False,
191
@@ -258,9 +246,6 @@ def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
246
# Skip cuda graph recording for fast test.
247
"enforce_eager": True,
248
261
- # Required for spec decode.
262
- "use_v2_block_manager": True,
263
-
249
# Precision
250
"dtype": PRECISION,
251
@@ -311,9 +296,6 @@ def test_mlp_e2e_greedy_correctness_with_preemption(
296
# Skip cuda graph recording for fast test.
297
"enforce_eager": True,
298
314
- # Required for spec decode.
315
- "use_v2_block_manager": True,
316
-
299
# Precision
300
"dtype": PRECISION,
301
@@ -366,9 +348,6 @@ def test_mlp_e2e_greedy_correctness_with_padding(
348
# Skip cuda graph recording for fast test.
349
"enforce_eager": True,
350
369
- # Required for spec decode.
370
- "use_v2_block_manager": True,
371
-
351
# Precision
352
"dtype": PRECISION,
353
@@ -419,9 +398,6 @@ def test_mlp_different_k(vllm_runner, common_llm_kwargs,
398
# Skip cuda graph recording for fast test.
399
"enforce_eager": True,
400
422
- # Required for spec decode.
423
- "use_v2_block_manager": True,
424
-
401
# Precision
402
"dtype": PRECISION,
403
@@ -469,9 +445,6 @@ def test_mlp_disable_queue(vllm_runner, common_llm_kwargs,
445
446
# Skip cuda graph recording for fast test.
447
"enforce_eager": True,
472
-
473
- # Required for spec decode.
474
- "use_v2_block_manager": True,
448
"speculative_model": SPEC_MODEL,
449
}])
450
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
tests/spec_decode/e2e/test_multistep_correctness.py
-36
@@ -55,9 +55,6 @@ from .conftest import (get_output_from_llm_generator,
55
56
# Skip cuda graph recording for fast test.
57
"enforce_eager": True,
58
-
59
- # Required for spec decode.
60
- "use_v2_block_manager": True,
58
}])
59
@pytest.mark.parametrize(
60
"per_test_common_llm_kwargs",
@@ -124,9 +121,6 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
121
# Skip cuda graph recording for fast test.
122
"enforce_eager": True,
123
127
- # Required for spec decode.
128
- "use_v2_block_manager": True,
129
-
124
# Print spec metrics.
125
"disable_log_stats": False,
126
}])
@@ -190,9 +184,6 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
184
# Skip cuda graph recording for fast test.
185
"enforce_eager": True,
186
193
- # Required for spec decode.
194
- "use_v2_block_manager": True,
195
-
187
# Print spec metrics.
188
"disable_log_stats": False,
189
}])
@@ -246,9 +237,6 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
237
[{
238
# Skip cuda graph recording for fast test.
239
"enforce_eager": True,
249
-
250
- # Required for spec decode.
251
- "use_v2_block_manager": True
240
}])
241
@pytest.mark.parametrize(
242
"per_test_common_llm_kwargs",
@@ -303,9 +291,6 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(
291
# Skip cuda graph recording for fast test.
292
"enforce_eager": True,
293
306
- # Required for spec decode.
307
- "use_v2_block_manager": True,
308
-
294
# Print spec metrics.
295
"disable_log_stats": False,
296
}])
@@ -353,9 +338,6 @@ def test_spec_decode_e2e_greedy_correctness_real_model_bs1(
338
# Skip cuda graph recording for fast test.
339
"enforce_eager": True,
340
356
- # Required for spec decode.
357
- "use_v2_block_manager": True,
358
-
341
# Print spec metrics.
342
"disable_log_stats": False,
343
}])
@@ -404,9 +386,6 @@ def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
386
387
# Skip cuda graph recording for fast test.
388
"enforce_eager": True,
407
-
408
- # Required for spec decode.
409
- "use_v2_block_manager": True
389
}])
390
@pytest.mark.parametrize("per_test_common_llm_kwargs", [
391
{
@@ -454,9 +433,6 @@ def test_spec_decode_e2e_greedy_correctness_with_preemption(
433
434
# Skip cuda graph recording for fast test.
435
"enforce_eager": True,
457
-
458
- # Required for spec decode.
459
- "use_v2_block_manager": True
436
}])
437
@pytest.mark.parametrize(
438
"per_test_common_llm_kwargs",
@@ -514,9 +490,6 @@ def test_spec_decode_different_block_size(vllm_runner, common_llm_kwargs,
490
491
# Skip cuda graph recording for fast test.
492
"enforce_eager": True,
517
-
518
- # Required for spec decode.
519
- "use_v2_block_manager": True
493
}])
494
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
495
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -570,9 +543,6 @@ def test_skip_speculation(vllm_runner, common_llm_kwargs,
543
544
# Skip cuda graph recording for fast test.
545
"enforce_eager": True,
573
-
574
- # Required for spec decode.
575
- "use_v2_block_manager": True
546
}])
547
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
548
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -611,9 +581,6 @@ def test_disable_speculation(vllm_runner, common_llm_kwargs,
581
582
# Skip cuda graph recording for fast test.
583
"enforce_eager": True,
614
-
615
- # Required for spec decode.
616
- "use_v2_block_manager": True
584
}])
585
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
586
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -660,9 +627,6 @@ def test_many_k(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
627
628
# Skip cuda graph recording for fast test.
629
"enforce_eager": True,
663
-
664
- # Required for spec decode.
665
- "use_v2_block_manager": True
630
}])
631
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
632
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
tests/spec_decode/e2e/test_ngram_correctness.py
-16
@@ -35,9 +35,6 @@ from .conftest import run_equality_correctness_test
35
# Skip cuda graph recording for fast test.
36
"enforce_eager": True,
37
38
- # Required for spec decode.
39
- "use_v2_block_manager": True,
40
-
38
# Print spec metrics.
39
"disable_log_stats": False,
40
}])
@@ -82,9 +79,6 @@ def test_ngram_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
79
# Skip cuda graph recording for fast test.
80
"enforce_eager": True,
81
85
- # Required for spec decode.
86
- "use_v2_block_manager": True,
87
-
82
# Print spec metrics.
83
"disable_log_stats": False,
84
}])
@@ -145,9 +139,6 @@ def test_ngram_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
139
140
# Skip cuda graph recording for fast test.
141
"enforce_eager": True,
148
-
149
- # Required for spec decode.
150
- "use_v2_block_manager": True
142
}])
143
@pytest.mark.parametrize("per_test_common_llm_kwargs", [
144
{
@@ -195,9 +186,6 @@ def test_ngram_e2e_greedy_correctness_with_preemption(
186
187
# Skip cuda graph recording for fast test.
188
"enforce_eager": True,
198
-
199
- # Required for spec decode.
200
- "use_v2_block_manager": True
189
}])
190
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
191
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -254,9 +242,6 @@ def test_ngram_different_k(vllm_runner, common_llm_kwargs,
242
243
# Skip cuda graph recording for fast test.
244
"enforce_eager": True,
257
-
258
- # Required for spec decode.
259
- "use_v2_block_manager": True
245
}])
246
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
247
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -303,7 +288,6 @@ def test_ngram_disable_queue(vllm_runner, common_llm_kwargs,
288
"enforce_eager": True,
289
290
# Required for spec decode.
306
- "use_v2_block_manager": True,
291
"speculative_model": "[ngram]",
292
"num_speculative_tokens": 5,
293
"ngram_prompt_lookup_max": 3,
tests/spec_decode/e2e/test_seed.py
-3
@@ -17,9 +17,6 @@ SPEC_MODEL = "JackFram/llama-160m"
17
# Skip cuda graph recording for fast test.
18
"enforce_eager": True,
19
20
- # Required for spec decode.
21
- "use_v2_block_manager": True,
22
-
20
# speculative model
21
"speculative_model": "JackFram/llama-160m",
22
tests/utils.py
-9
@@ -678,12 +678,3 @@ def get_client_text_logprob_generations(
678
return [(text_generations, text,
679
(None if x.logprobs is None else x.logprobs.top_logprobs))
680
for completion in completions for x in completion.choices]
681
-
682
-
683
-def check_deprecated_block_manager_usage(test_name: str):
684
- assert envs.VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1 is True, (
685
- f"To allow the use of deprecated BlockSpaceManagerV1, set the "
686
- f"environment variable VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1. "
687
- f"You can run the tests with: "
688
- f"`VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest {test_name}`" #noqa
689
- )
vllm/attention/backends/flash_attn.py
+3
-5
@@ -305,8 +305,6 @@ class FlashAttentionMetadataBuilder(
305
self.runner = input_builder.runner
306
self.sliding_window = input_builder.sliding_window
307
self.block_size = input_builder.block_size
308
- self.use_v2_block_manager = (
309
- input_builder.scheduler_config.use_v2_block_manager)
308
309
def _add_seq_group(
310
self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
@@ -355,9 +353,9 @@ class FlashAttentionMetadataBuilder(
353
354
# Compute slot mapping.
355
is_profile_run = is_block_tables_empty(block_tables)
358
- start_idx = compute_slot_mapping_start_idx(
359
- is_prompt, query_len, context_len, self.sliding_window,
360
- self.use_v2_block_manager)
356
+ start_idx = compute_slot_mapping_start_idx(is_prompt, query_len,
357
+ context_len,
358
+ self.sliding_window)
359
compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
360
seq_len, context_len, start_idx,
361
self.block_size, inter_data.block_tables)
vllm/attention/backends/flashinfer.py
+3
-5
@@ -475,8 +475,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
475
476
self.sliding_window = input_builder.sliding_window
477
self.block_size = input_builder.block_size
478
- self.use_v2_block_manager = (
479
- input_builder.scheduler_config.use_v2_block_manager)
478
479
# Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout
480
# for the precise definition of the following fields.
@@ -542,9 +540,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
540
is_profile_run = is_block_tables_empty(block_tables)
541
542
# Compute slot mapping.
545
- start_idx = compute_slot_mapping_start_idx(
546
- is_prompt, query_len, context_len, self.sliding_window,
547
- self.use_v2_block_manager)
543
+ start_idx = compute_slot_mapping_start_idx(is_prompt, query_len,
544
+ context_len,
545
+ self.sliding_window)
546
compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
547
seq_len, context_len, start_idx,
548
self.block_size, inter_data.block_tables)
vllm/attention/backends/utils.py
+4
-12
@@ -38,18 +38,12 @@ def is_block_tables_empty(block_tables: Union[None, Dict]):
38
39
40
def compute_slot_mapping_start_idx(is_prompt: bool, query_len: int,
41
- context_len: int, sliding_window: int,
42
- use_v2_block_manager: bool):
41
+ context_len: int, sliding_window: int):
42
"""
43
Compute the start index of slot mapping.
44
"""
45
start_idx = 0
46
if is_prompt and sliding_window is not None:
48
- assert use_v2_block_manager or context_len == 0, (
49
- "Prefix caching is currently not supported with "
50
- "sliding window attention in V1 block manager")
51
- # When prefill, we use it to not write slots to kv cache
52
- # to save memory.
47
start_idx = max(0, query_len - sliding_window)
48
return start_idx
49
@@ -138,8 +132,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
132
133
self.sliding_window = input_builder.sliding_window
134
self.block_size = input_builder.block_size
141
- self.use_v2_block_manager = (
142
- input_builder.scheduler_config.use_v2_block_manager)
135
136
def _add_seq_group(
137
self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
@@ -180,9 +172,9 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
172
173
# Compute slot mapping.
174
is_profile_run = is_block_tables_empty(block_tables)
183
- start_idx = compute_slot_mapping_start_idx(
184
- is_prompt, query_len, context_len, self.sliding_window,
185
- self.use_v2_block_manager)
175
+ start_idx = compute_slot_mapping_start_idx(is_prompt, query_len,
176
+ context_len,
177
+ self.sliding_window)
178
compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
179
seq_len, context_len, start_idx,
180
self.block_size, inter_data.block_tables)
vllm/commit_id.py
new
+1
@@ -0,0 +1 @@
1
+__commit__ = "93ec62b8556e279d2c050bdc1c3247831bd39466"
vllm/config.py
-24
@@ -949,7 +949,6 @@ class SchedulerConfig:
949
iteration.
950
max_model_len: Maximum length of a sequence (including prompt
951
and generated text).
952
- use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
952
num_lookahead_slots: The number of slots to allocate per sequence per
953
step, beyond the known token ids. This is used in speculative
954
decoding to store KV activations of tokens which may or may not be
@@ -976,7 +975,6 @@ class SchedulerConfig:
975
max_num_batched_tokens: Optional[int],
976
max_num_seqs: int,
977
max_model_len: int,
979
- use_v2_block_manager: bool = True,
978
num_lookahead_slots: int = 0,
979
delay_factor: float = 0.0,
980
enable_chunked_prefill: bool = False,
@@ -1026,7 +1024,6 @@ class SchedulerConfig:
1024
1025
self.max_num_seqs = max_num_seqs
1026
self.max_model_len = max_model_len
1029
- self.use_v2_block_manager = use_v2_block_manager
1027
self.num_lookahead_slots = num_lookahead_slots
1028
self.delay_factor = delay_factor
1029
self.chunked_prefill_enabled = enable_chunked_prefill
@@ -1067,18 +1064,6 @@ class SchedulerConfig:
1064
f"({self.num_scheduler_steps}) must be greater than or "
1065
"equal to 1.")
1066
1070
- if (not self.use_v2_block_manager \
1071
- and not envs.VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1):
1072
- raise ValueError(
1073
- "The use of BlockSpaceManagerV1 is deprecated and will "
1074
- "be removed in a future release. Please switch to "
1075
- "BlockSpaceManagerV2 by setting --use-v2-block-manager to "
1076
- "True. If you wish to suppress this error temporarily, "
1077
- "you can set the environment variable "
1078
- "`VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1. If your use "
1079
- "case is not supported in BlockSpaceManagerV2, please "
1080
- "file an issue with detailed information.")
1081
-
1067
@property
1068
def is_multi_step(self) -> bool:
1069
return self.num_scheduler_steps > 1
@@ -1137,7 +1122,6 @@ class SpeculativeConfig:
1122
speculative_disable_mqa_scorer: Optional[bool],
1123
speculative_max_model_len: Optional[int],
1124
enable_chunked_prefill: bool,
1140
- use_v2_block_manager: bool,
1125
disable_log_stats: bool,
1126
speculative_disable_by_batch_size: Optional[int],
1127
ngram_prompt_lookup_max: Optional[int],
@@ -1178,9 +1162,6 @@ class SpeculativeConfig:
1162
enable_chunked_prefill (bool): Whether vLLM is configured to use
1163
chunked prefill or not. Used for raising an error since its not
1164
yet compatible with spec decode.
1181
- use_v2_block_manager (bool): Whether vLLM is configured to use the
1182
- v2 block manager or not. Used for raising an error since the v2
1183
- block manager is required with spec decode.
1165
speculative_disable_by_batch_size (Optional[int]): Disable
1166
speculative decoding for new incoming requests when the number
1167
of enqueue requests is larger than this value, if provided.
@@ -1231,11 +1212,6 @@ class SpeculativeConfig:
1212
"Speculative decoding and chunked prefill are "
1213
f"currently mutually exclusive ({enable_chunked_prefill=}).")
1214
1234
- if not use_v2_block_manager:
1235
- raise ValueError(
1236
- "Speculative decoding requires usage of the V2 "
1237
- "block manager. Enable it with --use-v2-block-manager.")
1238
-
1215
# TODO: The user should be able to specify revision/max model len
1216
# for the draft model. It is not currently supported.
1217
draft_revision = None
vllm/core/block/utils.py
+1
-23
@@ -4,28 +4,6 @@ from vllm.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
4
STR_NOT_IMPL_ENC_DEC_SWA)
5
6
7
-def _get_block_mgr_sliding_window_attr(block_mgr):
8
- '''
9
- BlockManagerV1 and BlockManagerV2 have slightly different
10
- members related to sliding window attention (SWA). This
11
- function extracts the appropriate member to use for determining
12
- whether SWA is enabled.
13
-
14
- Arguments:
15
-
16
- * block_mgr: BlockManagerV1 or BlockManagerV2 instance
17
- '''
18
-
19
- if hasattr(block_mgr, 'block_sliding_window'):
20
- return block_mgr.block_sliding_window
21
- if hasattr(block_mgr, 'max_block_sliding_window'):
22
- return block_mgr.max_block_sliding_window
23
-
24
- raise AttributeError("Block manager instance has neither " + \
25
- "block_sliding_window nor " + \
26
- "max_block_sliding_window attributes.")
27
-
28
-
7
def check_no_caching_or_swa_for_blockmgr_encdec(
8
block_mgr, seq_group: SequenceGroup) -> None:
9
'''
@@ -41,7 +19,7 @@ def check_no_caching_or_swa_for_blockmgr_encdec(
19
'''
20
21
if seq_group.is_encoder_decoder():
44
- if _get_block_mgr_sliding_window_attr(block_mgr) is not None:
22
+ if block_mgr.max_block_sliding_window is not None:
23
raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_SWA)
24
25
if block_mgr.enable_caching:
vllm/core/block_manager.py
renamed
+1
-1
@@ -17,7 +17,7 @@ SeqId = int
17
EncoderSeqId = str
18
19
20
-class BlockSpaceManagerV2(BlockSpaceManager):
20
+class SelfAttnBlockSpaceManager(BlockSpaceManager):
21
"""BlockSpaceManager which manages the allocation of KV cache.
22
23
It owns responsibility for allocation, swapping, allocating memory for
vllm/core/block_manager_v1.py
deleted
-743
@@ -1,743 +0,0 @@
1
-"""A block manager that manages token blocks."""
2
-import math
3
-from abc import ABC, abstractmethod
4
-from itertools import count, takewhile
5
-from os.path import commonprefix
6
-from typing import Dict, List, Optional
7
-from typing import Sequence as GenericSequence
8
-from typing import Set, Tuple
9
-
10
-from vllm.block import BlockTable, PhysicalTokenBlock
11
-from vllm.core.block.common import CacheMetricData
12
-from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
13
-from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
14
-from vllm.core.interfaces import AllocStatus, BlockSpaceManager
15
-from vllm.logger import init_logger
16
-from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
17
-from vllm.utils import Device
18
-
19
-logger = init_logger(__name__)
20
-
21
-
22
-class BlockAllocatorBase(ABC):
23
- """Manages free physical token blocks for a device.
24
-
25
- The allocator maintains a list of free blocks and allocates a block when
26
- requested. When a block is freed, its reference count is decremented. If
27
- the reference count becomes zero, the block is added back to the free list.
28
- """
29
-
30
- @abstractmethod
31
- def __init__(self,
32
- device: Device,
33
- block_size: int,
34
- num_blocks: int,
35
- eviction_policy: EvictionPolicy = EvictionPolicy.LRU):
36
- pass
37
-
38
- @abstractmethod
39
- def allocate(self,
40
- block_hash: Optional[int] = None,
41
- num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
42
- pass
43
-
44
- @abstractmethod
45
- def free(self, block: PhysicalTokenBlock) -> None:
46
- pass
47
-
48
- @abstractmethod
49
- def get_num_free_blocks(self) -> int:
50
- pass
51
-
52
- @abstractmethod
53
- def get_num_total_blocks(self) -> int:
54
- pass
55
-
56
- @abstractmethod
57
- def contains_block(self, block_hash: int) -> bool:
58
- pass
59
-
60
- @abstractmethod
61
- def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
62
- pass
63
-
64
- @abstractmethod
65
- def get_prefix_cache_hit_rate(self) -> float:
66
- """Prefix cache hit rate. -1 means not supported or disabled."""
67
- pass
68
-
69
-
70
-class CachedBlockAllocator(BlockAllocatorBase):
71
- """Manages free physical token blocks for a device.
72
-
73
- The allocator maintains a list of free blocks and allocates a block when
74
- requested. When a block is freed, its reference count is decremented. If
75
- the reference count becomes zero, the block is added back to the free list.
76
- """
77
-
78
- def __init__(self,
79
- device: Device,
80
- block_size: int,
81
- num_blocks: int,
82
- eviction_policy: EvictionPolicy = EvictionPolicy.LRU) -> None:
83
- self.device = device
84
- self.block_size = block_size
85
- self.num_blocks = num_blocks
86
-
87
- self.current_num_blocks = 0
88
- self.cached_blocks: Dict[int, PhysicalTokenBlock] = {}
89
-
90
- self.evictor: Evictor = make_evictor(eviction_policy)
91
-
92
- self.default_hash_ctr = count()
93
-
94
- self.cache_metric_data = CacheMetricData()
95
-
96
- def allocate_block(self, block_hash: int,
97
- num_hashed_tokens: int) -> PhysicalTokenBlock:
98
- if self.current_num_blocks == self.num_blocks:
99
- block = self.evictor.evict()
100
- block.block_hash = block_hash
101
- block.num_hashed_tokens = num_hashed_tokens
102
- return block
103
- block = PhysicalTokenBlock(device=self.device,
104
- block_number=self.current_num_blocks,
105
- block_size=self.block_size,
106
- block_hash=block_hash,
107
- num_hashed_tokens=num_hashed_tokens)
108
- self.current_num_blocks += 1
109
- return block
110
-
111
- def allocate(self,
112
- block_hash: Optional[int] = None,
113
- num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
114
- if block_hash is None:
115
- block_hash = next(self.default_hash_ctr)
116
-
117
- if block_hash in self.evictor:
118
- assert block_hash not in self.cached_blocks
119
- block = self.evictor.remove(block_hash)
120
- assert block.ref_count == 0
121
- self.cached_blocks[block_hash] = block
122
-
123
- if block_hash in self.cached_blocks:
124
- self.cache_metric_data.query(hit=True)
125
- else:
126
- self.cache_metric_data.query(hit=False)
127
- self.cached_blocks[block_hash] = self.allocate_block(
128
- block_hash, num_hashed_tokens)
129
- block = self.cached_blocks[block_hash]
130
- assert block.block_hash == block_hash
131
- block.ref_count += 1
132
- return block
133
-
134
- def free(self, block: PhysicalTokenBlock) -> None:
135
- if block.ref_count == 0:
136
- raise ValueError(f"Double free! {block} is already freed.")
137
- block.ref_count -= 1
138
- if block.ref_count == 0:
139
- assert block.block_hash not in self.evictor
140
- self.evictor.add(block)
141
-
142
- # Remove the block from the cached_blocks
143
- del self.cached_blocks[block.block_hash]
144
-
145
- def get_num_free_blocks(self) -> int:
146
- return (self.num_blocks - self.current_num_blocks +
147
- self.evictor.num_blocks)
148
-
149
- def get_num_total_blocks(self) -> int:
150
- return self.num_blocks
151
-
152
- def contains_block(self, block_hash: int) -> bool:
153
- return block_hash in self.cached_blocks or block_hash in self.evictor
154
-
155
- def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
156
- # Update the hash of block and the cached_blocks dictionary.
157
- assert not self.contains_block(block_hash)
158
- old_hash = block.block_hash
159
- block.block_hash = block_hash
160
- del self.cached_blocks[old_hash]
161
- self.cached_blocks[block_hash] = block
162
-
163
- def get_prefix_cache_hit_rate(self) -> float:
164
- return self.cache_metric_data.get_hit_rate()
165
-
166
-
167
-class UncachedBlockAllocator(BlockAllocatorBase):
168
- """Manages free physical token blocks for a device.
169
-
170
- The allocator maintains a list of free blocks and allocates a block when
171
- requested. When a block is freed, its reference count is decremented. If
172
- the reference count becomes zero, the block is added back to the free list.
173
- """
174
-
175
- def __init__(
176
- self,
177
- device: Device,
178
- block_size: int,
179
- num_blocks: int,
180
- ) -> None:
181
- self.device = device
182
- self.block_size = block_size
183
- self.num_blocks = num_blocks
184
-
185
- # Initialize the free blocks.
186
- self.free_blocks: List[PhysicalTokenBlock] = []
187
- for i in range(num_blocks):
188
- block = PhysicalTokenBlock(device=device,
189
- block_number=i,
190
- block_size=block_size,
191
- block_hash=-1,
192
- num_hashed_tokens=0)
193
- self.free_blocks.append(block)
194
-
195
- def allocate(self,
196
- block_hash: Optional[int] = None,
197
- num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
198
- if not self.free_blocks:
199
- raise ValueError("Out of memory! No free blocks are available.")
200
- block = self.free_blocks.pop()
201
- block.ref_count = 1
202
- return block
203
-
204
- def free(self, block: PhysicalTokenBlock) -> None:
205
- if block.ref_count == 0:
206
- raise ValueError(f"Double free! {block} is already freed.")
207
- block.ref_count -= 1
208
- if block.ref_count == 0:
209
- self.free_blocks.append(block)
210
-
211
- def get_num_free_blocks(self) -> int:
212
- return len(self.free_blocks)
213
-
214
- def get_num_total_blocks(self) -> int:
215
- return self.num_blocks
216
-
217
- def contains_block(self, block_hash: int) -> bool:
218
- raise NotImplementedError(
219
- "Invalid codepath for uncached block allocator.")
220
-
221
- def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
222
- raise NotImplementedError(
223
- "Invalid codepath for uncached block allocator.")
224
-
225
- def get_prefix_cache_hit_rate(self) -> float:
226
- return -1
227
-
228
-
229
-class BlockSpaceManagerV1(BlockSpaceManager):
230
- """Manages the mapping between logical and physical token blocks."""
231
-
232
- def __init__(
233
- self,
234
- block_size: int,
235
- num_gpu_blocks: int,
236
- num_cpu_blocks: int,
237
- watermark: float = 0.01,
238
- sliding_window: Optional[int] = None,
239
- enable_caching: bool = False,
240
- ) -> None:
241
- self.block_size = block_size
242
- self.num_total_gpu_blocks = num_gpu_blocks
243
- self.num_total_cpu_blocks = num_cpu_blocks
244
-
245
- if enable_caching and sliding_window is not None:
246
- raise NotImplementedError(
247
- "Sliding window is not allowed with prefix caching enabled!")
248
-
249
- self.block_sliding_window = None
250
- if sliding_window is not None:
251
- # Round up to nearest block size to regularize sliding window
252
- # allocation sizes.
253
- self.block_sliding_window = math.ceil(sliding_window / block_size)
254
-
255
- self.watermark = watermark
256
- assert watermark >= 0.0
257
-
258
- self.enable_caching = enable_caching
259
-
260
- self.watermark_blocks = int(watermark * num_gpu_blocks)
261
-
262
- if self.enable_caching:
263
- logger.info("Automatic prefix caching is enabled.")
264
- self.gpu_allocator: BlockAllocatorBase = CachedBlockAllocator(
265
- Device.GPU, block_size, num_gpu_blocks)
266
- self.cpu_allocator: BlockAllocatorBase = CachedBlockAllocator(
267
- Device.CPU, block_size, num_cpu_blocks)
268
- else:
269
- self.gpu_allocator = UncachedBlockAllocator(
270
- Device.GPU, block_size, num_gpu_blocks)
271
- self.cpu_allocator = UncachedBlockAllocator(
272
- Device.CPU, block_size, num_cpu_blocks)
273
- # Mapping: seq_id -> BlockTable.
274
- self.block_tables: Dict[int, BlockTable] = {}
275
-
276
- # Mapping: req_id -> BlockTable
277
- # Note that each SequenceGroup has a unique
278
- # request ID
279
- self.cross_block_tables: Dict[str, BlockTable] = {}
280
-
281
- def _get_seq_num_required_blocks(self, seq: Optional[Sequence]) -> int:
282
- return 0 if seq is None else seq.n_blocks
283
-
284
- def can_allocate(self,
285
- seq_group: SequenceGroup,
286
- num_lookahead_slots: int = 0) -> AllocStatus:
287
- # FIXME(woosuk): Here we assume that all sequences in the group share
288
- # the same prompt. This may not be true for preempted sequences.
289
-
290
- assert (num_lookahead_slots == 0
291
- ), "lookahead allocation not supported in BlockSpaceManagerV1"
292
-
293
- check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
294
-
295
- self_num_required_blocks = self._get_seq_num_required_blocks(
296
- seq_group.get_seqs(status=SequenceStatus.WAITING)[0])
297
- cross_num_required_blocks = self._get_seq_num_required_blocks(
298
- seq_group.get_encoder_seq())
299
- num_required_blocks = self_num_required_blocks + \
300
- cross_num_required_blocks
301
-
302
- if self.block_sliding_window is not None:
303
-
304
- num_required_blocks = min(num_required_blocks,
305
- self.block_sliding_window)
306
- num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
307
-
308
- # Use watermark to avoid frequent cache eviction.
309
- if (self.num_total_gpu_blocks - num_required_blocks <
310
- self.watermark_blocks):
311
- return AllocStatus.NEVER
312
- if num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks:
313
- return AllocStatus.OK
314
- else:
315
- return AllocStatus.LATER
316
-
317
- def _allocate_sequence(self, \
318
- seq: Optional[Sequence], \
319
- ref_count: int, \
320
- is_encoder_decoder: bool = True) -> BlockTable:
321
- # Allocate new physical token blocks that will store the prompt tokens.
322
- num_prompt_blocks = self._get_seq_num_required_blocks(seq)
323
-
324
- block_table: BlockTable = BlockTable()
325
- assert seq is not None
326
- for logical_idx in range(num_prompt_blocks):
327
- if (self.block_sliding_window is not None
328
- and logical_idx >= self.block_sliding_window):
329
- block = block_table[logical_idx % self.block_sliding_window]
330
- # Set the reference counts of the token blocks.
331
- block.ref_count = ref_count
332
- elif not is_encoder_decoder and self.enable_caching:
333
- block = self.gpu_allocator.allocate(
334
- seq.hash_of_block(logical_idx),
335
- seq.num_hashed_tokens_of_block(logical_idx))
336
- else:
337
- block = self.gpu_allocator.allocate()
338
- # Set the reference counts of the token blocks.
339
- block.ref_count = ref_count
340
- block_table.append(block)
341
-
342
- return block_table
343
-
344
- def allocate(self, seq_group: SequenceGroup) -> None:
345
- is_encoder_decoder = seq_group.is_encoder_decoder()
346
- check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
347
-
348
- # Allocate decoder sequences
349
- #
350
- # NOTE: Here we assume that all sequences in the group have the same
351
- # decoder prompt.
352
- wait_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
353
- seq = wait_seqs[0]
354
- block_table: BlockTable = \
355
- self._allocate_sequence(seq,
356
- seq_group.num_seqs(),
357
- is_encoder_decoder)
358
-
359
- # Assign the self-attention block tables for each sequence.
360
- if len(wait_seqs) == 1:
361
- self.block_tables[seq.seq_id] = block_table
362
- else:
363
- for seq in wait_seqs:
364
- self.block_tables[seq.seq_id] = block_table.copy()
365
-
366
- # Allocate encoder sequence
367
- if is_encoder_decoder:
368
- # A SequenceGroup has only a single encoder sequence (at most),
369
- # thus allocate with a ref count of 1
370
- block_table = self._allocate_sequence(seq_group.get_encoder_seq(),
371
- 1, is_encoder_decoder)
372
- # Assign the cross-attention block table for the SequenceGroup.
373
- self.cross_block_tables[seq_group.request_id] = block_table
374
-
375
- def can_append_slots(self,
376
- seq_group: SequenceGroup,
377
- num_lookahead_slots: int = 0) -> bool:
378
- assert (num_lookahead_slots == 0
379
- ), "lookahead allocation not supported in BlockSpaceManagerV1"
380
-
381
- # Simple heuristic: If there is at least one free block
382
- # for each sequence, we can append.
383
- num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
384
- num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
385
- return num_seqs <= num_free_gpu_blocks
386
-
387
- def _promote_last_block(
388
- self,
389
- seq: Sequence,
390
- last_block: PhysicalTokenBlock,
391
- ) -> PhysicalTokenBlock:
392
- assert self.enable_caching
393
-
394
- # Compute a new hash for the block so that it can be shared by other
395
- # Sequences
396
- new_hash = seq.hash_of_block(seq.n_blocks - 1)
397
-
398
- # if new_hash is already in the cached table, then free last_block
399
- # and return the cached version
400
- if self.gpu_allocator.contains_block(new_hash):
401
- self.gpu_allocator.free(last_block)
402
- return self.gpu_allocator.allocate(new_hash)
403
- else:
404
- self.gpu_allocator.update_hash(new_hash, last_block)
405
- return last_block
406
-
407
- def _is_last_block_full(
408
- self,
409
- seq: Sequence,
410
- ) -> bool:
411
- token_ids_len = seq.data.get_len()
412
- return token_ids_len > 0 and token_ids_len % seq.block_size == 0
413
-
414
- def _maybe_promote_last_block(
415
- self,
416
- seq: Sequence,
417
- last_block: PhysicalTokenBlock,
418
- ) -> PhysicalTokenBlock:
419
- if self._is_last_block_full(seq):
420
- return self._promote_last_block(seq, last_block)
421
- else:
422
- return last_block
423
-
424
- def _allocate_last_physical_block(
425
- self,
426
- seq: Sequence,
427
- ) -> PhysicalTokenBlock:
428
- # Called before a new block is appended.
429
- # This is in charge of allocating a new physical block (to be appended).
430
-
431
- # None if the last block is not full. Otherwise, we set it to the
432
- # content hash.
433
- if not self.enable_caching:
434
- return self.gpu_allocator.allocate()
435
- block_hash: Optional[int] = None
436
- n_blocks = seq.n_blocks
437
- if (self._is_last_block_full(seq)):
438
- block_hash = seq.hash_of_block(n_blocks - 1)
439
- num_hashed_tokens = seq.num_hashed_tokens_of_block(n_blocks - 1)
440
-
441
- # num_hashed_tokens is used to compute future hashes
442
- # (e.g. in the hashing function, it is used to ask the sequence for
443
- # prefix tokens)
444
- new_block = self.gpu_allocator.allocate(block_hash, num_hashed_tokens)
445
-
446
- # If the block_hash is None, then the block is not full.
447
- # If the block is not full, then we expect it to have a refcount of 1.
448
- if block_hash is None:
449
- assert new_block.ref_count == 1
450
- return new_block
451
-
452
- def append_slots(
453
- self,
454
- seq: Sequence,
455
- num_lookahead_slots: int = 0,
456
- ) -> List[Tuple[int, int]]:
457
- """Allocate a physical slot for a new token."""
458
- n_blocks = seq.n_blocks
459
- block_table = self.block_tables[seq.seq_id]
460
- # If we need to allocate a new physical block
461
- if len(block_table) < n_blocks:
462
- # Currently this code only supports adding one physical block
463
- assert len(block_table) == n_blocks - 1
464
-
465
- if (self.block_sliding_window
466
- and len(block_table) >= self.block_sliding_window):
467
- # reuse a block
468
- block_table.append(block_table[len(block_table) %
469
- self.block_sliding_window])
470
- else:
471
- # The sequence hash a new logical block.
472
- # Allocate a new physical block.
473
- new_block = self._allocate_last_physical_block(seq)
474
- block_table.append(new_block)
475
- return []
476
-
477
- # We want to append the token to the last physical block.
478
- last_block = block_table[-1]
479
- assert last_block.device == Device.GPU
480
- if last_block.ref_count == 1:
481
- # Not shared with other sequences. Appendable.
482
- if self.enable_caching:
483
- # If the last block is now complete, we may reuse an old block
484
- # to save memory.
485
- maybe_new_block = self._maybe_promote_last_block(
486
- seq, last_block)
487
- block_table[-1] = maybe_new_block
488
- return []
489
- else:
490
- # The last block is shared with other sequences.
491
- # Copy on Write: Allocate a new block and copy the tokens.
492
- new_block = self._allocate_last_physical_block(seq)
493
-
494
- block_table[-1] = new_block
495
- self.gpu_allocator.free(last_block)
496
- return [(last_block.block_number, new_block.block_number)]
497
-
498
- def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
499
- # NOTE: fork does not allocate a new physical block.
500
- # Thus, it is always safe from OOM.
501
- if parent_seq.seq_id not in self.block_tables:
502
- # Parent sequence has either been freed or never existed.
503
- return
504
- src_block_table = self.block_tables[parent_seq.seq_id]
505
- self.block_tables[child_seq.seq_id] = src_block_table.copy()
506
-
507
- # When using a sliding window, blocks will be eventually reused.
508
- # In this case the block tables will contain repeated blocks.
509
- # When forking, we must make sure that each block's `ref_count`
510
- # is only incremented by one, so we deduplicate them by wrapping
511
- # them in a set.
512
- for block in set(src_block_table):
513
- block.ref_count += 1
514
-
515
- def _get_physical_blocks(
516
- self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
517
-
518
- # NOTE: Here, we assume that the physical blocks are only shared by
519
- # the sequences in the same group.
520
- request_id = seq_group.request_id
521
- blocks: Set[PhysicalTokenBlock] = set()
522
- for seq in seq_group.get_seqs():
523
- if seq.is_finished():
524
- continue
525
- blocks.update(self.block_tables[seq.seq_id])
526
- # Cross-attention blocks
527
- if seq_group.is_encoder_decoder():
528
- blocks.update(self.cross_block_tables[request_id])
529
- return list(blocks)
530
-
531
- def can_swap_in(self,
532
- seq_group: SequenceGroup,
533
- num_lookahead_slots: int = 0) -> AllocStatus:
534
- assert (num_lookahead_slots == 0
535
- ), "BlockSpaceManagerV1 does not support lookahead allocation"
536
-
537
- blocks = self._get_physical_blocks(seq_group)
538
- num_swapped_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
539
- if seq_group.is_encoder_decoder():
540
- num_swapped_seqs += 1
541
- num_free_blocks = self.gpu_allocator.get_num_free_blocks()
542
- # NOTE: Conservatively, we assume that every sequence will allocate
543
- # at least one free block right after the swap-in.
544
- # NOTE: This should match the logic in can_append_slot().
545
- num_required_blocks = len(blocks) + num_swapped_seqs
546
- if self.gpu_allocator.get_num_total_blocks() < num_required_blocks:
547
- return AllocStatus.NEVER
548
- elif num_free_blocks - num_required_blocks >= self.watermark_blocks:
549
- return AllocStatus.OK
550
- else:
551
- return AllocStatus.LATER
552
-
553
- def _swap_block_table(
554
- self, block_table: BlockTable, src_allocator: BlockAllocatorBase,
555
- dest_allocator: BlockAllocatorBase,
556
- mapping: Dict[PhysicalTokenBlock,
557
- PhysicalTokenBlock]) -> BlockTable:
558
- new_block_table: BlockTable = BlockTable()
559
-
560
- for from_block in block_table:
561
- if from_block in mapping:
562
- to_block = mapping[from_block]
563
- to_block.ref_count += 1
564
- else:
565
- to_block = dest_allocator.allocate(
566
- from_block.block_hash, from_block.num_hashed_tokens)
567
- mapping[from_block] = to_block
568
- new_block_table.append(to_block)
569
- # Free the source block swapped in to destination.
570
- src_allocator.free(from_block)
571
-
572
- return new_block_table
573
-
574
- def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
575
-
576
- request_id = seq_group.request_id
577
-
578
- # CPU block -> GPU block.
579
- # dict is efficient in lookup `if cpu_block in mapping`
580
- mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
581
- for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
582
- self.block_tables[seq.seq_id] = \
583
- self._swap_block_table(self.block_tables[seq.seq_id],
584
- self.cpu_allocator, self.gpu_allocator,
585
- mapping)
586
-
587
- if seq_group.is_encoder_decoder():
588
- self.cross_block_tables[request_id] = \
589
- self._swap_block_table(self.cross_block_tables[request_id],
590
- self.cpu_allocator,
591
- self.gpu_allocator,
592
- mapping)
593
-
594
- return [(cpu_block.block_number, gpu_block.block_number)
595
- for cpu_block, gpu_block in mapping.items()]
596
-
597
- def can_swap_out(self, seq_group: SequenceGroup) -> bool:
598
- blocks = self._get_physical_blocks(seq_group)
599
- return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
600
-
601
- def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
602
- request_id = seq_group.request_id
603
-
604
- # GPU block -> CPU block.
605
- # dict is efficient in lookup `if gpu_block in mapping`
606
- mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
607
- for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
608
- self.block_tables[seq.seq_id] = \
609
- self._swap_block_table(self.block_tables[seq.seq_id],
610
- self.gpu_allocator, self.cpu_allocator,
611
- mapping)
612
-
613
- if seq_group.is_encoder_decoder():
614
- self.cross_block_tables[request_id] = \
615
- self._swap_block_table(self.cross_block_tables[request_id],
616
- self.gpu_allocator,
617
- self.cpu_allocator,
618
- mapping)
619
-
620
- return [(cpu_block.block_number, gpu_block.block_number)
621
- for cpu_block, gpu_block in mapping.items()]
622
-
623
- def _free_block_table(self, block_table: BlockTable) -> None:
624
- # when using a sliding window, each seq will only use up
625
- # to `self.block_sliding_window` blocks. When freeing
626
- # the block table, we must make sure to not free blocks more
627
- # than once. If no sliding window is used, there is no block
628
- # reuse in the block table, so we must free all blocks.
629
- blocks_to_free = (block_table[-self.block_sliding_window:]
630
- if self.block_sliding_window is not None else
631
- block_table)
632
- for block in set(blocks_to_free):
633
- if block.device == Device.GPU:
634
- self.gpu_allocator.free(block)
635
- else:
636
- self.cpu_allocator.free(block)
637
-
638
- def free(self, seq: Sequence) -> None:
639
- if seq.seq_id not in self.block_tables:
640
- # Already freed or haven't been scheduled yet.
641
- return
642
- block_table = self.block_tables[seq.seq_id]
643
- self._free_block_table(block_table)
644
- del self.block_tables[seq.seq_id]
645
-
646
- def free_cross(self, seq_group: SequenceGroup) -> None:
647
- if seq_group.request_id not in self.cross_block_tables:
648
- # Already freed or hasn't ben scheduled yet.
649
- return
650
- block_table = self.cross_block_tables[seq_group.request_id]
651
- self._free_block_table(block_table)
652
- del self.cross_block_tables[seq_group.request_id]
653
-
654
- def reset(self) -> None:
655
- # Free decoder block tables
656
- for block_table in self.block_tables.values():
657
- self._free_block_table(block_table)
658
- self.block_tables.clear()
659
- # Free cross-attention block tables
660
- for block_table in self.cross_block_tables.values():
661
- self._free_block_table(block_table)
662
- self.cross_block_tables.clear()
663
-
664
- def get_block_table(self, seq: Sequence) -> List[int]:
665
- return self.block_tables[seq.seq_id].ids()
666
-
667
- def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
668
- block_table = self.cross_block_tables[seq_group.request_id]
669
- return [block.block_number for block in block_table]
670
-
671
- def get_num_free_gpu_blocks(self) -> int:
672
- return self.gpu_allocator.get_num_free_blocks()
673
-
674
- def get_num_free_cpu_blocks(self) -> int:
675
- return self.cpu_allocator.get_num_free_blocks()
676
-
677
- def access_all_blocks_in_seq(
678
- self,
679
- seq: Sequence,
680
- access_time: float,
681
- ) -> None:
682
- if self.enable_caching:
683
- # Update the last accessed time of all the blocks accessed
684
- # in this step.
685
- block_table = self.block_tables[seq.seq_id]
686
- for block in block_table:
687
- block.last_accessed = access_time
688
-
689
- def compute_full_blocks_in_seq(self, seq: Sequence, token_chunk_size: int):
690
- if seq.seq_id not in self.block_tables:
691
- return
692
-
693
- # When chunked prefill is enabled, the computed full blocks
694
- # should be calculated based on the number of computed tokens.
695
- max_computed_tokens = (seq.data.get_num_computed_tokens() +
696
- token_chunk_size)
697
- computed_full_blocks = max_computed_tokens // self.block_size
698
-
699
- block_table = self.block_tables[seq.seq_id]
700
- if computed_full_blocks == 0:
701
- return
702
- for i in reversed(range(computed_full_blocks)):
703
- if block_table[i].computed:
704
- break
705
- block_table[i].computed = True
706
-
707
- def get_all_computed_blocks(self, seq: Sequence) -> List[int]:
708
- if seq.seq_id not in self.block_tables:
709
- return []
710
- block_table = self.block_tables[seq.seq_id]
711
- # NOTE We exclude the last block to avoid the case where the entire
712
- # prompt is cached. This would cause erroneous behavior in model
713
- # runner.
714
- return [
715
- b.block_number
716
- for b in takewhile(lambda b: b.computed, block_table[:-1])
717
- ]
718
-
719
- def get_common_computed_block_ids(
720
- self, seqs: List[Sequence]) -> GenericSequence[int]:
721
- """Return the block ids that are common for a given sequence group.
722
-
723
- Used in prefill (can skip prefill of some blocks).
724
- """
725
- # Can return non-empty result only with prefix caching enabled.
726
- if not self.enable_caching:
727
- return []
728
-
729
- ids_list = [self.get_all_computed_blocks(seq) for seq in seqs]
730
- return commonprefix([ids for ids in ids_list if ids != []])
731
-
732
- def mark_blocks_as_computed(self, seq_group: SequenceGroup,
733
- token_chunk_size: int):
734
- if self.enable_caching:
735
- for seq in seq_group.get_seqs():
736
- self.compute_full_blocks_in_seq(seq, token_chunk_size)
737
-
738
- def get_prefix_cache_hit_rate(self, device: Device) -> float:
739
- if device == Device.GPU:
740
- return self.gpu_allocator.get_prefix_cache_hit_rate()
741
- if device == Device.CPU:
742
- return self.cpu_allocator.get_prefix_cache_hit_rate()
743
- raise ValueError(f"Invalid device: {device}")
vllm/core/interfaces.py
+3
-7
@@ -28,13 +28,9 @@ class BlockSpaceManager(ABC):
28
def get_block_space_manager_class(version: str):
29
version = version.lower()
30
31
- if version == "v1":
32
- from vllm.core.block_manager_v1 import BlockSpaceManagerV1
33
- return BlockSpaceManagerV1
34
-
35
- if version == "v2":
36
- from vllm.core.block_manager_v2 import BlockSpaceManagerV2
37
- return BlockSpaceManagerV2
31
+ if version == "selfattn":
32
+ from vllm.core.block_manager import SelfAttnBlockSpaceManager
33
+ return SelfAttnBlockSpaceManager
34
35
if version == "placeholder":
36
from vllm.core.placeholder_block_space_manager import (
vllm/core/scheduler.py
+1
-3
@@ -312,9 +312,7 @@ class Scheduler:
312
# LoRAs. This should be improved in the future.
313
self.lora_config = lora_config
314
315
- version = "v1"
316
- if self.scheduler_config.use_v2_block_manager:
317
- version = "v2"
315
+ version = "selfattn"
316
if (self.scheduler_config.embedding_mode
317
or self.cache_config.is_attention_free):
318
version = "placeholder"
vllm/engine/arg_utils.py
+17
-21
@@ -373,12 +373,13 @@ class EngineArgs:
373
action='store_true',
374
help='Disables sliding window, '
375
'capping to sliding window size')
376
- parser.add_argument(
377
- '--use-v2-block-manager',
378
- default=EngineArgs.use_v2_block_manager,
379
- action='store_true',
380
- help='Use BlockSpaceMangerV2. By default this is set to True. '
381
- 'Set to False to use BlockSpaceManagerV1')
376
+ parser.add_argument('--use-v2-block-manager',
377
+ action='store_true',
378
+ help='[DEPRECATED] block manager v1 has been '
379
+ 'removed and SelfAttnBlockSpaceManager (i.e. '
380
+ 'block manager v2) is now the default. '
381
+ 'Setting this flag to True or False'
382
+ ' has no effect on vLLM behavior.')
383
parser.add_argument(
384
'--num-lookahead-slots',
385
type=int,
@@ -969,12 +970,6 @@ class EngineArgs:
970
"in low performance due to small KV cache space. Consider "
971
"setting --max-model-len to a smaller value.", max_model_len)
972
972
- if self.num_scheduler_steps > 1 and not self.use_v2_block_manager:
973
- self.use_v2_block_manager = True
974
- logger.warning(
975
- "Enabled BlockSpaceManagerV2 because it is "
976
- "required for multi-step (--num-scheduler-steps > 1)")
977
-
973
speculative_config = SpeculativeConfig.maybe_create_spec_config(
974
target_model_config=model_config,
975
target_parallel_config=parallel_config,
@@ -990,7 +985,6 @@ class EngineArgs:
985
speculative_disable_by_batch_size,
986
speculative_max_model_len=self.speculative_max_model_len,
987
enable_chunked_prefill=self.enable_chunked_prefill,
993
- use_v2_block_manager=self.use_v2_block_manager,
988
disable_log_stats=self.disable_log_stats,
989
ngram_prompt_lookup_max=self.ngram_prompt_lookup_max,
990
ngram_prompt_lookup_min=self.ngram_prompt_lookup_min,
@@ -1021,11 +1015,20 @@ class EngineArgs:
1015
if speculative_config is None \
1016
else speculative_config.num_lookahead_slots
1017
1018
+ if not self.use_v2_block_manager:
1019
+ logger.warning(
1020
+ "[DEPRECATED] Block manager v1 has been removed, "
1021
+ "and setting --use-v2-block-manager to True or False has "
1022
+ "no effect on vLLM behavior. Please remove "
1023
+ "--use-v2-block-manager in your engine argument. "
1024
+ "If your use case is not supported by "
1025
+ "SelfAttnBlockSpaceManager (i.e. block manager v2),"
1026
+ " please file an issue with detailed information.")
1027
+
1028
scheduler_config = SchedulerConfig(
1029
max_num_batched_tokens=self.max_num_batched_tokens,
1030
max_num_seqs=self.max_num_seqs,
1031
max_model_len=model_config.max_model_len,
1028
- use_v2_block_manager=self.use_v2_block_manager,
1032
num_lookahead_slots=num_lookahead_slots,
1033
delay_factor=self.scheduler_delay_factor,
1034
enable_chunked_prefill=self.enable_chunked_prefill,
@@ -1081,13 +1084,6 @@ class EngineArgs:
1084
or "all" in detailed_trace_modules,
1085
)
1086
1084
- if (model_config.get_sliding_window() is not None
1085
- and scheduler_config.chunked_prefill_enabled
1086
- and not scheduler_config.use_v2_block_manager):
1087
- raise ValueError(
1088
- "Chunked prefill is not supported with sliding window. "
1089
- "Set --disable-sliding-window to disable sliding window.")
1090
-
1087
return EngineConfig(
1088
model_config=model_config,
1089
cache_config=cache_config,
vllm/engine/llm_engine.py
+1
-2
@@ -247,7 +247,7 @@ class LLMEngine:
247
"enforce_eager=%s, kv_cache_dtype=%s, "
248
"quantization_param_path=%s, device_config=%s, "
249
"decoding_config=%r, observability_config=%r, "
250
- "seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
250
+ "seed=%d, served_model_name=%s, "
251
"num_scheduler_steps=%d, chunked_prefill_enabled=%s "
252
"multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
253
"use_async_output_proc=%s, use_cached_outputs=%s, "
@@ -280,7 +280,6 @@ class LLMEngine:
280
observability_config,
281
model_config.seed,
282
model_config.served_model_name,
283
- scheduler_config.use_v2_block_manager,
283
scheduler_config.num_scheduler_steps,
284
scheduler_config.chunked_prefill_enabled,
285
scheduler_config.multi_step_stream_outputs,
vllm/envs.py
-6
@@ -64,7 +64,6 @@ if TYPE_CHECKING:
64
VLLM_USE_TRITON_AWQ: bool = False
65
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66
VLLM_SKIP_P2P_CHECK: bool = False
67
- VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1: bool = False
67
VLLM_TORCH_COMPILE_LEVEL: int = 0
68
VLLM_DISABLED_KERNELS: List[str] = []
69
@@ -427,11 +426,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
426
"VLLM_SKIP_P2P_CHECK":
427
lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "0") == "1",
428
430
- # If set, allowing the use of deprecated block manager V1
431
- "VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1":
432
- lambda: os.environ.get("VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1", "0"
433
- ) == "1",
434
-
429
# List of quantization kernels that should be disabled, used for testing
430
# and performance comparisons. Currently only affects MPLinearKernel
431
# selection
vllm/worker/model_runner.py
+6
-11
@@ -574,17 +574,12 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
574
# paged attn. We can remove it if we make paged attn kernel
575
# to properly handle slinding window attn.
576
curr_sliding_window_block = self.sliding_window_blocks
577
- if self.scheduler_config.use_v2_block_manager:
578
- # number of elements in last block
579
- suff_len = inter_data.seq_lens[seq_idx] % self.block_size
580
- sliding_seq_len = min(
581
- inter_data.seq_lens[seq_idx],
582
- self.block_aligned_sliding_window + suff_len)
583
- if suff_len > 0:
584
- curr_sliding_window_block += 1
585
- else:
586
- sliding_seq_len = min(inter_data.seq_lens[seq_idx],
587
- self.sliding_window)
577
+ # number of elements in last block
578
+ suff_len = inter_data.seq_lens[seq_idx] % self.block_size
579
+ sliding_seq_len = min(inter_data.seq_lens[seq_idx],
580
+ self.block_aligned_sliding_window + suff_len)
581
+ if suff_len > 0:
582
+ curr_sliding_window_block += 1
583
584
inter_data.curr_sliding_window_blocks[
585
seq_idx] = curr_sliding_window_block