@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 81ede99ca4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Core] Deprecating block manager v1 and make block manager v2 default (#8704)

Removing the block manager v1. This is the initial piece of prefix-caching-centric design. In order to achieve prefix-caching-centric design, we need to simplify the code path so that we only use v2 block manager (which has much higher performance on prefix caching).

Kuntai Du committed Oct 17, 2024 at 11:38 UTC 81ede99ca44a5b3518932a07ea4a76a719e7416e
45 files changed +206 -2109
.buildkite/test-pipeline.yaml
+6 -12
@@ -77,8 +77,8 @@ steps:
77 - vllm/
78 - tests/basic_correctness/test_chunked_prefill
79 commands:
80 - - VLLM_ATTENTION_BACKEND=XFORMERS VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s basic_correctness/test_chunked_prefill.py
81 - - VLLM_ATTENTION_BACKEND=FLASH_ATTN VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s basic_correctness/test_chunked_prefill.py
80 + - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
81 + - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
82
83 - label: Core Test # 10min
84 mirror_hardwares: [amd]
@@ -88,11 +88,7 @@ steps:
88 - vllm/distributed
89 - tests/core
90 commands:
91 - - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core/test_scheduler.py
92 - - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/test_chunked_prefill_scheduler.py
93 - - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/block/e2e/test_correctness.py
94 - - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/block/e2e/test_correctness_sliding_window.py
95 - - pytest -v -s core --ignore=core/block/e2e/test_correctness.py --ignore=core/test_scheduler.py --ignore=core/test_chunked_prefill_scheduler.py --ignore=core/block/e2e/test_correctness.py --ignore=core/block/e2e/test_correctness_sliding_window.py
91 + - pytest -v -s core
92
93 - label: Entrypoints Test # 40min
94 working_dir: "/vllm-workspace/tests"
@@ -192,8 +188,7 @@ steps:
188 - vllm/
189 - tests/prefix_caching
190 commands:
195 - - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s prefix_caching/test_prefix_caching.py
196 - - pytest -v -s prefix_caching --ignore=prefix_caching/test_prefix_caching.py
191 + - pytest -v -s prefix_caching
192
193 - label: Samplers Test # 36min
194 source_file_dependencies:
@@ -217,8 +212,7 @@ steps:
212 - tests/spec_decode
213 commands:
214 - pytest -v -s spec_decode/e2e/test_multistep_correctness.py
220 - - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s spec_decode/e2e/test_compatibility.py
221 - - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py --ignore=spec_decode/e2e/test_compatibility.py
215 + - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
216
217 - label: LoRA Test %N # 15min each
218 mirror_hardwares: [amd]
@@ -405,7 +399,7 @@ steps:
399 - pytest -v -s ./compile/test_basic_correctness.py
400 - pytest -v -s ./compile/test_wrapper.py
401 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
408 - - TARGET_TEST_SUITE=L4 VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest basic_correctness/ -v -s -m distributed_2_gpus
402 + - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
403 # Avoid importing model tests that cause CUDA reinitialization error
404 - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
405 - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
benchmarks/benchmark_latency.py
-4
@@ -38,7 +38,6 @@ def main(args: argparse.Namespace):
38 quantization_param_path=args.quantization_param_path,
39 device=args.device,
40 ray_workers_use_nsight=args.ray_workers_use_nsight,
41 - use_v2_block_manager=args.use_v2_block_manager,
41 enable_chunked_prefill=args.enable_chunked_prefill,
42 download_dir=args.download_dir,
43 block_size=args.block_size,
@@ -221,9 +220,6 @@ if __name__ == '__main__':
220 parser.add_argument("--enable-prefix-caching",
221 action='store_true',
222 help="Enable automatic prefix caching")
224 - parser.add_argument('--use-v2-block-manager',
225 - action='store_true',
226 - default=EngineArgs.use_v2_block_manager)
223 parser.add_argument(
224 "--ray-workers-use-nsight",
225 action='store_true',
benchmarks/benchmark_prefix_caching.py
-6
@@ -33,7 +33,6 @@ from typing import List, Optional, Tuple
33 from transformers import PreTrainedTokenizerBase
34
35 from vllm import LLM, SamplingParams
36 -from vllm.engine.arg_utils import EngineArgs
36 from vllm.utils import FlexibleArgumentParser
37
38 try:
@@ -134,7 +133,6 @@ def main(args):
133 tokenizer_mode='auto',
134 trust_remote_code=True,
135 enforce_eager=True,
137 - use_v2_block_manager=args.use_v2_block_manager,
136 tensor_parallel_size=args.tensor_parallel_size,
137 enable_prefix_caching=args.enable_prefix_caching)
138
@@ -176,10 +174,6 @@ if __name__ == "__main__":
174 parser.add_argument('--enable-prefix-caching',
175 action='store_true',
176 help='enable prefix caching')
179 - parser.add_argument('--use-v2-block-manager',
180 - action='store_true',
181 - default=EngineArgs.use_v2_block_manager,
182 - help='Use BlockSpaceMangerV2')
177 parser.add_argument('--num-prompts',
178 type=int,
179 default=1,
benchmarks/benchmark_throughput.py
+1 -10
@@ -86,7 +86,6 @@ def run_vllm(
86 distributed_executor_backend: Optional[str],
87 gpu_memory_utilization: float = 0.9,
88 num_scheduler_steps: int = 1,
89 - use_v2_block_manager: bool = False,
89 download_dir: Optional[str] = None,
90 load_format: str = EngineArgs.load_format,
91 disable_async_output_proc: bool = False,
@@ -113,7 +112,6 @@ def run_vllm(
112 distributed_executor_backend=distributed_executor_backend,
113 load_format=load_format,
114 num_scheduler_steps=num_scheduler_steps,
116 - use_v2_block_manager=use_v2_block_manager,
115 disable_async_output_proc=disable_async_output_proc,
116 )
117
@@ -176,7 +174,6 @@ async def run_vllm_async(
174 distributed_executor_backend: Optional[str],
175 gpu_memory_utilization: float = 0.9,
176 num_scheduler_steps: int = 1,
179 - use_v2_block_manager: bool = False,
177 download_dir: Optional[str] = None,
178 load_format: str = EngineArgs.load_format,
179 disable_async_output_proc: bool = False,
@@ -204,7 +201,6 @@ async def run_vllm_async(
201 distributed_executor_backend=distributed_executor_backend,
202 load_format=load_format,
203 num_scheduler_steps=num_scheduler_steps,
207 - use_v2_block_manager=use_v2_block_manager,
204 disable_async_output_proc=disable_async_output_proc,
205 worker_use_ray=False,
206 disable_log_requests=True,
@@ -341,8 +337,7 @@ def main(args: argparse.Namespace):
337 args.enable_prefix_caching, args.enable_chunked_prefill,
338 args.max_num_batched_tokens, args.distributed_executor_backend,
339 args.gpu_memory_utilization, args.num_scheduler_steps,
344 - args.use_v2_block_manager, args.download_dir, args.load_format,
345 - args.disable_async_output_proc
340 + args.download_dir, args.load_format, args.disable_async_output_proc
341 ]
342
343 if args.async_engine:
@@ -471,10 +466,6 @@ if __name__ == "__main__":
466 type=int,
467 default=1,
468 help="Maximum number of forward steps per scheduler call.")
474 - parser.add_argument("--use-v2-block-manager",
475 - action='store_true',
476 - default=EngineArgs.use_v2_block_manager,
477 - help="Enable block manager v2.")
469 parser.add_argument(
470 "--enable-prefix-caching",
471 action='store_true',
benchmarks/overheads/benchmark_hashing.py
-4
@@ -16,7 +16,6 @@ def main(args):
16 enforce_eager=True,
17 enable_prefix_caching=True,
18 tensor_parallel_size=args.tensor_parallel_size,
19 - use_v2_block_manager=args.use_v2_block_manager,
19 )
20
21 sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
@@ -56,8 +55,5 @@ if __name__ == "__main__":
55 parser.add_argument('--enable-prefix-caching',
56 action='store_true',
57 help='enable prefix caching')
59 - parser.add_argument('--use-v2-block-manager',
60 - action='store_true',
61 - help='Use BlockSpaceMangerV2')
58 args = parser.parse_args()
59 main(args)
docs/source/models/spec_decode.rst
-3
@@ -30,7 +30,6 @@ The following code configures vLLM in an offline mode to use speculative decodin
30 tensor_parallel_size=1,
31 speculative_model="facebook/opt-125m",
32 num_speculative_tokens=5,
33 - use_v2_block_manager=True,
33 )
34 outputs = llm.generate(prompts, sampling_params)
35
@@ -104,7 +103,6 @@ matching n-grams in the prompt. For more information read `this thread. <https:/
103 speculative_model="[ngram]",
104 num_speculative_tokens=5,
105 ngram_prompt_lookup_max=4,
107 - use_v2_block_manager=True,
106 )
107 outputs = llm.generate(prompts, sampling_params)
108
@@ -135,7 +133,6 @@ For more information see `this blog <https://pytorch.org/blog/hitchhikers-guide-
133 tensor_parallel_size=4,
134 speculative_model="ibm-fms/llama3-70b-accelerator",
135 speculative_draft_tensor_parallel_size=1,
138 - use_v2_block_manager=True,
136 )
137 outputs = llm.generate(prompts, sampling_params)
138
examples/offline_inference_mlpspeculator.py
-2
@@ -50,8 +50,6 @@ if __name__ == "__main__":
50 llm = LLM(
51 model="meta-llama/Llama-2-13b-chat-hf",
52 speculative_model="ibm-fms/llama-13b-accelerator",
53 - # These are currently required for MLPSpeculator decoding
54 - use_v2_block_manager=True,
53 )
54
55 print("With speculation")
tests/basic_correctness/test_chunked_prefill.py
+1 -10
@@ -12,7 +12,7 @@ from contextlib import nullcontext
12 import pytest
13
14 from ..models.utils import check_logprobs_close, check_outputs_equal
15 -from ..utils import check_deprecated_block_manager_usage, multi_gpu_test
15 +from ..utils import multi_gpu_test
16
17 MODELS = [
18 "facebook/opt-125m",
@@ -20,12 +20,6 @@ MODELS = [
20 ]
21
22
23 -@pytest.fixture(scope="module", autouse=True)
24 -def check_deprecated_block_manager():
25 - check_deprecated_block_manager_usage(
26 - 'tests/basic_correctness/test_chunked_prefill.py')
27 -
28 -
23 @pytest.mark.parametrize("model", MODELS)
24 @pytest.mark.parametrize("dtype", ["half"])
25 @pytest.mark.parametrize("max_tokens", [32])
@@ -197,7 +191,6 @@ def test_models_with_fp8_kv_cache(
191 @pytest.mark.parametrize("max_tokens", [16])
192 @pytest.mark.parametrize("enforce_eager", [False])
193 @pytest.mark.parametrize("chunk_size", [30, 32])
200 -@pytest.mark.parametrize("use_v2_block_manager", [False, True])
194 # NOTE: Increasing this in this suite will fail CI because we currently cannot
195 # reset distributed env properly. Use a value > 1 just when you test.
196 @pytest.mark.parametrize("tensor_parallel_size", [1])
@@ -206,7 +199,6 @@ def test_with_prefix_caching(
199 max_tokens: int,
200 enforce_eager: bool,
201 chunk_size: int,
209 - use_v2_block_manager: bool,
202 tensor_parallel_size: int,
203 ) -> None:
204 """
@@ -234,7 +226,6 @@ def test_with_prefix_caching(
226 enable_chunked_prefill=True,
227 enable_prefix_caching=enable,
228 tensor_parallel_size=tensor_parallel_size,
237 - use_v2_block_manager=use_v2_block_manager,
229 enforce_eager=enforce_eager,
230 max_num_seqs=max_num_seqs,
231 ) as vllm_model:
tests/core/block/e2e/test_correctness.py
+28 -50
@@ -2,18 +2,11 @@ from itertools import cycle
2
3 import pytest
4
5 -from tests.utils import check_deprecated_block_manager_usage
5 from vllm import SamplingParams
6
7 from .conftest import get_token_ids_from_llm_generator
8
9
11 -@pytest.fixture(scope="module", autouse=True)
12 -def check_deprecated_block_manager():
13 - check_deprecated_block_manager_usage(
14 - 'tests/core/block/e2e/test_correctness.py')
15 -
16 -
10 @pytest.mark.parametrize(
11 "common_llm_kwargs",
12 [{
@@ -28,32 +21,32 @@ def check_deprecated_block_manager():
21 "num_gpu_blocks_override": 5 * (64 + 1),
22 }])
23 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
31 -@pytest.mark.parametrize("baseline_llm_kwargs", [{
32 - "use_v2_block_manager": False
33 -}])
24 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
25 @pytest.mark.parametrize("test_llm_kwargs", [{
35 - "use_v2_block_manager": True,
26 "preemption_mode": "swap"
27 }, {
38 - "use_v2_block_manager": True,
28 "preemption_mode": "recompute"
29 }])
30 @pytest.mark.parametrize("batch_size", [10])
31 @pytest.mark.parametrize("seed", [1])
43 -def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
44 - test_llm_generator, batch_size):
45 - """Verify block manager v2 produces same outputs as block manager v1, even
46 - when there is preemption.
32 +def test_block_manager_with_preemption(baseline_llm_generator,
33 + test_llm_generator, batch_size):
34 + """Verify block manager produces same outputs even when there is preemption.
35
36 This constructs two LLM, each with limited number of GPU blocks. The limit
37 is decided such that as the sequences in the batch grow, sequences must be
38 preempted and removed from cache.
39
40 If the output token ids are equivalent, then we have confidence that the KV
53 - cache is not corrupted in the v2 block manager.
41 + cache is not corrupted.
42
43 NOTE: We want a significant number of generated tokens so that any incorrect
44 KV mapping has time to build up error.
45 +
46 + NOTE(Kuntai): Though we have removed block manager v1, this test is still
47 + useful as it asserts the behavior of block manager v2 (now it is called
48 + SelfAttnBlockSpaceManager) is the same when swapping / preemption, so we
49 + keep this test.
50 """
51 output_len = 1024
52 temperature = 0.0
@@ -77,11 +70,9 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
70 temperature=temperature,
71 )
72
80 - print('Getting token ids from block manager v1')
73 baseline_token_ids = get_token_ids_from_llm_generator(
74 baseline_llm_generator, prompts, sampling_params)
75
84 - print('Getting token ids from block manager v2')
76 test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
77 prompts, sampling_params)
78
@@ -104,9 +95,6 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
95
96 # skip cuda graph creation for fast test.
97 "enforce_eager": True,
107 -
108 - # Lookahead scheduling only supported in v2 block manager.
109 - "use_v2_block_manager": True,
98 }])
99 @pytest.mark.parametrize(
100 "per_test_common_llm_kwargs",
@@ -218,26 +206,22 @@ def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
206 "max_num_seqs": 10,
207 }])
208 @pytest.mark.parametrize("baseline_llm_kwargs", [
221 - {
222 - "use_v2_block_manager": False,
223 - },
209 + {},
210 ])
211 @pytest.mark.parametrize("test_llm_kwargs", [
212 {
227 - "use_v2_block_manager": True,
213 "num_lookahead_slots": 0,
214 },
215 {
231 - "use_v2_block_manager": True,
216 "num_lookahead_slots": 5,
217 },
218 ])
219 @pytest.mark.parametrize("batch_size", [4])
220 @pytest.mark.parametrize("seed", [1])
237 -def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
238 - test_llm_generator, batch_size):
239 - """Verify that chunked prefill works with BlockManagerV2, with and without
240 - lookahead scheduling.
221 +def test_chunked_prefill_block_manager(baseline_llm_generator,
222 + test_llm_generator, batch_size):
223 + """Verify that chunked prefill works with SelfAttnBlockSpaceManager,
224 + with and without lookahead scheduling.
225 """
226 output_len = 32
227 temperature = 0.0
@@ -258,11 +242,11 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
242 temperature=temperature,
243 )
244
261 - print('Getting token ids with BlockManagerV1')
245 + print('Getting token ids with BlockManager')
246 baseline_token_ids = get_token_ids_from_llm_generator(
247 baseline_llm_generator, prompts, sampling_params)
248
265 - print('Getting token ids with BlockManagerV2')
249 + print('Getting token ids with BlockManager, with lookahead slots.')
250 test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
251 prompts, sampling_params)
252
@@ -290,32 +274,32 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
274 "enable_prefix_caching": True,
275 }])
276 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
293 -@pytest.mark.parametrize("baseline_llm_kwargs", [{
294 - "use_v2_block_manager": False
295 -}])
277 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
278 @pytest.mark.parametrize("test_llm_kwargs", [{
297 - "use_v2_block_manager": True,
279 "preemption_mode": "swap"
280 }, {
300 - "use_v2_block_manager": True,
281 "preemption_mode": "recompute"
282 }])
283 @pytest.mark.parametrize("batch_size", [10])
284 @pytest.mark.parametrize("seed", [1])
305 -def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
285 +def test_block_manager_prefix_caching_enabled_with_preemption(
286 baseline_llm_generator, test_llm_generator, batch_size):
307 - """Verify block manager v2 produces same outputs as block manager v1, even
308 - when there is preemption.
287 + """Verify block manager produces same outputs even when there is preemption.
288
289 This constructs two LLM, each with limited number of GPU blocks. The limit
290 is decided such that as the sequences in the batch grow, sequences must be
291 preempted and removed from cache.
292
293 If the output token ids are equivalent, then we have confidence that the KV
315 - cache is not corrupted in the v2 block manager.
294 + cache is not corrupted.
295
296 NOTE: We want a significant number of generated tokens so that any incorrect
297 KV mapping has time to build up error.
298 +
299 + NOTE(Kuntai): Though we have removed block manager v1, this test is still
300 + useful as it asserts the behavior of block manager v2 (now it is called
301 + SelfAttnBlockSpaceManager) is the same when swapping / preemption, so we
302 + keep this test.
303 """
304 output_len = 1024
305 temperature = 0.0
@@ -339,11 +323,11 @@ def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
323 temperature=temperature,
324 )
325
342 - print('Getting token ids from block manager v1')
326 + print('Getting token ids from block manager')
327 baseline_token_ids = get_token_ids_from_llm_generator(
328 baseline_llm_generator, prompts, sampling_params)
329
346 - print('Getting token ids from block manager v2')
330 + print('Getting token ids from block manager, with preemption')
331 test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
332 prompts, sampling_params)
333
@@ -366,9 +350,6 @@ def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
350 # Allow only 5 sequences of ~1024 tokens in worst case.
351 "block_size": 16,
352 "num_gpu_blocks_override": 5 * (64 + 1),
369 -
370 - # Test APC in v2 block
371 - "use_v2_block_manager": True,
353 }])
354 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
355 @pytest.mark.parametrize("baseline_llm_kwargs", [{
@@ -444,9 +425,6 @@ def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
425 "max_model_len": 48,
426 "block_size": 16,
427 "num_gpu_blocks_override": 3,
447 -
448 - # Test APC in v2 block
449 - "use_v2_block_manager": True,
428 }])
429 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
430 @pytest.mark.parametrize("baseline_llm_kwargs", [{
tests/core/block/e2e/test_correctness_sliding_window.py
+3 -16
@@ -3,7 +3,6 @@ from typing import List
3
4 import pytest
5
6 -from tests.utils import check_deprecated_block_manager_usage
6 from vllm import LLM, SamplingParams
7
8 from .conftest import get_text_from_llm_generator
@@ -13,12 +12,6 @@ MODEL = "bigcode/starcoder2-3b"
12 BLOCK_SIZE = 16
13
14
16 -@pytest.fixture(scope="module", autouse=True)
17 -def check_deprecated_block_manager():
18 - check_deprecated_block_manager_usage(
19 - 'tests/core/block/e2e/test_correctness_sliding_window.py')
20 -
21 -
15 @pytest.mark.parametrize(
16 "common_llm_kwargs",
17 [{
@@ -31,10 +24,8 @@ def check_deprecated_block_manager():
24 "num_gpu_blocks_override": 100000 // BLOCK_SIZE,
25 }])
26 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
34 -@pytest.mark.parametrize("baseline_llm_kwargs", [{
35 - "use_v2_block_manager": False
36 -}])
37 -@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
27 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
28 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
29 @pytest.mark.parametrize("batch_size", [5])
30 @pytest.mark.parametrize("seed", [1])
31 def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
@@ -55,7 +46,6 @@ def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
46
47 prompts, answer, indices = prep_prompts(batch_size)
48
58 - print('Getting token ids from block manager v1')
49 baseline_texts = get_text_from_llm_generator(baseline_llm_generator,
50 prompts,
51 sampling_params,
@@ -91,10 +81,7 @@ def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
81 "num_gpu_blocks_override": 100000 // BLOCK_SIZE,
82 }])
83 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
94 -@pytest.mark.parametrize("test_llm_kwargs", [{
95 - "use_v2_block_manager": True,
96 - "enable_chunked_prefill": True
97 -}])
84 +@pytest.mark.parametrize("test_llm_kwargs", [{"enable_chunked_prefill": True}])
85 @pytest.mark.parametrize("batch_size", [5])
86 @pytest.mark.parametrize("seed", [1])
87 def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed):
tests/core/block/test_block_manager.py renamed
+29 -28
@@ -2,7 +2,7 @@ import pytest
2
3 from vllm.core.block.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
4 STR_NOT_IMPL_ENC_DEC_SWA)
5 -from vllm.core.block_manager_v2 import BlockSpaceManagerV2
5 +from vllm.core.block_manager import SelfAttnBlockSpaceManager
6 from vllm.core.interfaces import AllocStatus
7 from vllm.sequence import Logprob, SequenceStatus
8 from vllm.utils import chunk_list
@@ -17,7 +17,7 @@ from ..utils import (create_dummy_prompt, create_seq_group,
17 @pytest.mark.parametrize("watermark", [0.0, 0.5])
18 def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
19 num_gpu_blocks: int, watermark: float):
20 - block_manager = BlockSpaceManagerV2(
20 + block_manager = SelfAttnBlockSpaceManager(
21 block_size=block_size,
22 num_gpu_blocks=num_gpu_blocks,
23 num_cpu_blocks=1024,
@@ -63,7 +63,7 @@ def test_can_allocate_seq_group_encoder_decoder(block_size: int,
63 num_seqs_per_group: int,
64 num_gpu_blocks: int,
65 watermark: float):
66 - block_manager = BlockSpaceManagerV2(
66 + block_manager = SelfAttnBlockSpaceManager(
67 block_size=block_size,
68 num_gpu_blocks=num_gpu_blocks,
69 num_cpu_blocks=1024,
@@ -117,16 +117,16 @@ def test_can_allocate_encoder_decoder_fails_with_swa(block_size: int,
117 '''
118 SWA short for Sliding Window Attention.
119
120 - At time of writing block manager v2 does not support SWA.
120 + At time of writing block manager does not support SWA.
121
122 - However even when SWA is implemented for block manager v2,
122 + However even when SWA is implemented for block manager,
123 there will still most likely be a separate workstream required
124 to enable SWA for encoder/decoder models.
125
126 Therefore this test enforces that one of the following cases
127 hold true:
128 - 1. Block manager v2 does not support SWA at all (true at time of writing)
129 - 2. Block manager v2 fails with NotImplementError when SWA is enabled
128 + 1. Block manager does not support SWA at all (true at time of writing)
129 + 2. Block manager fails with NotImplementError when SWA is enabled
130 AND a SequenceGroup with an encoder sequence (i.e. in support of an
131 encoder/decoder model) is passed into can_allocate() as an argument
132
@@ -135,7 +135,7 @@ def test_can_allocate_encoder_decoder_fails_with_swa(block_size: int,
135 '''
136
137 with pytest.raises((NotImplementedError, AssertionError)) as exc_info:
138 - block_manager = BlockSpaceManagerV2(
138 + block_manager = SelfAttnBlockSpaceManager(
139 block_size=block_size,
140 num_gpu_blocks=num_gpu_blocks,
141 num_cpu_blocks=1024,
@@ -158,7 +158,7 @@ def test_can_allocate_encoder_decoder_fails_with_swa(block_size: int,
158 block_manager.can_allocate(seq_group)
159
160 # Assert that either
161 - # 1. Block manager v2 constructor fails with assertion that sliding window
161 + # 1. Block manager constructor fails with assertion that sliding window
162 # is not yet supported (most likely near-term outcome at time of
163 # writing), or
164 # 2. can_allocate() fails with NotImplementedError due to combination of
@@ -177,7 +177,7 @@ def test_can_allocate_encoder_decoder_fails_with_prefix_cache(
177 block_size: int, num_seqs_per_group: int, num_gpu_blocks: int,
178 watermark: float):
179
180 - block_manager = BlockSpaceManagerV2(
180 + block_manager = SelfAttnBlockSpaceManager(
181 block_size=block_size,
182 num_gpu_blocks=num_gpu_blocks,
183 num_cpu_blocks=1024,
@@ -217,7 +217,7 @@ def test_append_slots(block_size, prompt_len, num_slots_to_append,
217
218 num_gpu_blocks = 1024
219 watermark = 0.1
220 - block_manager = BlockSpaceManagerV2(
220 + block_manager = SelfAttnBlockSpaceManager(
221 block_size=block_size,
222 num_gpu_blocks=num_gpu_blocks,
223 num_cpu_blocks=0,
@@ -269,14 +269,15 @@ def test_swap(block_size, num_cpu_blocks, num_gpu_blocks, num_lookahead_slots,
269 """Verify blocks number on src/desc device is correct after swapping in/out
270 sequence group (not missing or extra blocks).
271 """
272 - block_manager = BlockSpaceManagerV2(block_size,
273 - num_cpu_blocks,
274 - num_gpu_blocks,
275 - watermark=0,
276 - enable_caching=enable_caching)
272 + block_manager = SelfAttnBlockSpaceManager(block_size,
273 + num_cpu_blocks,
274 + num_gpu_blocks,
275 + watermark=0,
276 + enable_caching=enable_caching)
277 prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
278 prompt.status = SequenceStatus.WAITING
279 block_manager.allocate(seq_group)
280 +
281 # Emulate a forward pass by appending a single token.
282 # The block manager then knows how many unprocessed
283 # tokens will be written in the next forward pass.
@@ -321,11 +322,11 @@ def test_can_swap(block_size, num_gpu_blocks, num_lookahead_slots,
322 can be swapped in/out.
323 """
324 num_cpu_blocks = num_gpu_blocks
324 - block_manager = BlockSpaceManagerV2(block_size,
325 - num_cpu_blocks,
326 - num_gpu_blocks,
327 - watermark=0,
328 - enable_caching=enable_caching)
325 + block_manager = SelfAttnBlockSpaceManager(block_size,
326 + num_cpu_blocks,
327 + num_gpu_blocks,
328 + watermark=0,
329 + enable_caching=enable_caching)
330 prompt, seq_group = create_dummy_prompt(
331 "1", prompt_length=(num_gpu_blocks - 1) * block_size - 1)
332 prompt.status = SequenceStatus.WAITING
@@ -382,11 +383,11 @@ def test_swap_in_infeasible(num_lookahead_slots, enable_caching):
383 block_size = 8
384 num_cpu_blocks = 1
385 num_gpu_blocks = 1
385 - block_manager = BlockSpaceManagerV2(block_size,
386 - num_cpu_blocks,
387 - num_gpu_blocks,
388 - watermark=0,
389 - enable_caching=enable_caching)
386 + block_manager = SelfAttnBlockSpaceManager(block_size,
387 + num_cpu_blocks,
388 + num_gpu_blocks,
389 + watermark=0,
390 + enable_caching=enable_caching)
391 prompt_length = block_size - 3
392 assert prompt_length > 0
393 prompt, seq_group = create_dummy_prompt("1", prompt_length=prompt_length)
@@ -434,7 +435,7 @@ def test_sliding_window(block_size, prompt_len, num_slots_to_append,
435
436 num_gpu_blocks = 1024
437 watermark = 0.1
437 - block_manager = BlockSpaceManagerV2(
438 + block_manager = SelfAttnBlockSpaceManager(
439 block_size=block_size,
440 num_gpu_blocks=num_gpu_blocks,
441 num_cpu_blocks=0,
@@ -474,7 +475,7 @@ def test_sliding_window(block_size, prompt_len, num_slots_to_append,
475 seq.data.update_num_computed_tokens(prompt_len)
476 check_used(num_blocks(prompt_len))
477
477 - # this is how we compute it in BlockSpaceManagerV2.__init__
478 + # this is how we compute it in SelfAttnBlockSpaceManager.__init__
479 sliding_blocks = (sliding_window // block_size) + 2
480 # plus one block for null block
481 sliding_blocks += 1
tests/core/test_block_manager.py deleted
-637
@@ -1,637 +0,0 @@
1 -import time
2 -from collections import defaultdict
3 -from typing import List
4 -
5 -import pytest
6 -
7 -from vllm import SamplingParams
8 -from vllm.block import PhysicalTokenBlock
9 -from vllm.core.block.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
10 - STR_NOT_IMPL_ENC_DEC_SWA)
11 -from vllm.core.block_manager_v1 import (BlockSpaceManagerV1,
12 - UncachedBlockAllocator)
13 -from vllm.core.interfaces import AllocStatus
14 -from vllm.sequence import Logprob, Sequence, SequenceGroup, SequenceStatus
15 -from vllm.utils import Device
16 -
17 -from .utils import create_dummy_prompt, create_dummy_prompt_encoder_decoder
18 -
19 -
20 -def test_block_allocator_allocate():
21 - block_size = 4
22 - num_cpu_blocks = 4
23 - cpu_allocator = UncachedBlockAllocator(Device.CPU, block_size,
24 - num_cpu_blocks)
25 -
26 - # Allocate all available cpu blocks.
27 - num_free = num_cpu_blocks
28 - assert cpu_allocator.get_num_free_blocks() == num_free
29 - for _ in range(num_cpu_blocks):
30 - block = cpu_allocator.allocate()
31 - num_free -= 1
32 -
33 - assert block not in cpu_allocator.free_blocks
34 - assert cpu_allocator.get_num_free_blocks() == num_free
35 -
36 - with pytest.raises(ValueError):
37 - cpu_allocator.allocate()
38 -
39 -
40 -def test_block_allocator_free():
41 - block_size = 4
42 - num_cpu_blocks = 4
43 - cpu_allocator = UncachedBlockAllocator(Device.CPU, block_size,
44 - num_cpu_blocks)
45 -
46 - # Allocate all available cpu blocks.
47 - blocks: List[PhysicalTokenBlock] = []
48 - for _ in range(num_cpu_blocks):
49 - block = cpu_allocator.allocate()
50 - blocks.append(block)
51 - assert block not in cpu_allocator.free_blocks
52 -
53 - # Free all allocated cpu blocks.
54 - num_free = 0
55 - assert cpu_allocator.get_num_free_blocks() == num_free
56 - for block in blocks:
57 - cpu_allocator.free(block)
58 - num_free += 1
59 - assert block in cpu_allocator.free_blocks
60 - assert cpu_allocator.get_num_free_blocks() == num_free
61 -
62 - with pytest.raises(ValueError):
63 - cpu_allocator.free(block)
64 -
65 -
66 -def test_allocate():
67 - block_size = 4
68 - num_cpu_blocks = 4
69 - num_gpu_blocks = 4
70 - block_manager = BlockSpaceManagerV1(block_size,
71 - num_cpu_blocks,
72 - num_gpu_blocks,
73 - watermark=0)
74 -
75 - # Allocate same sequence group to all available gpu blocks.
76 - for i in range(num_gpu_blocks):
77 - _, seq_group = create_dummy_prompt(str(i), block_size)
78 - assert block_manager.can_allocate(seq_group) == AllocStatus.OK
79 - block_manager.allocate(seq_group)
80 - assert block_manager.can_allocate(seq_group) != AllocStatus.OK
81 -
82 - # Allocate same sequence group to all available gpu blocks.
83 - # Use watermark to reserve one gpu block.
84 - block_manager = BlockSpaceManagerV1(block_size,
85 - num_cpu_blocks,
86 - num_gpu_blocks,
87 - watermark=1 / num_gpu_blocks)
88 - for i in range(num_gpu_blocks - 1):
89 - _, seq_group = create_dummy_prompt(str(i), block_size)
90 - assert block_manager.can_allocate(seq_group) == AllocStatus.OK
91 - block_manager.allocate(seq_group)
92 - assert block_manager.can_allocate(seq_group) != AllocStatus.OK
93 -
94 -
95 -def test_allocate_encoder_decoder():
96 - block_size = 4
97 - num_cpu_blocks = 4
98 - num_gpu_blocks = 4
99 - block_req_per_seq_group = 2
100 - block_manager = BlockSpaceManagerV1(block_size,
101 - num_cpu_blocks,
102 - num_gpu_blocks,
103 - watermark=0)
104 -
105 - # Allocate same sequence group to all available gpu blocks.
106 - for i in range(num_gpu_blocks // block_req_per_seq_group):
107 - _, _, seq_group = create_dummy_prompt_encoder_decoder(
108 - str(i),
109 - decoder_prompt_length=block_size,
110 - encoder_prompt_length=block_size)
111 - assert block_manager.can_allocate(seq_group) == AllocStatus.OK
112 - block_manager.allocate(seq_group)
113 - assert block_manager.can_allocate(seq_group) != AllocStatus.OK
114 -
115 - # Allocate same sequence group to all available gpu blocks.
116 - # Use watermark to reserve one gpu block.
117 - block_manager = BlockSpaceManagerV1(block_size,
118 - num_cpu_blocks,
119 - num_gpu_blocks,
120 - watermark=1 / num_gpu_blocks)
121 - for i in range((num_gpu_blocks - 1) // block_req_per_seq_group):
122 - _, _, seq_group = create_dummy_prompt_encoder_decoder(
123 - str(i),
124 - decoder_prompt_length=block_size,
125 - encoder_prompt_length=block_size)
126 - assert block_manager.can_allocate(seq_group) == AllocStatus.OK
127 - block_manager.allocate(seq_group)
128 - assert block_manager.can_allocate(seq_group) != AllocStatus.OK
129 -
130 -
131 -def test_allocate_encoder_decoder_fails_with_swa():
132 - # SWA short for sliding window attention
133 -
134 - block_size = 4
135 - num_cpu_blocks = 4
136 - num_gpu_blocks = 4
137 - block_manager = BlockSpaceManagerV1(block_size,
138 - num_cpu_blocks,
139 - num_gpu_blocks,
140 - watermark=0,
141 - sliding_window=5) # swa
142 -
143 - # Allocate same sequence group to all available gpu blocks.
144 - _, _, seq_group = create_dummy_prompt_encoder_decoder(
145 - "0",
146 - decoder_prompt_length=block_size,
147 - encoder_prompt_length=block_size)
148 -
149 - # Assert that can_allocate() fails due to SWA
150 - with pytest.raises(NotImplementedError) as exc_info:
151 - block_manager.can_allocate(seq_group)
152 -
153 - assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
154 -
155 - # Assert that allocate() fails due to SWA
156 - with pytest.raises(NotImplementedError) as exc_info:
157 - block_manager.allocate(seq_group)
158 -
159 - assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
160 -
161 -
162 -def test_allocate_encoder_decoder_fails_with_prefix_caching():
163 - block_size = 4
164 - num_cpu_blocks = 4
165 - num_gpu_blocks = 4
166 - block_manager = BlockSpaceManagerV1(block_size,
167 - num_cpu_blocks,
168 - num_gpu_blocks,
169 - watermark=0,
170 - enable_caching=True) # Prefix cache
171 -
172 - # Allocate same sequence group to all available gpu blocks.
173 - _, _, seq_group = create_dummy_prompt_encoder_decoder(
174 - "0",
175 - decoder_prompt_length=block_size,
176 - encoder_prompt_length=block_size)
177 -
178 - # Assert that can_allocate() fails due to prefix caching
179 - with pytest.raises(NotImplementedError) as exc_info:
180 - block_manager.can_allocate(seq_group)
181 -
182 - assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
183 -
184 - # Assert that allocate() fails due to prefix caching
185 - with pytest.raises(NotImplementedError) as exc_info:
186 - block_manager.allocate(seq_group)
187 -
188 - assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
189 -
190 -
191 -def test_append_slot_single_seq():
192 - block_size = 4
193 - num_cpu_blocks = 4
194 - num_gpu_blocks = 4
195 - block_manager = BlockSpaceManagerV1(block_size,
196 - num_cpu_blocks,
197 - num_gpu_blocks,
198 - watermark=0)
199 -
200 - # Allocate single seq to gpu block.
201 - prompt, seq_group = create_dummy_prompt("1", block_size)
202 - block_manager.allocate(seq_group)
203 -
204 - # Nothing to append. Sequence has no new logical blocks.
205 - assert block_manager.can_append_slots(seq_group)
206 - before_blocks = block_manager.get_num_free_gpu_blocks()
207 - assert not block_manager.append_slots(prompt)
208 - after_blocks = block_manager.get_num_free_gpu_blocks()
209 - assert before_blocks == after_blocks
210 -
211 - # Add block_size number of new tokens and append slot.
212 - for i in range(block_size):
213 - token_id = i + 5
214 - prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
215 -
216 - assert block_manager.can_append_slots(seq_group)
217 - before_blocks = block_manager.get_num_free_gpu_blocks()
218 - assert not block_manager.append_slots(prompt)
219 - after_blocks = block_manager.get_num_free_gpu_blocks()
220 - assert before_blocks - after_blocks == 1
221 -
222 -
223 -def test_append_slot_cow():
224 - block_size = 4
225 - num_cpu_blocks = 4
226 - num_gpu_blocks = 4
227 - block_manager = BlockSpaceManagerV1(block_size=block_size,
228 - num_cpu_blocks=num_cpu_blocks,
229 - num_gpu_blocks=num_gpu_blocks,
230 - watermark=0)
231 -
232 - # Allocate prompt to gpu block. There is one slot left in the block.
233 - prompt = Sequence(seq_id=1,
234 - inputs={
235 - "prompt": "one two three",
236 - "prompt_token_ids": [1, 2, 3],
237 - },
238 - block_size=block_size)
239 -
240 - # Fork the sequence, such that a COW will be required when we append a new
241 - # token id.
242 - child = prompt.fork(new_seq_id=2)
243 -
244 - # Allocate space for the sequence group.
245 - seq_group = SequenceGroup(request_id="1",
246 - seqs=[prompt, child],
247 - arrival_time=time.time(),
248 - sampling_params=SamplingParams())
249 - block_manager.allocate(seq_group)
250 -
251 - # Fork and append a new token id. We expect a COW to be scheduled.
252 - token_id = 4
253 - child.append_token_id(token_id, {token_id: Logprob(0.0)})
254 - block_manager.fork(prompt, child)
255 -
256 - assert block_manager.can_append_slots(seq_group)
257 - before_blocks = block_manager.get_num_free_gpu_blocks()
258 -
259 - cows = block_manager.append_slots(child)
260 - assert cows
261 - dict_cows = defaultdict(list)
262 - for src_block, dst_block in cows:
263 - dict_cows[src_block].append(dst_block)
264 - for src_block, dst_blocks in dict_cows.items():
265 - assert src_block not in dst_blocks
266 -
267 - after_blocks = block_manager.get_num_free_gpu_blocks()
268 - assert before_blocks - after_blocks == 1
269 -
270 -
271 -def test_fork():
272 - block_size = 4
273 - num_cpu_blocks = 4
274 - num_gpu_blocks = 4
275 - block_manager = BlockSpaceManagerV1(block_size,
276 - num_cpu_blocks,
277 - num_gpu_blocks,
278 - watermark=0)
279 -
280 - prompt, seq_group = create_dummy_prompt("1",
281 - block_size - 1,
282 - block_size=block_size)
283 - block_manager.allocate(seq_group)
284 -
285 - # Fork prompt and copy block tables.
286 - child = prompt.fork(2)
287 - block_manager.fork(prompt, child)
288 - assert block_manager.get_block_table(
289 - prompt) == block_manager.get_block_table(child)
290 - token_id = 4
291 - # Append token to child. Block is shared so copy on write occurs.
292 - child.append_token_id(token_id, {token_id: Logprob(0.0)})
293 - block_manager.append_slots(child)
294 - assert block_manager.get_block_table(
295 - prompt) != block_manager.get_block_table(child)
296 -
297 -
298 -def test_swap():
299 - block_size = 4
300 - num_cpu_blocks = 4
301 - num_gpu_blocks = 4
302 - block_manager = BlockSpaceManagerV1(block_size,
303 - num_cpu_blocks,
304 - num_gpu_blocks,
305 - watermark=0)
306 -
307 - prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
308 - prompt.status = SequenceStatus.WAITING
309 - block_manager.allocate(seq_group)
310 -
311 - # Emulate a forward pass by appending a single token.
312 - # The block manager then knows how many unprocessed
313 - # tokens will be written in the next forward pass.
314 - token_id = 0
315 - prompt.status = SequenceStatus.RUNNING
316 - prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
317 -
318 - # Swap seq group from GPU -> CPU.
319 - gpu_blocks = block_manager.get_block_table(prompt)
320 - assert block_manager.can_swap_out(seq_group)
321 - before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
322 - before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
323 - mapping = block_manager.swap_out(seq_group)
324 - assert [x[0] for x in mapping] == gpu_blocks
325 - after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
326 - after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
327 - assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
328 - assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
329 - prompt.status = SequenceStatus.SWAPPED
330 -
331 - # Swap seq group from CPU -> GPU.
332 - cpu_blocks = block_manager.get_block_table(prompt)
333 - assert block_manager.can_swap_in(seq_group) == AllocStatus.OK
334 - before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
335 - before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
336 - mapping = block_manager.swap_in(seq_group)
337 - assert [x[0] for x in mapping] == cpu_blocks
338 - after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
339 - after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
340 - assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
341 - assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
342 -
343 -
344 -def test_swap_encoder_decoder():
345 - block_size = 4
346 - num_cpu_blocks = 4
347 - num_gpu_blocks = 4
348 - block_manager = BlockSpaceManagerV1(block_size,
349 - num_cpu_blocks,
350 - num_gpu_blocks,
351 - watermark=0)
352 -
353 - decoder_prompt, encoder_prompt, seq_group = \
354 - create_dummy_prompt_encoder_decoder(
355 - "1",
356 - decoder_prompt_length=block_size,
357 - encoder_prompt_length=block_size)
358 - decoder_prompt.status = SequenceStatus.WAITING
359 - encoder_prompt.status = SequenceStatus.WAITING
360 - block_manager.allocate(seq_group)
361 -
362 - # Emulate a forward pass by appending a single token.
363 - # The block manager then knows how many unprocessed
364 - # tokens will be written in the next forward pass.
365 - token_id = 0
366 - decoder_prompt.status = SequenceStatus.RUNNING
367 - decoder_prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
368 -
369 - # Swap encoder/decoder seq group from GPU -> CPU.
370 - decoder_gpu_blocks = block_manager.get_block_table(decoder_prompt)
371 - cross_gpu_blocks = block_manager.get_cross_block_table(seq_group)
372 - gpu_blocks = decoder_gpu_blocks + cross_gpu_blocks
373 - assert block_manager.can_swap_out(seq_group)
374 - before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
375 - before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
376 - mapping = block_manager.swap_out(seq_group)
377 - assert [x[0] for x in mapping] == gpu_blocks
378 - #assert list(mapping.keys()) == gpu_blocks
379 - after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
380 - after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
381 - assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
382 - assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
383 - decoder_prompt.status = SequenceStatus.SWAPPED
384 -
385 - # Swap encoder/decoder seq group from CPU -> GPU.
386 - decoder_cpu_blocks = block_manager.get_block_table(decoder_prompt)
387 - cross_cpu_blocks = block_manager.get_cross_block_table(seq_group)
388 - cpu_blocks = decoder_cpu_blocks + cross_cpu_blocks
389 - assert block_manager.can_swap_in(seq_group) == AllocStatus.OK
390 - before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
391 - before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
392 - mapping = block_manager.swap_in(seq_group)
393 - assert [x[0] for x in mapping] == cpu_blocks
394 - after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
395 - after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
396 - assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
397 - assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
398 -
399 -
400 -def test_free():
401 - block_size = 4
402 - num_cpu_blocks = 4
403 - num_gpu_blocks = 4
404 - block_manager = BlockSpaceManagerV1(block_size,
405 - num_cpu_blocks,
406 - num_gpu_blocks,
407 - watermark=0)
408 -
409 - prompt, seq_group = create_dummy_prompt("1", block_size)
410 - block_manager.allocate(seq_group)
411 -
412 - # Free allocated seq.
413 - prompt_blocks = len(block_manager.get_block_table(prompt))
414 - before_blocks = block_manager.get_num_free_gpu_blocks()
415 - block_manager.free(prompt)
416 - after_blocks = block_manager.get_num_free_gpu_blocks()
417 - assert after_blocks == before_blocks + prompt_blocks
418 -
419 - # Block table for freed seq is deleted.
420 - with pytest.raises(KeyError):
421 - block_manager.get_block_table(prompt)
422 -
423 -
424 -def test_free_encoder_decoder():
425 - block_size = 4
426 - num_cpu_blocks = 4
427 - num_gpu_blocks = 4
428 - block_manager = BlockSpaceManagerV1(block_size,
429 - num_cpu_blocks,
430 - num_gpu_blocks,
431 - watermark=0)
432 -
433 - decoder_prompt, encoder_prompt, seq_group = \
434 - create_dummy_prompt_encoder_decoder(
435 - "1",
436 - decoder_prompt_length=block_size,
437 - encoder_prompt_length=block_size)
438 - block_manager.allocate(seq_group)
439 -
440 - # Free allocated seq.
441 - decoder_prompt_blocks = len(block_manager.get_block_table(decoder_prompt))
442 - encoder_prompt_blocks = len(block_manager.get_cross_block_table(seq_group))
443 - prompt_blocks = decoder_prompt_blocks + encoder_prompt_blocks
444 - before_blocks = block_manager.get_num_free_gpu_blocks()
445 - block_manager.free(decoder_prompt)
446 - block_manager.free_cross(seq_group)
447 - after_blocks = block_manager.get_num_free_gpu_blocks()
448 - assert after_blocks == before_blocks + prompt_blocks
449 -
450 - # Block table for freed encoder & decoder seq's are deleted.
451 - with pytest.raises(KeyError):
452 - block_manager.get_block_table(decoder_prompt)
453 -
454 - # Block table for freed encoder & decoder seq's are deleted.
455 - with pytest.raises(KeyError):
456 - block_manager.get_block_table(encoder_prompt)
457 -
458 -
459 -def test_reset():
460 - block_size = 4
461 - num_cpu_blocks = 4
462 - num_gpu_blocks = 4
463 - block_manager = BlockSpaceManagerV1(block_size,
464 - num_cpu_blocks,
465 - num_gpu_blocks,
466 - watermark=0)
467 -
468 - # Allocate same seq group on all available gpu blocks.
469 - original_blocks = block_manager.get_num_free_gpu_blocks()
470 - for i in range(num_gpu_blocks):
471 - _, seq_group = create_dummy_prompt(str(i), block_size)
472 - block_manager.allocate(seq_group)
473 - assert block_manager.get_num_free_gpu_blocks() == 0
474 -
475 - # Resetting block manager frees all allocated blocks.
476 - block_manager.reset()
477 - assert block_manager.get_num_free_gpu_blocks() == original_blocks
478 -
479 -
480 -def test_reset_encoder_decoder():
481 - block_size = 4
482 - num_cpu_blocks = 4
483 - num_gpu_blocks = 4
484 - block_req_per_seq_group = 2
485 - block_manager = BlockSpaceManagerV1(block_size,
486 - num_cpu_blocks,
487 - num_gpu_blocks,
488 - watermark=0)
489 -
490 - # Allocate same seq group on all available gpu blocks.
491 - original_blocks = block_manager.get_num_free_gpu_blocks()
492 - for i in range(num_gpu_blocks // block_req_per_seq_group):
493 - _, _, seq_group = create_dummy_prompt_encoder_decoder(
494 - f"{i}",
495 - decoder_prompt_length=block_size,
496 - encoder_prompt_length=block_size)
497 - block_manager.allocate(seq_group)
498 - assert block_manager.get_num_free_gpu_blocks() == 0
499 -
500 - # Resetting block manager frees all allocated blocks.
501 - block_manager.reset()
502 - assert block_manager.get_num_free_gpu_blocks() == original_blocks
503 -
504 -
505 -def test_sliding_window_multi_seq():
506 - """
507 - Tests that memory allocation and deallocation is handled
508 - correctly with multiple sequences that exceed the sliding
509 - window's capacity.
510 - """
511 - block_size = 1
512 - num_cpu_blocks = 8
513 - num_gpu_blocks = 8
514 - sliding_window = 2
515 - block_manager = BlockSpaceManagerV1(block_size,
516 - num_cpu_blocks,
517 - num_gpu_blocks,
518 - sliding_window=sliding_window,
519 - watermark=0)
520 -
521 - assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
522 -
523 - parent = Sequence(seq_id=1,
524 - inputs={
525 - "prompt": "one two three",
526 - "prompt_token_ids": [0, 1, 2],
527 - },
528 - block_size=block_size)
529 - seq_group = SequenceGroup(request_id="1",
530 - seqs=[parent],
531 - arrival_time=time.time(),
532 - sampling_params=SamplingParams(),
533 - lora_request=None)
534 - block_manager.allocate(seq_group)
535 -
536 - # assert the number of blocks allocated is correct
537 - # the parent seq has len 3, but since sliding_window is 2,
538 - # we will use at most 2 blocks
539 - assert block_manager.get_num_free_gpu_blocks(
540 - ) == num_gpu_blocks - sliding_window
541 -
542 - # Fork prompt and copy block tables.
543 - child = parent.fork(2)
544 - block_manager.fork(parent, child)
545 -
546 - # assert the number of blocks allocated is correct
547 - # forking does not increase memory consumption
548 - assert block_manager.get_num_free_gpu_blocks(
549 - ) == num_gpu_blocks - sliding_window
550 -
551 - # assert both parent and child share all blocks
552 - assert block_manager.get_block_table(
553 - parent) == block_manager.get_block_table(child)
554 -
555 - token_id = 4
556 - # Append token to child. Block is shared so copy on write occurs.
557 - child.append_token_id(token_id, {token_id: Logprob(0.0)})
558 - block_manager.append_slots(child)
559 -
560 - # assert the number of blocks allocated is correct
561 - # we will use now one block more. Each seq will use 2 blocks,
562 - # but only one can be shared
563 - assert block_manager.get_num_free_gpu_blocks(
564 - ) == num_gpu_blocks - sliding_window - 1
565 -
566 - token_id = 5
567 - parent.append_token_id(token_id, {token_id: Logprob(0.0)})
568 - block_manager.append_slots(parent)
569 -
570 - # assert the number of blocks allocated is correct
571 - # no change, because both sequences are still just sharing one block
572 - assert block_manager.get_num_free_gpu_blocks(
573 - ) == num_gpu_blocks - sliding_window - 1
574 -
575 - block_table_parent = block_manager.get_block_table(parent)
576 - block_table_child = block_manager.get_block_table(child)
577 -
578 - assert block_table_parent != block_table_child
579 -
580 - # assert both blocks are sharing the second-last block
581 - assert block_table_parent[-2] == block_table_child[-2]
582 -
583 - # now let's clean up...
584 - block_manager.free(parent)
585 -
586 - # assert the number of blocks allocated is correct
587 - # We have freed one seq, reducing the ref count of two blocks by one.
588 - # One of the two was only used by the parent seq, so this is now free.
589 - # The child seq still consumes sliding_window blocks
590 - assert block_manager.get_num_free_gpu_blocks(
591 - ) == num_gpu_blocks - sliding_window
592 -
593 - # free all blocks
594 - block_manager.free(child)
595 -
596 - # assert all blocks are free now
597 - assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
598 -
599 -
600 -def test_mark_blocks_as_computed_with_prefix_cache_and_chunked_prefill():
601 - """When prefix cache and chunked prefill are enabled, the block manager
602 - should only mark a chunk of blocks as computed instead of all blocks.
603 - """
604 -
605 - block_size = 4
606 - num_cpu_blocks = 0
607 - num_gpu_blocks = 16
608 - block_manager = BlockSpaceManagerV1(block_size,
609 - num_gpu_blocks,
610 - num_cpu_blocks,
611 - watermark=0,
612 - enable_caching=True)
613 -
614 - # Set prompt size to have num_gpu_blocks - 1 full blocks.
615 - prompt_length = block_size * num_gpu_blocks - 1
616 -
617 - # Allocate (reserve) all blocks.
618 - _, seq_group = create_dummy_prompt("0",
619 - prompt_length,
620 - block_size=block_size)
621 - block_manager.allocate(seq_group)
622 - assert seq_group.seqs[0].n_blocks == num_gpu_blocks
623 -
624 - # 1st chunk: Compute 2 and half blocks. Should mark 2 blocks as computed.
625 - token_chunk_size = int(block_size * 2.5)
626 - block_manager.mark_blocks_as_computed(seq_group, token_chunk_size)
627 - computed_blocks = block_manager.get_all_computed_blocks(seq_group.seqs[0])
628 - assert len(computed_blocks) == 2
629 -
630 - # Actual computed tokens.
631 - seq_group.seqs[0].data.update_num_computed_tokens(token_chunk_size)
632 -
633 - # 2nd chunk: Complete 3rd block and additional 4 blocks.
634 - token_chunk_size = int(block_size * 4.5)
635 - block_manager.mark_blocks_as_computed(seq_group, token_chunk_size)
636 - computed_blocks = block_manager.get_all_computed_blocks(seq_group.seqs[0])
637 - assert len(computed_blocks) == 7
tests/core/test_chunked_prefill_scheduler.py
+24 -44
@@ -8,7 +8,6 @@ from vllm.core.interfaces import AllocStatus
8 from vllm.core.scheduler import Scheduler
9 from vllm.sequence import Logprob, SequenceGroup
10
11 -from ..utils import check_deprecated_block_manager_usage
11 from .utils import create_dummy_prompt
12
13
@@ -28,25 +27,16 @@ def schedule_and_update_computed_tokens(scheduler):
27 return metas, out
28
29
31 -@pytest.fixture(scope="module", autouse=True)
32 -def check_deprecated_block_manager():
33 - check_deprecated_block_manager_usage(
34 - 'tests/core/test_chunked_prefill_scheduler.py')
35 -
36 -
37 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
38 -def test_simple(use_v2_block_manager: bool):
30 +def test_simple():
31 """Verify basic scheduling works."""
32 block_size = 4
33 num_seq_group = 4
34 max_model_len = 16
35 max_num_batched_tokens = 64
44 - scheduler_config = SchedulerConfig(
45 - max_num_batched_tokens,
46 - num_seq_group,
47 - max_model_len,
48 - enable_chunked_prefill=True,
49 - use_v2_block_manager=use_v2_block_manager)
36 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
37 + num_seq_group,
38 + max_model_len,
39 + enable_chunked_prefill=True)
40 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
41 cache_config.num_cpu_blocks = 8
42 cache_config.num_gpu_blocks = 8
@@ -81,8 +71,7 @@ def test_simple(use_v2_block_manager: bool):
71 assert len(seq_group_meta) == num_seq_group
72
73
84 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
85 -def test_chunk(use_v2_block_manager: bool):
74 +def test_chunk():
75 """Verify prefills are chunked properly."""
76 block_size = 4
77 max_seqs = 60
@@ -93,7 +82,7 @@ def test_chunk(use_v2_block_manager: bool):
82 max_seqs,
83 max_model_len,
84 enable_chunked_prefill=True,
96 - use_v2_block_manager=use_v2_block_manager)
85 + )
86 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
87 cache_config.num_cpu_blocks = 32
88 cache_config.num_gpu_blocks = 32
@@ -131,8 +120,7 @@ def test_chunk(use_v2_block_manager: bool):
120 assert out.num_batched_tokens == 57
121
122
134 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
135 -def test_complex(use_v2_block_manager: bool):
123 +def test_complex():
124 block_size = 4
125 max_seqs = 60
126 max_model_len = 80
@@ -142,7 +130,7 @@ def test_complex(use_v2_block_manager: bool):
130 max_seqs,
131 max_model_len,
132 enable_chunked_prefill=True,
145 - use_v2_block_manager=use_v2_block_manager)
133 + )
134 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
135 cache_config.num_cpu_blocks = 64
136 cache_config.num_gpu_blocks = 64
@@ -201,8 +189,7 @@ def test_complex(use_v2_block_manager: bool):
189 assert running[2].is_prefill()
190
191
204 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
205 -def test_maximal_decoding(use_v2_block_manager: bool):
192 +def test_maximal_decoding():
193 """Verify decoding requests are prioritized."""
194 block_size = 4
195 max_seqs = 2
@@ -213,7 +200,7 @@ def test_maximal_decoding(use_v2_block_manager: bool):
200 max_seqs,
201 max_model_len,
202 enable_chunked_prefill=True,
216 - use_v2_block_manager=use_v2_block_manager)
203 + )
204 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
205 cache_config.num_cpu_blocks = 8
206 cache_config.num_gpu_blocks = 8
@@ -295,8 +282,7 @@ def test_maximal_decoding(use_v2_block_manager: bool):
282 assert out.num_batched_tokens == 2
283
284
298 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
299 -def test_prompt_limit(use_v2_block_manager: bool):
285 +def test_prompt_limit():
286 """Verify max_num_batched_tokens < max_model_len is possible."""
287 block_size = 4
288 max_seqs = 32
@@ -307,7 +293,7 @@ def test_prompt_limit(use_v2_block_manager: bool):
293 max_seqs,
294 max_model_len,
295 enable_chunked_prefill=True,
310 - use_v2_block_manager=use_v2_block_manager)
296 + )
297 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
298 cache_config.num_cpu_blocks = 16
299 cache_config.num_gpu_blocks = 16
@@ -330,8 +316,7 @@ def test_prompt_limit(use_v2_block_manager: bool):
316 assert out.num_batched_tokens == 32
317
318
333 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
334 -def test_prompt_limit_exceed(use_v2_block_manager: bool):
319 +def test_prompt_limit_exceed():
320 block_size = 4
321 max_seqs = 64
322 max_model_len = 32
@@ -356,8 +341,7 @@ def test_prompt_limit_exceed(use_v2_block_manager: bool):
341 assert out.ignored_seq_groups[0] == seq_group
342
343
359 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
360 -def test_swap(use_v2_block_manager: bool):
344 +def test_swap():
345 """Verify swapping works with chunked prefill requests"""
346 block_size = 4
347 max_seqs = 30
@@ -368,7 +352,7 @@ def test_swap(use_v2_block_manager: bool):
352 max_seqs,
353 max_model_len,
354 enable_chunked_prefill=True,
371 - use_v2_block_manager=use_v2_block_manager)
355 + )
356 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
357 cache_config.num_cpu_blocks = 16
358 cache_config.num_gpu_blocks = 16
@@ -414,8 +398,7 @@ def test_swap(use_v2_block_manager: bool):
398 assert out.blocks_to_swap_out == []
399
400
417 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
418 -def test_running_prefill_prioritized_over_swap(use_v2_block_manager: bool):
401 +def test_running_prefill_prioritized_over_swap():
402 block_size = 4
403 max_seqs = 30
404 max_model_len = 200
@@ -425,7 +408,7 @@ def test_running_prefill_prioritized_over_swap(use_v2_block_manager: bool):
408 max_seqs,
409 max_model_len,
410 enable_chunked_prefill=True,
428 - use_v2_block_manager=use_v2_block_manager)
411 + )
412 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
413 cache_config.num_cpu_blocks = 32
414 cache_config.num_gpu_blocks = 32
@@ -508,8 +491,7 @@ def test_running_prefill_prioritized_over_swap(use_v2_block_manager: bool):
491 assert out.blocks_to_swap_out == []
492
493
511 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
512 -def test_chunked_prefill_preempt(use_v2_block_manager: bool):
494 +def test_chunked_prefill_preempt():
495 """Verify preempt works with chunked prefill requests"""
496 block_size = 4
497 max_seqs = 30
@@ -520,7 +502,7 @@ def test_chunked_prefill_preempt(use_v2_block_manager: bool):
502 max_seqs,
503 max_model_len,
504 enable_chunked_prefill=True,
523 - use_v2_block_manager=use_v2_block_manager)
505 + )
506 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
507 cache_config.num_cpu_blocks = 16
508 cache_config.num_gpu_blocks = 16
@@ -575,8 +557,7 @@ def test_chunked_prefill_preempt(use_v2_block_manager: bool):
557 assert out.num_batched_tokens == max_num_batched_tokens
558
559
578 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
579 -def test_chunked_prefill_max_seqs(use_v2_block_manager: bool):
560 +def test_chunked_prefill_max_seqs():
561 block_size = 4
562 max_seqs = 2
563 max_model_len = 80
@@ -586,7 +567,7 @@ def test_chunked_prefill_max_seqs(use_v2_block_manager: bool):
567 max_seqs,
568 max_model_len,
569 enable_chunked_prefill=True,
589 - use_v2_block_manager=use_v2_block_manager)
570 + )
571 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
572 cache_config.num_cpu_blocks = 128
573 cache_config.num_gpu_blocks = 128
@@ -629,8 +610,7 @@ def test_chunked_prefill_max_seqs(use_v2_block_manager: bool):
610 assert not running[1].is_prefill()
611
612
632 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
633 -def test_perfix_caching(use_v2_block_manager: bool):
613 +def test_perfix_caching():
614 """Verify allocating full blocks when prefix caching is enabled."""
615 block_size = 4
616 max_seqs = 10
@@ -641,7 +621,7 @@ def test_perfix_caching(use_v2_block_manager: bool):
621 max_seqs,
622 max_model_len,
623 enable_chunked_prefill=True,
644 - use_v2_block_manager=use_v2_block_manager)
624 + )
625 cache_config = CacheConfig(block_size,
626 1.0,
627 1,
tests/core/test_num_computed_tokens_update.py
-1
@@ -31,7 +31,6 @@ def test_num_computed_tokens_update(num_scheduler_steps: int,
31 # Make a vllm engine
32 runner = VllmRunner(model_name=MODEL,
33 gpu_memory_utilization=0.7,
34 - use_v2_block_manager=True,
34 num_scheduler_steps=num_scheduler_steps,
35 enable_chunked_prefill=enable_chunked_prefill,
36 enforce_eager=enforce_eager)
tests/core/test_scheduler.py
+55 -95
@@ -3,7 +3,7 @@ from collections import deque
3 from typing import List, Set, Tuple
4 from unittest.mock import MagicMock
5
6 -import pytest
6 +import pytest # noqa
7 from torch import Use # noqa
8
9 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
@@ -12,23 +12,18 @@ from vllm.core.scheduler import Scheduler, SchedulingBudget
12 from vllm.lora.request import LoRARequest
13 from vllm.sequence import SequenceGroup, SequenceStatus
14
15 -from ..utils import check_deprecated_block_manager_usage
15 from .utils import (append_new_token, append_new_token_seq_group,
16 create_dummy_prompt, get_sequence_groups,
17 schedule_and_update_computed_tokens)
18
19
21 -@pytest.fixture(scope="module", autouse=True)
22 -def check_deprecated_block_manager():
23 - check_deprecated_block_manager_usage(
24 - "tests/core/test_chunked_prefill_scheduler.py")
25 -
26 -
27 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
28 -def test_scheduler_add_seq_group(use_v2_block_manager: bool):
20 +def test_scheduler_add_seq_group():
21 block_size = 4
22 scheduler_config = SchedulerConfig(
31 - 100, 64, 1, use_v2_block_manager=use_v2_block_manager)
23 + 100,
24 + 64,
25 + 1,
26 + )
27 cache_config = CacheConfig(block_size, 1.0, 1, cache_dtype="auto")
28 cache_config.num_cpu_blocks = 4
29 cache_config.num_gpu_blocks = 4
@@ -44,11 +39,13 @@ def test_scheduler_add_seq_group(use_v2_block_manager: bool):
39 assert scheduler.get_num_unfinished_seq_groups() == i + 1
40
41
47 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
48 -def test_scheduler_abort_seq_group(use_v2_block_manager: bool):
42 +def test_scheduler_abort_seq_group():
43 block_size = 4
44 scheduler_config = SchedulerConfig(
51 - 100, 64, 1, use_v2_block_manager=use_v2_block_manager)
45 + 100,
46 + 64,
47 + 1,
48 + )
49 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
50 cache_config.num_cpu_blocks = 4
51 cache_config.num_gpu_blocks = 4
@@ -68,8 +65,7 @@ def test_scheduler_abort_seq_group(use_v2_block_manager: bool):
65 assert scheduler.get_num_unfinished_seq_groups() == 0
66
67
71 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
72 -def test_scheduler_schedule_simple(use_v2_block_manager: bool):
68 +def test_scheduler_schedule_simple():
69 block_size = 4
70 num_seq_group = 4
71 max_model_len = 16
@@ -77,7 +73,7 @@ def test_scheduler_schedule_simple(use_v2_block_manager: bool):
73 64,
74 num_seq_group,
75 max_model_len,
80 - use_v2_block_manager=use_v2_block_manager)
76 + )
77 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
78 cache_config.num_cpu_blocks = 8
79 cache_config.num_gpu_blocks = 8
@@ -112,8 +108,7 @@ def test_scheduler_schedule_simple(use_v2_block_manager: bool):
108 append_new_token(out, 1)
109
110
115 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
116 -def test_scheduler_prefill_prioritized(use_v2_block_manager: bool):
111 +def test_scheduler_prefill_prioritized():
112 """Verify running batched tokens are not applied to prefill requests."""
113 block_size = 4
114 max_model_len = 30
@@ -122,7 +117,7 @@ def test_scheduler_prefill_prioritized(use_v2_block_manager: bool):
117 max_batched_num_tokens,
118 2,
119 max_model_len,
125 - use_v2_block_manager=use_v2_block_manager)
120 + )
121 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
122 cache_config.num_cpu_blocks = 16
123 cache_config.num_gpu_blocks = 16
@@ -146,12 +141,14 @@ def test_scheduler_prefill_prioritized(use_v2_block_manager: bool):
141 assert get_sequence_groups(out) == [seq_group_b]
142
143
149 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
150 -def test_scheduler_schedule_preempt_abort(use_v2_block_manager: bool):
144 +def test_scheduler_schedule_preempt_abort():
145 block_size = 4
146 max_model_len = 16
147 scheduler_config = SchedulerConfig(
154 - 64, 2, max_model_len, use_v2_block_manager=use_v2_block_manager)
148 + 64,
149 + 2,
150 + max_model_len,
151 + )
152 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
153 cache_config.num_cpu_blocks = 2
154 cache_config.num_gpu_blocks = 2
@@ -201,8 +198,7 @@ def test_scheduler_schedule_preempt_abort(use_v2_block_manager: bool):
198 assert scheduler.get_num_unfinished_seq_groups() == 1
199
200
204 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
205 -def test_scheduler_max_seqs(use_v2_block_manager: bool):
201 +def test_scheduler_max_seqs():
202 block_size = 4
203 num_seq_group = 4
204 max_seq_group = 2
@@ -211,7 +207,7 @@ def test_scheduler_max_seqs(use_v2_block_manager: bool):
207 64,
208 max_seq_group,
209 max_model_len,
214 - use_v2_block_manager=use_v2_block_manager)
210 + )
211 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
212 cache_config.num_cpu_blocks = 8
213 cache_config.num_gpu_blocks = 8
@@ -249,15 +245,14 @@ def test_scheduler_max_seqs(use_v2_block_manager: bool):
245 assert set(get_sequence_groups(out)) == set([all_seq_groups[1]])
246
247
252 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
253 -def test_scheduler_delay_factor(use_v2_block_manager: bool):
248 +def test_scheduler_delay_factor():
249 block_size = 4
250 scheduler_config = SchedulerConfig(
251 100,
252 64,
253 16,
254 delay_factor=0.5,
260 - use_v2_block_manager=use_v2_block_manager)
255 + )
256 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
257 cache_config.num_cpu_blocks = 8
258 cache_config.num_gpu_blocks = 8
@@ -294,12 +289,10 @@ def test_scheduler_delay_factor(use_v2_block_manager: bool):
289 append_new_token(out, 1)
290
291
297 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
298 -def test_swapped_out_prioritized(use_v2_block_manager: bool):
292 +def test_swapped_out_prioritized():
293 block_size = 4
294 scheduler = initialize_scheduler(max_num_seqs=6,
295 block_size=block_size,
302 - use_v2_block_manager=use_v2_block_manager,
296 num_cpu_blocks=64,
297 num_gpu_blocks=64)
298 # best_of=2 * 3 == 6 sequences.
@@ -351,7 +344,6 @@ def initialize_scheduler(
344 max_token_budget=1000,
345 max_model_len=1000,
346 lora_config=None,
354 - use_v2_block_manager=False,
347 block_size=4,
348 num_cpu_blocks=8,
349 num_gpu_blocks=8,
@@ -361,7 +353,7 @@ def initialize_scheduler(
353 max_token_budget,
354 max_num_seqs,
355 max_model_len,
364 - use_v2_block_manager=use_v2_block_manager)
356 + )
357 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
358 cache_config.num_cpu_blocks = num_cpu_blocks
359 cache_config.num_gpu_blocks = num_gpu_blocks
@@ -386,15 +378,12 @@ def add_token_budget(budget: SchedulingBudget,
378 budget.add_num_seqs(mock_seq_group.request_id, num_curr_seqs)
379
380
389 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
390 -def test_prefill_schedule_max_prompt_len(use_v2_block_manager: bool):
381 +def test_prefill_schedule_max_prompt_len():
382 """
383 Test prompt longer than max_prompt_len is aborted.
384 """
385 block_size = 4
395 - scheduler = initialize_scheduler(max_model_len=30,
396 - use_v2_block_manager=use_v2_block_manager,
397 - block_size=block_size)
386 + scheduler = initialize_scheduler(max_model_len=30, block_size=block_size)
387 _, seq_group = create_dummy_prompt("0",
388 prompt_length=60,
389 block_size=block_size)
@@ -409,14 +398,12 @@ def test_prefill_schedule_max_prompt_len(use_v2_block_manager: bool):
398 assert len(remaining_waiting) == 0
399
400
412 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
413 -def test_prefill_schedule_token_budget(use_v2_block_manager: bool):
401 +def test_prefill_schedule_token_budget():
402 """
403 Test token budget respected.
404 """
405 block_size = 4
418 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
419 - block_size=block_size,
406 + scheduler = initialize_scheduler(block_size=block_size,
407 num_cpu_blocks=64,
408 num_gpu_blocks=64)
409 budget = create_token_budget(token_budget=0)
@@ -446,8 +433,7 @@ def test_prefill_schedule_token_budget(use_v2_block_manager: bool):
433 assert len(remaining_waiting) == 1
434
435 # Test when current_batched_tokens respected.
449 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
450 - block_size=block_size,
436 + scheduler = initialize_scheduler(block_size=block_size,
437 num_cpu_blocks=16,
438 num_gpu_blocks=16)
439 budget = create_token_budget(token_budget=60)
@@ -474,14 +460,12 @@ def test_prefill_schedule_token_budget(use_v2_block_manager: bool):
460 assert len(remaining_waiting) == 0
461
462
477 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
478 -def test_prefill_schedule_max_seqs(use_v2_block_manager: bool):
463 +def test_prefill_schedule_max_seqs():
464 """
465 Test max seq respected.
466 """
467 block_size = 4
483 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
484 - block_size=block_size,
468 + scheduler = initialize_scheduler(block_size=block_size,
469 num_cpu_blocks=64,
470 num_gpu_blocks=64)
471 budget = create_token_budget(max_num_seqs=2)
@@ -515,15 +499,13 @@ def test_prefill_schedule_max_seqs(use_v2_block_manager: bool):
499 assert len(remaining_waiting) == 1
500
501
518 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
519 -def test_prefill_schedule_max_lora(use_v2_block_manager: bool):
502 +def test_prefill_schedule_max_lora():
503 """
504 Test max lora is respected and prioritized.
505 """
506 block_size = 4
507 lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
508 scheduler = initialize_scheduler(lora_config=lora_config,
526 - use_v2_block_manager=use_v2_block_manager,
509 block_size=block_size,
510 num_cpu_blocks=64,
511 num_gpu_blocks=64)
@@ -570,14 +552,12 @@ def test_prefill_schedule_max_lora(use_v2_block_manager: bool):
552 assert budget.num_batched_tokens == 60
553
554
573 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
574 -def test_prefill_schedule_no_block_manager_capacity(use_v2_block_manager):
555 +def test_prefill_schedule_no_block_manager_capacity():
556 """
557 Test sequence cannot be scheduled due to block manager has no capacity.
558 """
559 block_size = 4
579 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
580 - block_size=block_size,
560 + scheduler = initialize_scheduler(block_size=block_size,
561 num_gpu_blocks=128,
562 num_cpu_blocks=128)
563 budget = create_token_budget()
@@ -614,14 +594,12 @@ def test_prefill_schedule_no_block_manager_capacity(use_v2_block_manager):
594 assert len(remaining_waiting) == 0
595
596
617 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
618 -def test_decode_schedule_preempted(use_v2_block_manager: bool):
597 +def test_decode_schedule_preempted():
598 """
599 Test decodes cannot be scheduled and preempted.
600 """
601 block_size = 4
623 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
624 - block_size=block_size,
602 + scheduler = initialize_scheduler(block_size=block_size,
603 num_cpu_blocks=64,
604 num_gpu_blocks=64)
605 curr_loras = None
@@ -660,14 +638,12 @@ def test_decode_schedule_preempted(use_v2_block_manager: bool):
638 assert output.blocks_to_copy == []
639
640
663 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
664 -def test_decode_swap_beam_search(use_v2_block_manager: bool):
641 +def test_decode_swap_beam_search():
642 """
643 Test best_of > 1 swap out blocks
644 """
645 block_size = 4
669 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
670 - block_size=block_size,
646 + scheduler = initialize_scheduler(block_size=block_size,
647 num_gpu_blocks=64,
648 num_cpu_blocks=64)
649 curr_loras = None
@@ -716,14 +692,12 @@ def test_decode_swap_beam_search(use_v2_block_manager: bool):
692 assert output.blocks_to_copy == []
693
694
719 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
720 -def test_schedule_decode_blocks_to_copy_update(use_v2_block_manager: bool):
695 +def test_schedule_decode_blocks_to_copy_update():
696 """
697 Verify blocks_to_copy is updated.
698 """
699 block_size = 4
725 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
726 - block_size=4,
700 + scheduler = initialize_scheduler(block_size=4,
701 num_cpu_blocks=16,
702 num_gpu_blocks=16)
703 _, seq_group = create_dummy_prompt("1",
@@ -754,11 +728,9 @@ def test_schedule_decode_blocks_to_copy_update(use_v2_block_manager: bool):
728 assert output.blocks_to_copy == [(2, 3)]
729
730
757 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
758 -def test_schedule_swapped_simple(use_v2_block_manager: bool):
731 +def test_schedule_swapped_simple():
732 block_size = 4
760 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
761 - block_size=block_size)
733 + scheduler = initialize_scheduler(block_size=block_size)
734 curr_loras = None
735 blocks_to_swap_out: List[Tuple[int, int]] = []
736 _, seq_group = create_dummy_prompt("1",
@@ -785,11 +757,9 @@ def test_schedule_swapped_simple(use_v2_block_manager: bool):
757 assert blocks_to_swap_out == blocks_to_swap_in_reverse
758
759
788 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
789 -def test_schedule_swapped_max_token_budget(use_v2_block_manager: bool):
760 +def test_schedule_swapped_max_token_budget():
761 block_size = 4
791 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
792 - block_size=block_size,
762 + scheduler = initialize_scheduler(block_size=block_size,
763 num_cpu_blocks=32,
764 num_gpu_blocks=32)
765 curr_loras = None
@@ -822,11 +792,9 @@ def test_schedule_swapped_max_token_budget(use_v2_block_manager: bool):
792 assert len(output.prefill_seq_groups) == 0
793
794
825 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
826 -def test_schedule_swapped_max_seqs(use_v2_block_manager: bool):
795 +def test_schedule_swapped_max_seqs():
796 block_size = 4
828 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
829 - block_size=block_size,
797 + scheduler = initialize_scheduler(block_size=block_size,
798 num_cpu_blocks=64,
799 num_gpu_blocks=64)
800 curr_loras = None
@@ -859,12 +827,10 @@ def test_schedule_swapped_max_seqs(use_v2_block_manager: bool):
827 assert len(output.prefill_seq_groups) == 0
828
829
862 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
863 -def test_schedule_swapped_max_loras(use_v2_block_manager: bool):
830 +def test_schedule_swapped_max_loras():
831 block_size = 4
832 lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
833 scheduler = initialize_scheduler(lora_config=lora_config,
867 - use_v2_block_manager=use_v2_block_manager,
834 block_size=block_size,
835 num_cpu_blocks=32,
836 num_gpu_blocks=32)
@@ -894,11 +860,9 @@ def test_schedule_swapped_max_loras(use_v2_block_manager: bool):
860 assert len(curr_loras) == 1
861
862
897 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
898 -def test_schedule_swapped_cannot_swap_in(use_v2_block_manager: bool):
863 +def test_schedule_swapped_cannot_swap_in():
864 block_size = 4
900 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
901 - block_size=block_size,
865 + scheduler = initialize_scheduler(block_size=block_size,
866 num_cpu_blocks=32,
867 num_gpu_blocks=32)
868 curr_loras = None
@@ -927,11 +891,9 @@ def test_schedule_swapped_cannot_swap_in(use_v2_block_manager: bool):
891 assert len(output.prefill_seq_groups) == 0
892
893
930 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
931 -def test_infeasible_swap(use_v2_block_manager: bool):
894 +def test_infeasible_swap():
895 block_size = 4
933 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
934 - block_size=block_size,
896 + scheduler = initialize_scheduler(block_size=block_size,
897 num_cpu_blocks=32,
898 num_gpu_blocks=32)
899 curr_loras = None
@@ -961,11 +923,9 @@ def test_infeasible_swap(use_v2_block_manager: bool):
923 assert len(output.prefill_seq_groups) == 0
924
925
964 -@pytest.mark.parametrize('use_v2_block_manager', [True, False])
965 -def test_schedule_swapped_blocks_to_copy(use_v2_block_manager: bool):
926 +def test_schedule_swapped_blocks_to_copy():
927 block_size = 4
967 - scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
968 - block_size=block_size,
928 + scheduler = initialize_scheduler(block_size=block_size,
929 num_cpu_blocks=32,
930 num_gpu_blocks=32)
931 curr_loras = None
tests/metrics/test_metrics.py
+8 -8
@@ -185,13 +185,14 @@ def test_metric_spec_decode(
185 ) -> None:
186 k = 5
187
188 - with vllm_runner(model,
189 - dtype=dtype,
190 - disable_log_stats=False,
191 - gpu_memory_utilization=0.4,
192 - speculative_model=model,
193 - num_speculative_tokens=k,
194 - use_v2_block_manager=True) as vllm_model:
188 + with vllm_runner(
189 + model,
190 + dtype=dtype,
191 + disable_log_stats=False,
192 + gpu_memory_utilization=0.4,
193 + speculative_model=model,
194 + num_speculative_tokens=k,
195 + ) as vllm_model:
196
197 # Force log interval to be 0 to catch all metrics.
198 stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
@@ -242,7 +243,6 @@ def test_metric_spec_decode_interval(
243 gpu_memory_utilization=0.4,
244 speculative_model=model,
245 num_speculative_tokens=k,
245 - use_v2_block_manager=True,
246 enforce_eager=True)
247
248 engine = LLMEngine.from_engine_args(engine_args)
tests/multi_step/test_correctness_async_llm.py
-1
@@ -17,7 +17,6 @@ NUM_PROMPTS = [10]
17
18 DEFAULT_SERVER_ARGS: List[str] = [
19 "--disable-log-requests",
20 - "--use-v2-block-manager",
20 "--worker-use-ray",
21 "--gpu-memory-utilization",
22 "0.85",
tests/multi_step/test_correctness_llm.py
-4
@@ -76,7 +76,6 @@ def test_multi_step_llm(
76 enforce_eager=enforce_eager,
77 gpu_memory_utilization=0.7,
78 tensor_parallel_size=tp_size,
79 - use_v2_block_manager=True,
79 enable_chunked_prefill=enable_chunked_prefill,
80 num_scheduler_steps=num_scheduler_steps,
81 ) as vllm_model:
@@ -169,7 +168,6 @@ def test_multi_step_llm_w_prompt_logprobs(
168 enforce_eager=enforce_eager,
169 gpu_memory_utilization=0.7,
170 tensor_parallel_size=tp_size,
172 - use_v2_block_manager=True,
171 num_scheduler_steps=num_scheduler_steps,
172 ) as vllm_model:
173 vllm_outputs = vllm_model.generate_greedy_logprobs(
@@ -305,7 +303,6 @@ def test_multi_step_llm_chunked_prefill_prefix_cache(
303 enforce_eager=enforce_eager,
304 gpu_memory_utilization=0.7,
305 tensor_parallel_size=tp_size,
308 - use_v2_block_manager=True,
306 num_scheduler_steps=num_scheduler_steps,
307 max_model_len=48,
308 max_num_batched_tokens=48,
@@ -324,7 +321,6 @@ def test_multi_step_llm_chunked_prefill_prefix_cache(
321 enforce_eager=enforce_eager,
322 gpu_memory_utilization=0.7,
323 tensor_parallel_size=tp_size,
327 - use_v2_block_manager=True,
324 enable_chunked_prefill=True,
325 enable_prefix_caching=True,
326 num_scheduler_steps=num_scheduler_steps,
tests/prefix_caching/test_prefix_caching.py
-89
@@ -2,15 +2,9 @@
2
3 Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4 """
5 -from typing import List
6 -
5 import pytest
6
7 from tests.kernels.utils import override_backend_env_variable
10 -from tests.utils import check_deprecated_block_manager_usage
11 -from vllm.block import PhysicalTokenBlock
12 -from vllm.core.block_manager_v1 import CachedBlockAllocator
13 -from vllm.utils import Device
8
9 from ..models.utils import check_outputs_equal
10
@@ -19,92 +13,11 @@ MODELS = [
13 ]
14
15
22 -@pytest.fixture(scope="module", autouse=True)
23 -def check_deprecated_block_manager():
24 - check_deprecated_block_manager_usage(
25 - 'tests/prefix_caching/test_prefix_caching.py')
26 -
27 -
28 -@pytest.mark.parametrize("block_size", [16])
29 -@pytest.mark.parametrize("num_blocks", [16])
30 -def test_block_allocator(
31 - block_size: int,
32 - num_blocks: int,
33 -):
34 - block_hash = 1
35 - block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
36 -
37 - # Allocate two PysicalTokenBlocks with the same hash and check
38 - # that they are the same PhysicalTokenBlock
39 - first_block = block_allocator.allocate(block_hash, 0)
40 - second_block = block_allocator.allocate(block_hash, 0)
41 - assert (first_block == second_block)
42 - assert (second_block.ref_count == 2)
43 -
44 - # Check metric: 1 hit of 2 queries
45 - assert block_allocator.get_prefix_cache_hit_rate() == 0.5
46 -
47 - # Free the first_block and confirm that the ref_count is correctly
48 - # decremented on the second block
49 - block_allocator.free(first_block)
50 - assert (second_block.ref_count == 1)
51 -
52 - # Free the second block
53 - block_allocator.free(second_block)
54 -
55 - # Reallocate the first block and confirm that, even after the block
56 - # had its ref_count go to 0, we still get the same block back
57 - first_block = block_allocator.allocate(block_hash, 0)
58 - assert (first_block == second_block)
59 - assert (first_block.block_hash == block_hash)
60 -
61 - # Allocate one more time to get 3/4 hit rate for easy checking
62 - block_allocator.allocate(block_hash, 0)
63 - assert block_allocator.get_prefix_cache_hit_rate() == 0.75
64 -
65 -
66 -@pytest.mark.parametrize("num_blocks", [16])
67 -def test_eviction(num_blocks: int, ):
68 - block_size = 16
69 - block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
70 - blocks: List[PhysicalTokenBlock] = []
71 -
72 - for i in range(num_blocks):
73 - # use i as the block_hash
74 - blocks.append(block_allocator.allocate(i, 0))
75 -
76 - #Free all blocks
77 - for block in blocks:
78 - block_allocator.free(block)
79 -
80 - # Allocate a new block and confirm that it's the first block freed.
81 - # I.E The Least Recently Used block
82 - new_block_hash = block_size
83 - new_block = block_allocator.allocate(new_block_hash, 0)
84 - assert (new_block == blocks[0])
85 - assert (new_block.block_hash == new_block_hash)
86 -
87 - # Reallocate the second in blocks to remove it from the free list
88 - realloc_block_hash = 1
89 - realloc_block = block_allocator.allocate(realloc_block_hash, 0)
90 - assert (realloc_block == blocks[realloc_block_hash])
91 - assert (realloc_block.block_hash == realloc_block_hash)
92 -
93 - # Allocate a new block and confirm that it's not the realloc_block,
94 - # since the realloc_block shouldn't be in the free list
95 - new_block_hash = block_size + 1
96 - new_block = block_allocator.allocate(new_block_hash, 0)
97 - assert (realloc_block != new_block)
98 - assert (new_block.block_hash == new_block_hash)
99 - assert (new_block.block_number == 2)
100 -
101 -
16 @pytest.mark.parametrize("model", MODELS)
17 @pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
18 @pytest.mark.parametrize("dtype", ["half"])
19 @pytest.mark.parametrize("max_tokens", [5])
20 @pytest.mark.parametrize("cached_position", [0, 1])
107 -@pytest.mark.parametrize("use_v2_block_manager", [False, True])
21 def test_mixed_requests(
22 hf_runner,
23 vllm_runner,
@@ -114,7 +27,6 @@ def test_mixed_requests(
27 dtype: str,
28 max_tokens: int,
29 cached_position: int,
117 - use_v2_block_manager: bool,
30 monkeypatch,
31 ) -> None:
32 """
@@ -132,7 +44,6 @@ def test_mixed_requests(
44 model,
45 dtype=dtype,
46 enable_prefix_caching=True,
135 - use_v2_block_manager=use_v2_block_manager,
47 ) as vllm_model:
48 # Run the first prompt so the cache is populated
49 vllm_outputs = vllm_model.generate_greedy([cached_prompt], max_tokens)
tests/spec_decode/e2e/test_compatibility.py
+10 -58
@@ -1,27 +1,15 @@
1 import pytest
2
3 -from tests.utils import check_deprecated_block_manager_usage
3 from vllm import SamplingParams
4
5 from .conftest import get_output_from_llm_generator
6
7
9 -@pytest.fixture(scope="module", autouse=True)
10 -def check_deprecated_block_manager():
11 - check_deprecated_block_manager_usage(
12 - 'tests/spec_decode/e2e/test_compatibility.py')
13 -
14 -
15 -@pytest.mark.parametrize(
16 - "common_llm_kwargs",
17 - [{
18 - "model": "JackFram/llama-68m",
19 - "speculative_model": "JackFram/llama-68m",
20 - "num_speculative_tokens": 5,
21 -
22 - # Required for spec decode.
23 - "use_v2_block_manager": True
24 - }])
8 +@pytest.mark.parametrize("common_llm_kwargs", [{
9 + "model": "JackFram/llama-68m",
10 + "speculative_model": "JackFram/llama-68m",
11 + "num_speculative_tokens": 5,
12 +}])
13 @pytest.mark.parametrize("per_test_common_llm_kwargs", [
14 {
15 "enable_chunked_prefill": True,
@@ -51,16 +39,11 @@ def test_spec_decode_xfail_chunked_prefill(test_llm_generator):
39 sampling_params)
40
41
54 -@pytest.mark.parametrize(
55 - "common_llm_kwargs",
56 - [{
57 - "model": "meta-llama/Llama-2-7b-chat-hf",
58 - "speculative_model": "JackFram/llama-68m",
59 - "num_speculative_tokens": 5,
60 -
61 - # Required for spec decode.
62 - "use_v2_block_manager": True
63 - }])
42 +@pytest.mark.parametrize("common_llm_kwargs", [{
43 + "model": "meta-llama/Llama-2-7b-chat-hf",
44 + "speculative_model": "JackFram/llama-68m",
45 + "num_speculative_tokens": 5,
46 +}])
47 @pytest.mark.parametrize(
48 "per_test_common_llm_kwargs",
49 [
@@ -101,34 +84,3 @@ def test_spec_decode_xfail_spec_max_model_len(test_llm_generator):
84 with pytest.raises(ValueError, match="cannot be larger than"):
85 get_output_from_llm_generator(test_llm_generator, prompts,
86 sampling_params)
104 -
105 -
106 -@pytest.mark.parametrize("common_llm_kwargs", [{
107 - "model": "JackFram/llama-68m",
108 - "speculative_model": "JackFram/llama-68m",
109 - "num_speculative_tokens": 5,
110 - "use_v2_block_manager": False,
111 -}])
112 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
113 -@pytest.mark.parametrize("test_llm_kwargs", [{}])
114 -@pytest.mark.parametrize("seed", [1])
115 -def test_spec_decode_xfail_block_manager_v1(test_llm_generator):
116 - """Verify that speculative decoding with block manager v1 fails.
117 - """
118 - output_len = 128
119 - temperature = 0.0
120 -
121 - prompts = [
122 - "Hello, my name is",
123 - ]
124 -
125 - sampling_params = SamplingParams(
126 - max_tokens=output_len,
127 - ignore_eos=True,
128 - temperature=temperature,
129 - )
130 -
131 - with pytest.raises(ValueError,
132 - match="Speculative decoding requires usage of the V2"):
133 - get_output_from_llm_generator(test_llm_generator, prompts,
134 - sampling_params)
tests/spec_decode/e2e/test_eagle_correctness.py
-18
@@ -43,9 +43,6 @@ PRECISION = "float32"
43 # Skip cuda graph recording for fast test.
44 "enforce_eager": True,
45
46 - # Required for spec decode.
47 - "use_v2_block_manager": True,
48 -
46 # Print spec metrics.
47 "disable_log_stats": False,
48
@@ -86,9 +83,6 @@ def test_eagle_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
83 # Skip cuda graph recording for fast test.
84 "enforce_eager": True,
85
89 - # Required for spec decode.
90 - "use_v2_block_manager": True,
91 -
86 # Print spec metrics.
87 "disable_log_stats": False,
88
@@ -143,9 +137,6 @@ def test_eagle_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
137 [{
138 "enforce_eager": False,
139
146 - # Required for spec decode.
147 - "use_v2_block_manager": True,
148 -
140 # Print spec metrics.
141 "disable_log_stats": False,
142
@@ -191,9 +182,6 @@ def test_eagle_e2e_greedy_correctness_cuda_graph(
182 # Skip cuda graph recording for fast test.
183 "enforce_eager": True,
184
194 - # Required for spec decode.
195 - "use_v2_block_manager": True,
196 -
185 # Precision
186 "dtype": PRECISION,
187
@@ -235,9 +223,6 @@ def test_eagle_e2e_greedy_correctness_with_preemption(
223 # Skip cuda graph recording for fast test.
224 "enforce_eager": True,
225
238 - # Required for spec decode.
239 - "use_v2_block_manager": True,
240 -
226 # Precision
227 "dtype": PRECISION,
228
@@ -283,9 +268,6 @@ def test_eagle_different_k(vllm_runner, common_llm_kwargs,
268 # Skip cuda graph recording for fast test.
269 "enforce_eager": True,
270
286 - # Required for spec decode.
287 - "use_v2_block_manager": True,
288 -
271 # Precision
272 "dtype": PRECISION,
273
tests/spec_decode/e2e/test_integration.py
-8
@@ -12,8 +12,6 @@ MAIN_MODEL = "JackFram/llama-68m"
12 @pytest.mark.parametrize(
13 "common_llm_kwargs",
14 [{
15 - # Required for spec decode.
16 - "use_v2_block_manager": True,
15
16 # Verify equality when cuda graphs allowed.
17 "enforce_eager": False,
@@ -57,9 +55,6 @@ def test_spec_decode_cuda_graph(vllm_runner, common_llm_kwargs,
55
56 # Skip cuda graph recording for fast test.
57 "enforce_eager": True,
60 -
61 - # Required for spec decode.
62 - "use_v2_block_manager": True,
58 }])
59 @pytest.mark.parametrize("per_test_common_llm_kwargs", [
60 {
@@ -111,9 +106,6 @@ def test_speculative_model_quantization_config(vllm_runner, common_llm_kwargs,
106
107 # Skip cuda graph recording for fast test.
108 "enforce_eager": True,
114 -
115 - # Required for spec decode.
116 - "use_v2_block_manager": True,
109 "speculative_model": "JackFram/llama-68m",
110 "num_speculative_tokens": 3,
111 }])
tests/spec_decode/e2e/test_integration_dist_tp2.py
-6
@@ -17,9 +17,6 @@ from .conftest import run_equality_correctness_test_tp
17 [[
18 # Skip cuda graph recording for fast test.
19 "--enforce-eager",
20 -
21 - # Required for spec decode.
22 - "--use-v2-block-manager",
20 "--tensor-parallel-size",
21 "2"
22 ]])
@@ -74,9 +71,6 @@ def test_target_model_tp_gt_1(common_llm_kwargs, per_test_common_llm_kwargs,
71 [[
72 # Skip cuda graph recording for fast test.
73 "--enforce-eager",
77 -
78 - # Required for spec decode.
79 - "--use_v2_block_manager",
74 "--tensor_parallel_size",
75 "2",
76
tests/spec_decode/e2e/test_integration_dist_tp4.py
-6
@@ -19,9 +19,6 @@ SPEC_MODEL = "JackFram/llama-68m"
19 [[
20 # Skip cuda graph recording for fast test.
21 "--enforce_eager",
22 -
23 - # Required for spec decode.
24 - "--use-v2-block-manager",
22 "--tensor-parallel-size",
23 "4",
24 ]])
@@ -71,9 +68,6 @@ def test_draft_model_tp_lt_target_model_tp4(common_llm_kwargs,
68
69 # Skip cuda graph recording for fast test.
70 "--enforce-eager",
74 -
75 - # Required for spec decode.
76 - "--use-v2-block-manager",
71 "--tensor-parallel-size",
72 "4",
73 ]])
tests/spec_decode/e2e/test_logprobs.py
-14
@@ -14,9 +14,6 @@ from .conftest import run_equality_correctness_test
14
15 # Skip cuda graph recording for fast test.
16 "enforce_eager": True,
17 -
18 - # Required for spec decode.
19 - "use_v2_block_manager": True,
17 }])
18 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
19 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -67,9 +64,6 @@ def test_logprobs_equality(vllm_runner, common_llm_kwargs,
64
65 # Skip cuda graph recording for fast test.
66 "enforce_eager": True,
70 -
71 - # Required for spec decode.
72 - "use_v2_block_manager": True
67 }])
68 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
69 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -119,9 +113,6 @@ def test_logprobs_different_k(vllm_runner, common_llm_kwargs,
113
114 # Skip cuda graph recording for fast test.
115 "enforce_eager": True,
122 -
123 - # Required for spec decode.
124 - "use_v2_block_manager": True
116 }])
117 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
118 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -173,9 +164,6 @@ def test_logprobs_when_skip_speculation(vllm_runner, common_llm_kwargs,
164
165 # Skip cuda graph recording for fast test.
166 "enforce_eager": True,
176 -
177 - # Required for spec decode.
178 - "use_v2_block_manager": True
167 }])
168 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
169 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -251,8 +239,6 @@ def test_logprobs_temp_1(vllm_runner, common_llm_kwargs,
239 "model_name": "JackFram/llama-160m",
240 # Skip cuda graph recording for fast test.
241 "enforce_eager": True,
254 - # Required for spec decode.
255 - "use_v2_block_manager": True,
242 }])
243 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
244 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
tests/spec_decode/e2e/test_medusa_correctness.py
-21
@@ -45,9 +45,6 @@ PRECISION = "float32"
45 # Skip cuda graph recording for fast test.
46 "enforce_eager": True,
47
48 - # Required for spec decode.
49 - "use_v2_block_manager": True,
50 -
48 # Print spec metrics.
49 "disable_log_stats": False,
50
@@ -93,9 +90,6 @@ def test_medusa_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
90 # Skip cuda graph recording for fast test.
91 "enforce_eager": True,
92
96 - # Required for spec decode.
97 - "use_v2_block_manager": True,
98 -
93 # Print spec metrics.
94 "disable_log_stats": False,
95
@@ -151,9 +145,6 @@ def test_medusa_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
145 [{
146 "enforce_eager": False,
147
154 - # Required for spec decode.
155 - "use_v2_block_manager": True,
156 -
148 # Print spec metrics.
149 "disable_log_stats": False,
150
@@ -204,9 +195,6 @@ def test_medusa_e2e_greedy_correctness_cuda_graph(
195 # Skip cuda graph recording for fast test.
196 "enforce_eager": True,
197
207 - # Required for spec decode.
208 - "use_v2_block_manager": True,
209 -
198 # Precision
199 "dtype": PRECISION,
200
@@ -253,9 +241,6 @@ def test_medusa_e2e_greedy_correctness_with_preemption(
241 # Skip cuda graph recording for fast test.
242 "enforce_eager": True,
243
256 - # Required for spec decode.
257 - "use_v2_block_manager": True,
258 -
244 # Precision
245 "dtype": PRECISION,
246
@@ -306,9 +291,6 @@ def test_medusa_different_k(vllm_runner, common_llm_kwargs,
291 # Skip cuda graph recording for fast test.
292 "enforce_eager": True,
293
309 - # Required for spec decode.
310 - "use_v2_block_manager": True,
311 -
294 # Precision
295 "dtype": PRECISION,
296
@@ -356,9 +338,6 @@ def test_medusa_disable_queue(vllm_runner, common_llm_kwargs,
338 # Skip cuda graph recording for fast test.
339 "enforce_eager": True,
340
359 - # Required for spec decode.
360 - "use_v2_block_manager": True,
361 -
341 # Precision
342 "dtype": PRECISION,
343
tests/spec_decode/e2e/test_mlp_correctness.py
-27
@@ -47,9 +47,6 @@ PRECISION = "float32"
47 # Skip cuda graph recording for fast test.
48 "enforce_eager": True,
49
50 - # Required for spec decode.
51 - "use_v2_block_manager": True,
52 -
50 # Print spec metrics.
51 "disable_log_stats": False,
52
@@ -94,9 +91,6 @@ def test_mlp_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
91 # Skip cuda graph recording for fast test.
92 "enforce_eager": True,
93
97 - # Required for spec decode.
98 - "use_v2_block_manager": True,
99 -
94 # Print spec metrics.
95 "disable_log_stats": False,
96
@@ -149,9 +143,6 @@ def test_mlp_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
143 # Skip cuda graph recording for fast test.
144 "enforce_eager": True,
145
152 - # Required for spec decode.
153 - "use_v2_block_manager": True,
154 -
146 # Print spec metrics.
147 "disable_log_stats": False,
148
@@ -195,9 +186,6 @@ def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
186 # Skip cuda graph recording for fast test.
187 "enforce_eager": True,
188
198 - # Required for spec decode.
199 - "use_v2_block_manager": True,
200 -
189 # Print spec metrics.
190 "disable_log_stats": False,
191
@@ -258,9 +246,6 @@ def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
246 # Skip cuda graph recording for fast test.
247 "enforce_eager": True,
248
261 - # Required for spec decode.
262 - "use_v2_block_manager": True,
263 -
249 # Precision
250 "dtype": PRECISION,
251
@@ -311,9 +296,6 @@ def test_mlp_e2e_greedy_correctness_with_preemption(
296 # Skip cuda graph recording for fast test.
297 "enforce_eager": True,
298
314 - # Required for spec decode.
315 - "use_v2_block_manager": True,
316 -
299 # Precision
300 "dtype": PRECISION,
301
@@ -366,9 +348,6 @@ def test_mlp_e2e_greedy_correctness_with_padding(
348 # Skip cuda graph recording for fast test.
349 "enforce_eager": True,
350
369 - # Required for spec decode.
370 - "use_v2_block_manager": True,
371 -
351 # Precision
352 "dtype": PRECISION,
353
@@ -419,9 +398,6 @@ def test_mlp_different_k(vllm_runner, common_llm_kwargs,
398 # Skip cuda graph recording for fast test.
399 "enforce_eager": True,
400
422 - # Required for spec decode.
423 - "use_v2_block_manager": True,
424 -
401 # Precision
402 "dtype": PRECISION,
403
@@ -469,9 +445,6 @@ def test_mlp_disable_queue(vllm_runner, common_llm_kwargs,
445
446 # Skip cuda graph recording for fast test.
447 "enforce_eager": True,
472 -
473 - # Required for spec decode.
474 - "use_v2_block_manager": True,
448 "speculative_model": SPEC_MODEL,
449 }])
450 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
tests/spec_decode/e2e/test_multistep_correctness.py
-36
@@ -55,9 +55,6 @@ from .conftest import (get_output_from_llm_generator,
55
56 # Skip cuda graph recording for fast test.
57 "enforce_eager": True,
58 -
59 - # Required for spec decode.
60 - "use_v2_block_manager": True,
58 }])
59 @pytest.mark.parametrize(
60 "per_test_common_llm_kwargs",
@@ -124,9 +121,6 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
121 # Skip cuda graph recording for fast test.
122 "enforce_eager": True,
123
127 - # Required for spec decode.
128 - "use_v2_block_manager": True,
129 -
124 # Print spec metrics.
125 "disable_log_stats": False,
126 }])
@@ -190,9 +184,6 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
184 # Skip cuda graph recording for fast test.
185 "enforce_eager": True,
186
193 - # Required for spec decode.
194 - "use_v2_block_manager": True,
195 -
187 # Print spec metrics.
188 "disable_log_stats": False,
189 }])
@@ -246,9 +237,6 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
237 [{
238 # Skip cuda graph recording for fast test.
239 "enforce_eager": True,
249 -
250 - # Required for spec decode.
251 - "use_v2_block_manager": True
240 }])
241 @pytest.mark.parametrize(
242 "per_test_common_llm_kwargs",
@@ -303,9 +291,6 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(
291 # Skip cuda graph recording for fast test.
292 "enforce_eager": True,
293
306 - # Required for spec decode.
307 - "use_v2_block_manager": True,
308 -
294 # Print spec metrics.
295 "disable_log_stats": False,
296 }])
@@ -353,9 +338,6 @@ def test_spec_decode_e2e_greedy_correctness_real_model_bs1(
338 # Skip cuda graph recording for fast test.
339 "enforce_eager": True,
340
356 - # Required for spec decode.
357 - "use_v2_block_manager": True,
358 -
341 # Print spec metrics.
342 "disable_log_stats": False,
343 }])
@@ -404,9 +386,6 @@ def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
386
387 # Skip cuda graph recording for fast test.
388 "enforce_eager": True,
407 -
408 - # Required for spec decode.
409 - "use_v2_block_manager": True
389 }])
390 @pytest.mark.parametrize("per_test_common_llm_kwargs", [
391 {
@@ -454,9 +433,6 @@ def test_spec_decode_e2e_greedy_correctness_with_preemption(
433
434 # Skip cuda graph recording for fast test.
435 "enforce_eager": True,
457 -
458 - # Required for spec decode.
459 - "use_v2_block_manager": True
436 }])
437 @pytest.mark.parametrize(
438 "per_test_common_llm_kwargs",
@@ -514,9 +490,6 @@ def test_spec_decode_different_block_size(vllm_runner, common_llm_kwargs,
490
491 # Skip cuda graph recording for fast test.
492 "enforce_eager": True,
517 -
518 - # Required for spec decode.
519 - "use_v2_block_manager": True
493 }])
494 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
495 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -570,9 +543,6 @@ def test_skip_speculation(vllm_runner, common_llm_kwargs,
543
544 # Skip cuda graph recording for fast test.
545 "enforce_eager": True,
573 -
574 - # Required for spec decode.
575 - "use_v2_block_manager": True
546 }])
547 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
548 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -611,9 +581,6 @@ def test_disable_speculation(vllm_runner, common_llm_kwargs,
581
582 # Skip cuda graph recording for fast test.
583 "enforce_eager": True,
614 -
615 - # Required for spec decode.
616 - "use_v2_block_manager": True
584 }])
585 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
586 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -660,9 +627,6 @@ def test_many_k(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
627
628 # Skip cuda graph recording for fast test.
629 "enforce_eager": True,
663 -
664 - # Required for spec decode.
665 - "use_v2_block_manager": True
630 }])
631 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
632 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
tests/spec_decode/e2e/test_ngram_correctness.py
-16
@@ -35,9 +35,6 @@ from .conftest import run_equality_correctness_test
35 # Skip cuda graph recording for fast test.
36 "enforce_eager": True,
37
38 - # Required for spec decode.
39 - "use_v2_block_manager": True,
40 -
38 # Print spec metrics.
39 "disable_log_stats": False,
40 }])
@@ -82,9 +79,6 @@ def test_ngram_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
79 # Skip cuda graph recording for fast test.
80 "enforce_eager": True,
81
85 - # Required for spec decode.
86 - "use_v2_block_manager": True,
87 -
82 # Print spec metrics.
83 "disable_log_stats": False,
84 }])
@@ -145,9 +139,6 @@ def test_ngram_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
139
140 # Skip cuda graph recording for fast test.
141 "enforce_eager": True,
148 -
149 - # Required for spec decode.
150 - "use_v2_block_manager": True
142 }])
143 @pytest.mark.parametrize("per_test_common_llm_kwargs", [
144 {
@@ -195,9 +186,6 @@ def test_ngram_e2e_greedy_correctness_with_preemption(
186
187 # Skip cuda graph recording for fast test.
188 "enforce_eager": True,
198 -
199 - # Required for spec decode.
200 - "use_v2_block_manager": True
189 }])
190 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
191 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -254,9 +242,6 @@ def test_ngram_different_k(vllm_runner, common_llm_kwargs,
242
243 # Skip cuda graph recording for fast test.
244 "enforce_eager": True,
257 -
258 - # Required for spec decode.
259 - "use_v2_block_manager": True
245 }])
246 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
247 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -303,7 +288,6 @@ def test_ngram_disable_queue(vllm_runner, common_llm_kwargs,
288 "enforce_eager": True,
289
290 # Required for spec decode.
306 - "use_v2_block_manager": True,
291 "speculative_model": "[ngram]",
292 "num_speculative_tokens": 5,
293 "ngram_prompt_lookup_max": 3,
tests/spec_decode/e2e/test_seed.py
-3
@@ -17,9 +17,6 @@ SPEC_MODEL = "JackFram/llama-160m"
17 # Skip cuda graph recording for fast test.
18 "enforce_eager": True,
19
20 - # Required for spec decode.
21 - "use_v2_block_manager": True,
22 -
20 # speculative model
21 "speculative_model": "JackFram/llama-160m",
22
tests/utils.py
-9
@@ -678,12 +678,3 @@ def get_client_text_logprob_generations(
678 return [(text_generations, text,
679 (None if x.logprobs is None else x.logprobs.top_logprobs))
680 for completion in completions for x in completion.choices]
681 -
682 -
683 -def check_deprecated_block_manager_usage(test_name: str):
684 - assert envs.VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1 is True, (
685 - f"To allow the use of deprecated BlockSpaceManagerV1, set the "
686 - f"environment variable VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1. "
687 - f"You can run the tests with: "
688 - f"`VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest {test_name}`" #noqa
689 - )
vllm/attention/backends/flash_attn.py
+3 -5
@@ -305,8 +305,6 @@ class FlashAttentionMetadataBuilder(
305 self.runner = input_builder.runner
306 self.sliding_window = input_builder.sliding_window
307 self.block_size = input_builder.block_size
308 - self.use_v2_block_manager = (
309 - input_builder.scheduler_config.use_v2_block_manager)
308
309 def _add_seq_group(
310 self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
@@ -355,9 +353,9 @@ class FlashAttentionMetadataBuilder(
353
354 # Compute slot mapping.
355 is_profile_run = is_block_tables_empty(block_tables)
358 - start_idx = compute_slot_mapping_start_idx(
359 - is_prompt, query_len, context_len, self.sliding_window,
360 - self.use_v2_block_manager)
356 + start_idx = compute_slot_mapping_start_idx(is_prompt, query_len,
357 + context_len,
358 + self.sliding_window)
359 compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
360 seq_len, context_len, start_idx,
361 self.block_size, inter_data.block_tables)
vllm/attention/backends/flashinfer.py
+3 -5
@@ -475,8 +475,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
475
476 self.sliding_window = input_builder.sliding_window
477 self.block_size = input_builder.block_size
478 - self.use_v2_block_manager = (
479 - input_builder.scheduler_config.use_v2_block_manager)
478
479 # Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout
480 # for the precise definition of the following fields.
@@ -542,9 +540,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
540 is_profile_run = is_block_tables_empty(block_tables)
541
542 # Compute slot mapping.
545 - start_idx = compute_slot_mapping_start_idx(
546 - is_prompt, query_len, context_len, self.sliding_window,
547 - self.use_v2_block_manager)
543 + start_idx = compute_slot_mapping_start_idx(is_prompt, query_len,
544 + context_len,
545 + self.sliding_window)
546 compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
547 seq_len, context_len, start_idx,
548 self.block_size, inter_data.block_tables)
vllm/attention/backends/utils.py
+4 -12
@@ -38,18 +38,12 @@ def is_block_tables_empty(block_tables: Union[None, Dict]):
38
39
40 def compute_slot_mapping_start_idx(is_prompt: bool, query_len: int,
41 - context_len: int, sliding_window: int,
42 - use_v2_block_manager: bool):
41 + context_len: int, sliding_window: int):
42 """
43 Compute the start index of slot mapping.
44 """
45 start_idx = 0
46 if is_prompt and sliding_window is not None:
48 - assert use_v2_block_manager or context_len == 0, (
49 - "Prefix caching is currently not supported with "
50 - "sliding window attention in V1 block manager")
51 - # When prefill, we use it to not write slots to kv cache
52 - # to save memory.
47 start_idx = max(0, query_len - sliding_window)
48 return start_idx
49
@@ -138,8 +132,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
132
133 self.sliding_window = input_builder.sliding_window
134 self.block_size = input_builder.block_size
141 - self.use_v2_block_manager = (
142 - input_builder.scheduler_config.use_v2_block_manager)
135
136 def _add_seq_group(
137 self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
@@ -180,9 +172,9 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
172
173 # Compute slot mapping.
174 is_profile_run = is_block_tables_empty(block_tables)
183 - start_idx = compute_slot_mapping_start_idx(
184 - is_prompt, query_len, context_len, self.sliding_window,
185 - self.use_v2_block_manager)
175 + start_idx = compute_slot_mapping_start_idx(is_prompt, query_len,
176 + context_len,
177 + self.sliding_window)
178 compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
179 seq_len, context_len, start_idx,
180 self.block_size, inter_data.block_tables)
vllm/commit_id.py new
+1
@@ -0,0 +1 @@
1 +__commit__ = "93ec62b8556e279d2c050bdc1c3247831bd39466"
vllm/config.py
-24
@@ -949,7 +949,6 @@ class SchedulerConfig:
949 iteration.
950 max_model_len: Maximum length of a sequence (including prompt
951 and generated text).
952 - use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
952 num_lookahead_slots: The number of slots to allocate per sequence per
953 step, beyond the known token ids. This is used in speculative
954 decoding to store KV activations of tokens which may or may not be
@@ -976,7 +975,6 @@ class SchedulerConfig:
975 max_num_batched_tokens: Optional[int],
976 max_num_seqs: int,
977 max_model_len: int,
979 - use_v2_block_manager: bool = True,
978 num_lookahead_slots: int = 0,
979 delay_factor: float = 0.0,
980 enable_chunked_prefill: bool = False,
@@ -1026,7 +1024,6 @@ class SchedulerConfig:
1024
1025 self.max_num_seqs = max_num_seqs
1026 self.max_model_len = max_model_len
1029 - self.use_v2_block_manager = use_v2_block_manager
1027 self.num_lookahead_slots = num_lookahead_slots
1028 self.delay_factor = delay_factor
1029 self.chunked_prefill_enabled = enable_chunked_prefill
@@ -1067,18 +1064,6 @@ class SchedulerConfig:
1064 f"({self.num_scheduler_steps}) must be greater than or "
1065 "equal to 1.")
1066
1070 - if (not self.use_v2_block_manager \
1071 - and not envs.VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1):
1072 - raise ValueError(
1073 - "The use of BlockSpaceManagerV1 is deprecated and will "
1074 - "be removed in a future release. Please switch to "
1075 - "BlockSpaceManagerV2 by setting --use-v2-block-manager to "
1076 - "True. If you wish to suppress this error temporarily, "
1077 - "you can set the environment variable "
1078 - "`VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1. If your use "
1079 - "case is not supported in BlockSpaceManagerV2, please "
1080 - "file an issue with detailed information.")
1081 -
1067 @property
1068 def is_multi_step(self) -> bool:
1069 return self.num_scheduler_steps > 1
@@ -1137,7 +1122,6 @@ class SpeculativeConfig:
1122 speculative_disable_mqa_scorer: Optional[bool],
1123 speculative_max_model_len: Optional[int],
1124 enable_chunked_prefill: bool,
1140 - use_v2_block_manager: bool,
1125 disable_log_stats: bool,
1126 speculative_disable_by_batch_size: Optional[int],
1127 ngram_prompt_lookup_max: Optional[int],
@@ -1178,9 +1162,6 @@ class SpeculativeConfig:
1162 enable_chunked_prefill (bool): Whether vLLM is configured to use
1163 chunked prefill or not. Used for raising an error since its not
1164 yet compatible with spec decode.
1181 - use_v2_block_manager (bool): Whether vLLM is configured to use the
1182 - v2 block manager or not. Used for raising an error since the v2
1183 - block manager is required with spec decode.
1165 speculative_disable_by_batch_size (Optional[int]): Disable
1166 speculative decoding for new incoming requests when the number
1167 of enqueue requests is larger than this value, if provided.
@@ -1231,11 +1212,6 @@ class SpeculativeConfig:
1212 "Speculative decoding and chunked prefill are "
1213 f"currently mutually exclusive ({enable_chunked_prefill=}).")
1214
1234 - if not use_v2_block_manager:
1235 - raise ValueError(
1236 - "Speculative decoding requires usage of the V2 "
1237 - "block manager. Enable it with --use-v2-block-manager.")
1238 -
1215 # TODO: The user should be able to specify revision/max model len
1216 # for the draft model. It is not currently supported.
1217 draft_revision = None
vllm/core/block/utils.py
+1 -23
@@ -4,28 +4,6 @@ from vllm.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
4 STR_NOT_IMPL_ENC_DEC_SWA)
5
6
7 -def _get_block_mgr_sliding_window_attr(block_mgr):
8 - '''
9 - BlockManagerV1 and BlockManagerV2 have slightly different
10 - members related to sliding window attention (SWA). This
11 - function extracts the appropriate member to use for determining
12 - whether SWA is enabled.
13 -
14 - Arguments:
15 -
16 - * block_mgr: BlockManagerV1 or BlockManagerV2 instance
17 - '''
18 -
19 - if hasattr(block_mgr, 'block_sliding_window'):
20 - return block_mgr.block_sliding_window
21 - if hasattr(block_mgr, 'max_block_sliding_window'):
22 - return block_mgr.max_block_sliding_window
23 -
24 - raise AttributeError("Block manager instance has neither " + \
25 - "block_sliding_window nor " + \
26 - "max_block_sliding_window attributes.")
27 -
28 -
7 def check_no_caching_or_swa_for_blockmgr_encdec(
8 block_mgr, seq_group: SequenceGroup) -> None:
9 '''
@@ -41,7 +19,7 @@ def check_no_caching_or_swa_for_blockmgr_encdec(
19 '''
20
21 if seq_group.is_encoder_decoder():
44 - if _get_block_mgr_sliding_window_attr(block_mgr) is not None:
22 + if block_mgr.max_block_sliding_window is not None:
23 raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_SWA)
24
25 if block_mgr.enable_caching:
vllm/core/block_manager.py renamed
+1 -1
@@ -17,7 +17,7 @@ SeqId = int
17 EncoderSeqId = str
18
19
20 -class BlockSpaceManagerV2(BlockSpaceManager):
20 +class SelfAttnBlockSpaceManager(BlockSpaceManager):
21 """BlockSpaceManager which manages the allocation of KV cache.
22
23 It owns responsibility for allocation, swapping, allocating memory for
vllm/core/block_manager_v1.py deleted
-743
@@ -1,743 +0,0 @@
1 -"""A block manager that manages token blocks."""
2 -import math
3 -from abc import ABC, abstractmethod
4 -from itertools import count, takewhile
5 -from os.path import commonprefix
6 -from typing import Dict, List, Optional
7 -from typing import Sequence as GenericSequence
8 -from typing import Set, Tuple
9 -
10 -from vllm.block import BlockTable, PhysicalTokenBlock
11 -from vllm.core.block.common import CacheMetricData
12 -from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
13 -from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
14 -from vllm.core.interfaces import AllocStatus, BlockSpaceManager
15 -from vllm.logger import init_logger
16 -from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
17 -from vllm.utils import Device
18 -
19 -logger = init_logger(__name__)
20 -
21 -
22 -class BlockAllocatorBase(ABC):
23 - """Manages free physical token blocks for a device.
24 -
25 - The allocator maintains a list of free blocks and allocates a block when
26 - requested. When a block is freed, its reference count is decremented. If
27 - the reference count becomes zero, the block is added back to the free list.
28 - """
29 -
30 - @abstractmethod
31 - def __init__(self,
32 - device: Device,
33 - block_size: int,
34 - num_blocks: int,
35 - eviction_policy: EvictionPolicy = EvictionPolicy.LRU):
36 - pass
37 -
38 - @abstractmethod
39 - def allocate(self,
40 - block_hash: Optional[int] = None,
41 - num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
42 - pass
43 -
44 - @abstractmethod
45 - def free(self, block: PhysicalTokenBlock) -> None:
46 - pass
47 -
48 - @abstractmethod
49 - def get_num_free_blocks(self) -> int:
50 - pass
51 -
52 - @abstractmethod
53 - def get_num_total_blocks(self) -> int:
54 - pass
55 -
56 - @abstractmethod
57 - def contains_block(self, block_hash: int) -> bool:
58 - pass
59 -
60 - @abstractmethod
61 - def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
62 - pass
63 -
64 - @abstractmethod
65 - def get_prefix_cache_hit_rate(self) -> float:
66 - """Prefix cache hit rate. -1 means not supported or disabled."""
67 - pass
68 -
69 -
70 -class CachedBlockAllocator(BlockAllocatorBase):
71 - """Manages free physical token blocks for a device.
72 -
73 - The allocator maintains a list of free blocks and allocates a block when
74 - requested. When a block is freed, its reference count is decremented. If
75 - the reference count becomes zero, the block is added back to the free list.
76 - """
77 -
78 - def __init__(self,
79 - device: Device,
80 - block_size: int,
81 - num_blocks: int,
82 - eviction_policy: EvictionPolicy = EvictionPolicy.LRU) -> None:
83 - self.device = device
84 - self.block_size = block_size
85 - self.num_blocks = num_blocks
86 -
87 - self.current_num_blocks = 0
88 - self.cached_blocks: Dict[int, PhysicalTokenBlock] = {}
89 -
90 - self.evictor: Evictor = make_evictor(eviction_policy)
91 -
92 - self.default_hash_ctr = count()
93 -
94 - self.cache_metric_data = CacheMetricData()
95 -
96 - def allocate_block(self, block_hash: int,
97 - num_hashed_tokens: int) -> PhysicalTokenBlock:
98 - if self.current_num_blocks == self.num_blocks:
99 - block = self.evictor.evict()
100 - block.block_hash = block_hash
101 - block.num_hashed_tokens = num_hashed_tokens
102 - return block
103 - block = PhysicalTokenBlock(device=self.device,
104 - block_number=self.current_num_blocks,
105 - block_size=self.block_size,
106 - block_hash=block_hash,
107 - num_hashed_tokens=num_hashed_tokens)
108 - self.current_num_blocks += 1
109 - return block
110 -
111 - def allocate(self,
112 - block_hash: Optional[int] = None,
113 - num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
114 - if block_hash is None:
115 - block_hash = next(self.default_hash_ctr)
116 -
117 - if block_hash in self.evictor:
118 - assert block_hash not in self.cached_blocks
119 - block = self.evictor.remove(block_hash)
120 - assert block.ref_count == 0
121 - self.cached_blocks[block_hash] = block
122 -
123 - if block_hash in self.cached_blocks:
124 - self.cache_metric_data.query(hit=True)
125 - else:
126 - self.cache_metric_data.query(hit=False)
127 - self.cached_blocks[block_hash] = self.allocate_block(
128 - block_hash, num_hashed_tokens)
129 - block = self.cached_blocks[block_hash]
130 - assert block.block_hash == block_hash
131 - block.ref_count += 1
132 - return block
133 -
134 - def free(self, block: PhysicalTokenBlock) -> None:
135 - if block.ref_count == 0:
136 - raise ValueError(f"Double free! {block} is already freed.")
137 - block.ref_count -= 1
138 - if block.ref_count == 0:
139 - assert block.block_hash not in self.evictor
140 - self.evictor.add(block)
141 -
142 - # Remove the block from the cached_blocks
143 - del self.cached_blocks[block.block_hash]
144 -
145 - def get_num_free_blocks(self) -> int:
146 - return (self.num_blocks - self.current_num_blocks +
147 - self.evictor.num_blocks)
148 -
149 - def get_num_total_blocks(self) -> int:
150 - return self.num_blocks
151 -
152 - def contains_block(self, block_hash: int) -> bool:
153 - return block_hash in self.cached_blocks or block_hash in self.evictor
154 -
155 - def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
156 - # Update the hash of block and the cached_blocks dictionary.
157 - assert not self.contains_block(block_hash)
158 - old_hash = block.block_hash
159 - block.block_hash = block_hash
160 - del self.cached_blocks[old_hash]
161 - self.cached_blocks[block_hash] = block
162 -
163 - def get_prefix_cache_hit_rate(self) -> float:
164 - return self.cache_metric_data.get_hit_rate()
165 -
166 -
167 -class UncachedBlockAllocator(BlockAllocatorBase):
168 - """Manages free physical token blocks for a device.
169 -
170 - The allocator maintains a list of free blocks and allocates a block when
171 - requested. When a block is freed, its reference count is decremented. If
172 - the reference count becomes zero, the block is added back to the free list.
173 - """
174 -
175 - def __init__(
176 - self,
177 - device: Device,
178 - block_size: int,
179 - num_blocks: int,
180 - ) -> None:
181 - self.device = device
182 - self.block_size = block_size
183 - self.num_blocks = num_blocks
184 -
185 - # Initialize the free blocks.
186 - self.free_blocks: List[PhysicalTokenBlock] = []
187 - for i in range(num_blocks):
188 - block = PhysicalTokenBlock(device=device,
189 - block_number=i,
190 - block_size=block_size,
191 - block_hash=-1,
192 - num_hashed_tokens=0)
193 - self.free_blocks.append(block)
194 -
195 - def allocate(self,
196 - block_hash: Optional[int] = None,
197 - num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
198 - if not self.free_blocks:
199 - raise ValueError("Out of memory! No free blocks are available.")
200 - block = self.free_blocks.pop()
201 - block.ref_count = 1
202 - return block
203 -
204 - def free(self, block: PhysicalTokenBlock) -> None:
205 - if block.ref_count == 0:
206 - raise ValueError(f"Double free! {block} is already freed.")
207 - block.ref_count -= 1
208 - if block.ref_count == 0:
209 - self.free_blocks.append(block)
210 -
211 - def get_num_free_blocks(self) -> int:
212 - return len(self.free_blocks)
213 -
214 - def get_num_total_blocks(self) -> int:
215 - return self.num_blocks
216 -
217 - def contains_block(self, block_hash: int) -> bool:
218 - raise NotImplementedError(
219 - "Invalid codepath for uncached block allocator.")
220 -
221 - def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
222 - raise NotImplementedError(
223 - "Invalid codepath for uncached block allocator.")
224 -
225 - def get_prefix_cache_hit_rate(self) -> float:
226 - return -1
227 -
228 -
229 -class BlockSpaceManagerV1(BlockSpaceManager):
230 - """Manages the mapping between logical and physical token blocks."""
231 -
232 - def __init__(
233 - self,
234 - block_size: int,
235 - num_gpu_blocks: int,
236 - num_cpu_blocks: int,
237 - watermark: float = 0.01,
238 - sliding_window: Optional[int] = None,
239 - enable_caching: bool = False,
240 - ) -> None:
241 - self.block_size = block_size
242 - self.num_total_gpu_blocks = num_gpu_blocks
243 - self.num_total_cpu_blocks = num_cpu_blocks
244 -
245 - if enable_caching and sliding_window is not None:
246 - raise NotImplementedError(
247 - "Sliding window is not allowed with prefix caching enabled!")
248 -
249 - self.block_sliding_window = None
250 - if sliding_window is not None:
251 - # Round up to nearest block size to regularize sliding window
252 - # allocation sizes.
253 - self.block_sliding_window = math.ceil(sliding_window / block_size)
254 -
255 - self.watermark = watermark
256 - assert watermark >= 0.0
257 -
258 - self.enable_caching = enable_caching
259 -
260 - self.watermark_blocks = int(watermark * num_gpu_blocks)
261 -
262 - if self.enable_caching:
263 - logger.info("Automatic prefix caching is enabled.")
264 - self.gpu_allocator: BlockAllocatorBase = CachedBlockAllocator(
265 - Device.GPU, block_size, num_gpu_blocks)
266 - self.cpu_allocator: BlockAllocatorBase = CachedBlockAllocator(
267 - Device.CPU, block_size, num_cpu_blocks)
268 - else:
269 - self.gpu_allocator = UncachedBlockAllocator(
270 - Device.GPU, block_size, num_gpu_blocks)
271 - self.cpu_allocator = UncachedBlockAllocator(
272 - Device.CPU, block_size, num_cpu_blocks)
273 - # Mapping: seq_id -> BlockTable.
274 - self.block_tables: Dict[int, BlockTable] = {}
275 -
276 - # Mapping: req_id -> BlockTable
277 - # Note that each SequenceGroup has a unique
278 - # request ID
279 - self.cross_block_tables: Dict[str, BlockTable] = {}
280 -
281 - def _get_seq_num_required_blocks(self, seq: Optional[Sequence]) -> int:
282 - return 0 if seq is None else seq.n_blocks
283 -
284 - def can_allocate(self,
285 - seq_group: SequenceGroup,
286 - num_lookahead_slots: int = 0) -> AllocStatus:
287 - # FIXME(woosuk): Here we assume that all sequences in the group share
288 - # the same prompt. This may not be true for preempted sequences.
289 -
290 - assert (num_lookahead_slots == 0
291 - ), "lookahead allocation not supported in BlockSpaceManagerV1"
292 -
293 - check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
294 -
295 - self_num_required_blocks = self._get_seq_num_required_blocks(
296 - seq_group.get_seqs(status=SequenceStatus.WAITING)[0])
297 - cross_num_required_blocks = self._get_seq_num_required_blocks(
298 - seq_group.get_encoder_seq())
299 - num_required_blocks = self_num_required_blocks + \
300 - cross_num_required_blocks
301 -
302 - if self.block_sliding_window is not None:
303 -
304 - num_required_blocks = min(num_required_blocks,
305 - self.block_sliding_window)
306 - num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
307 -
308 - # Use watermark to avoid frequent cache eviction.
309 - if (self.num_total_gpu_blocks - num_required_blocks <
310 - self.watermark_blocks):
311 - return AllocStatus.NEVER
312 - if num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks:
313 - return AllocStatus.OK
314 - else:
315 - return AllocStatus.LATER
316 -
317 - def _allocate_sequence(self, \
318 - seq: Optional[Sequence], \
319 - ref_count: int, \
320 - is_encoder_decoder: bool = True) -> BlockTable:
321 - # Allocate new physical token blocks that will store the prompt tokens.
322 - num_prompt_blocks = self._get_seq_num_required_blocks(seq)
323 -
324 - block_table: BlockTable = BlockTable()
325 - assert seq is not None
326 - for logical_idx in range(num_prompt_blocks):
327 - if (self.block_sliding_window is not None
328 - and logical_idx >= self.block_sliding_window):
329 - block = block_table[logical_idx % self.block_sliding_window]
330 - # Set the reference counts of the token blocks.
331 - block.ref_count = ref_count
332 - elif not is_encoder_decoder and self.enable_caching:
333 - block = self.gpu_allocator.allocate(
334 - seq.hash_of_block(logical_idx),
335 - seq.num_hashed_tokens_of_block(logical_idx))
336 - else:
337 - block = self.gpu_allocator.allocate()
338 - # Set the reference counts of the token blocks.
339 - block.ref_count = ref_count
340 - block_table.append(block)
341 -
342 - return block_table
343 -
344 - def allocate(self, seq_group: SequenceGroup) -> None:
345 - is_encoder_decoder = seq_group.is_encoder_decoder()
346 - check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
347 -
348 - # Allocate decoder sequences
349 - #
350 - # NOTE: Here we assume that all sequences in the group have the same
351 - # decoder prompt.
352 - wait_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
353 - seq = wait_seqs[0]
354 - block_table: BlockTable = \
355 - self._allocate_sequence(seq,
356 - seq_group.num_seqs(),
357 - is_encoder_decoder)
358 -
359 - # Assign the self-attention block tables for each sequence.
360 - if len(wait_seqs) == 1:
361 - self.block_tables[seq.seq_id] = block_table
362 - else:
363 - for seq in wait_seqs:
364 - self.block_tables[seq.seq_id] = block_table.copy()
365 -
366 - # Allocate encoder sequence
367 - if is_encoder_decoder:
368 - # A SequenceGroup has only a single encoder sequence (at most),
369 - # thus allocate with a ref count of 1
370 - block_table = self._allocate_sequence(seq_group.get_encoder_seq(),
371 - 1, is_encoder_decoder)
372 - # Assign the cross-attention block table for the SequenceGroup.
373 - self.cross_block_tables[seq_group.request_id] = block_table
374 -
375 - def can_append_slots(self,
376 - seq_group: SequenceGroup,
377 - num_lookahead_slots: int = 0) -> bool:
378 - assert (num_lookahead_slots == 0
379 - ), "lookahead allocation not supported in BlockSpaceManagerV1"
380 -
381 - # Simple heuristic: If there is at least one free block
382 - # for each sequence, we can append.
383 - num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
384 - num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
385 - return num_seqs <= num_free_gpu_blocks
386 -
387 - def _promote_last_block(
388 - self,
389 - seq: Sequence,
390 - last_block: PhysicalTokenBlock,
391 - ) -> PhysicalTokenBlock:
392 - assert self.enable_caching
393 -
394 - # Compute a new hash for the block so that it can be shared by other
395 - # Sequences
396 - new_hash = seq.hash_of_block(seq.n_blocks - 1)
397 -
398 - # if new_hash is already in the cached table, then free last_block
399 - # and return the cached version
400 - if self.gpu_allocator.contains_block(new_hash):
401 - self.gpu_allocator.free(last_block)
402 - return self.gpu_allocator.allocate(new_hash)
403 - else:
404 - self.gpu_allocator.update_hash(new_hash, last_block)
405 - return last_block
406 -
407 - def _is_last_block_full(
408 - self,
409 - seq: Sequence,
410 - ) -> bool:
411 - token_ids_len = seq.data.get_len()
412 - return token_ids_len > 0 and token_ids_len % seq.block_size == 0
413 -
414 - def _maybe_promote_last_block(
415 - self,
416 - seq: Sequence,
417 - last_block: PhysicalTokenBlock,
418 - ) -> PhysicalTokenBlock:
419 - if self._is_last_block_full(seq):
420 - return self._promote_last_block(seq, last_block)
421 - else:
422 - return last_block
423 -
424 - def _allocate_last_physical_block(
425 - self,
426 - seq: Sequence,
427 - ) -> PhysicalTokenBlock:
428 - # Called before a new block is appended.
429 - # This is in charge of allocating a new physical block (to be appended).
430 -
431 - # None if the last block is not full. Otherwise, we set it to the
432 - # content hash.
433 - if not self.enable_caching:
434 - return self.gpu_allocator.allocate()
435 - block_hash: Optional[int] = None
436 - n_blocks = seq.n_blocks
437 - if (self._is_last_block_full(seq)):
438 - block_hash = seq.hash_of_block(n_blocks - 1)
439 - num_hashed_tokens = seq.num_hashed_tokens_of_block(n_blocks - 1)
440 -
441 - # num_hashed_tokens is used to compute future hashes
442 - # (e.g. in the hashing function, it is used to ask the sequence for
443 - # prefix tokens)
444 - new_block = self.gpu_allocator.allocate(block_hash, num_hashed_tokens)
445 -
446 - # If the block_hash is None, then the block is not full.
447 - # If the block is not full, then we expect it to have a refcount of 1.
448 - if block_hash is None:
449 - assert new_block.ref_count == 1
450 - return new_block
451 -
452 - def append_slots(
453 - self,
454 - seq: Sequence,
455 - num_lookahead_slots: int = 0,
456 - ) -> List[Tuple[int, int]]:
457 - """Allocate a physical slot for a new token."""
458 - n_blocks = seq.n_blocks
459 - block_table = self.block_tables[seq.seq_id]
460 - # If we need to allocate a new physical block
461 - if len(block_table) < n_blocks:
462 - # Currently this code only supports adding one physical block
463 - assert len(block_table) == n_blocks - 1
464 -
465 - if (self.block_sliding_window
466 - and len(block_table) >= self.block_sliding_window):
467 - # reuse a block
468 - block_table.append(block_table[len(block_table) %
469 - self.block_sliding_window])
470 - else:
471 - # The sequence hash a new logical block.
472 - # Allocate a new physical block.
473 - new_block = self._allocate_last_physical_block(seq)
474 - block_table.append(new_block)
475 - return []
476 -
477 - # We want to append the token to the last physical block.
478 - last_block = block_table[-1]
479 - assert last_block.device == Device.GPU
480 - if last_block.ref_count == 1:
481 - # Not shared with other sequences. Appendable.
482 - if self.enable_caching:
483 - # If the last block is now complete, we may reuse an old block
484 - # to save memory.
485 - maybe_new_block = self._maybe_promote_last_block(
486 - seq, last_block)
487 - block_table[-1] = maybe_new_block
488 - return []
489 - else:
490 - # The last block is shared with other sequences.
491 - # Copy on Write: Allocate a new block and copy the tokens.
492 - new_block = self._allocate_last_physical_block(seq)
493 -
494 - block_table[-1] = new_block
495 - self.gpu_allocator.free(last_block)
496 - return [(last_block.block_number, new_block.block_number)]
497 -
498 - def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
499 - # NOTE: fork does not allocate a new physical block.
500 - # Thus, it is always safe from OOM.
501 - if parent_seq.seq_id not in self.block_tables:
502 - # Parent sequence has either been freed or never existed.
503 - return
504 - src_block_table = self.block_tables[parent_seq.seq_id]
505 - self.block_tables[child_seq.seq_id] = src_block_table.copy()
506 -
507 - # When using a sliding window, blocks will be eventually reused.
508 - # In this case the block tables will contain repeated blocks.
509 - # When forking, we must make sure that each block's `ref_count`
510 - # is only incremented by one, so we deduplicate them by wrapping
511 - # them in a set.
512 - for block in set(src_block_table):
513 - block.ref_count += 1
514 -
515 - def _get_physical_blocks(
516 - self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
517 -
518 - # NOTE: Here, we assume that the physical blocks are only shared by
519 - # the sequences in the same group.
520 - request_id = seq_group.request_id
521 - blocks: Set[PhysicalTokenBlock] = set()
522 - for seq in seq_group.get_seqs():
523 - if seq.is_finished():
524 - continue
525 - blocks.update(self.block_tables[seq.seq_id])
526 - # Cross-attention blocks
527 - if seq_group.is_encoder_decoder():
528 - blocks.update(self.cross_block_tables[request_id])
529 - return list(blocks)
530 -
531 - def can_swap_in(self,
532 - seq_group: SequenceGroup,
533 - num_lookahead_slots: int = 0) -> AllocStatus:
534 - assert (num_lookahead_slots == 0
535 - ), "BlockSpaceManagerV1 does not support lookahead allocation"
536 -
537 - blocks = self._get_physical_blocks(seq_group)
538 - num_swapped_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
539 - if seq_group.is_encoder_decoder():
540 - num_swapped_seqs += 1
541 - num_free_blocks = self.gpu_allocator.get_num_free_blocks()
542 - # NOTE: Conservatively, we assume that every sequence will allocate
543 - # at least one free block right after the swap-in.
544 - # NOTE: This should match the logic in can_append_slot().
545 - num_required_blocks = len(blocks) + num_swapped_seqs
546 - if self.gpu_allocator.get_num_total_blocks() < num_required_blocks:
547 - return AllocStatus.NEVER
548 - elif num_free_blocks - num_required_blocks >= self.watermark_blocks:
549 - return AllocStatus.OK
550 - else:
551 - return AllocStatus.LATER
552 -
553 - def _swap_block_table(
554 - self, block_table: BlockTable, src_allocator: BlockAllocatorBase,
555 - dest_allocator: BlockAllocatorBase,
556 - mapping: Dict[PhysicalTokenBlock,
557 - PhysicalTokenBlock]) -> BlockTable:
558 - new_block_table: BlockTable = BlockTable()
559 -
560 - for from_block in block_table:
561 - if from_block in mapping:
562 - to_block = mapping[from_block]
563 - to_block.ref_count += 1
564 - else:
565 - to_block = dest_allocator.allocate(
566 - from_block.block_hash, from_block.num_hashed_tokens)
567 - mapping[from_block] = to_block
568 - new_block_table.append(to_block)
569 - # Free the source block swapped in to destination.
570 - src_allocator.free(from_block)
571 -
572 - return new_block_table
573 -
574 - def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
575 -
576 - request_id = seq_group.request_id
577 -
578 - # CPU block -> GPU block.
579 - # dict is efficient in lookup `if cpu_block in mapping`
580 - mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
581 - for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
582 - self.block_tables[seq.seq_id] = \
583 - self._swap_block_table(self.block_tables[seq.seq_id],
584 - self.cpu_allocator, self.gpu_allocator,
585 - mapping)
586 -
587 - if seq_group.is_encoder_decoder():
588 - self.cross_block_tables[request_id] = \
589 - self._swap_block_table(self.cross_block_tables[request_id],
590 - self.cpu_allocator,
591 - self.gpu_allocator,
592 - mapping)
593 -
594 - return [(cpu_block.block_number, gpu_block.block_number)
595 - for cpu_block, gpu_block in mapping.items()]
596 -
597 - def can_swap_out(self, seq_group: SequenceGroup) -> bool:
598 - blocks = self._get_physical_blocks(seq_group)
599 - return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
600 -
601 - def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
602 - request_id = seq_group.request_id
603 -
604 - # GPU block -> CPU block.
605 - # dict is efficient in lookup `if gpu_block in mapping`
606 - mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
607 - for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
608 - self.block_tables[seq.seq_id] = \
609 - self._swap_block_table(self.block_tables[seq.seq_id],
610 - self.gpu_allocator, self.cpu_allocator,
611 - mapping)
612 -
613 - if seq_group.is_encoder_decoder():
614 - self.cross_block_tables[request_id] = \
615 - self._swap_block_table(self.cross_block_tables[request_id],
616 - self.gpu_allocator,
617 - self.cpu_allocator,
618 - mapping)
619 -
620 - return [(cpu_block.block_number, gpu_block.block_number)
621 - for cpu_block, gpu_block in mapping.items()]
622 -
623 - def _free_block_table(self, block_table: BlockTable) -> None:
624 - # when using a sliding window, each seq will only use up
625 - # to `self.block_sliding_window` blocks. When freeing
626 - # the block table, we must make sure to not free blocks more
627 - # than once. If no sliding window is used, there is no block
628 - # reuse in the block table, so we must free all blocks.
629 - blocks_to_free = (block_table[-self.block_sliding_window:]
630 - if self.block_sliding_window is not None else
631 - block_table)
632 - for block in set(blocks_to_free):
633 - if block.device == Device.GPU:
634 - self.gpu_allocator.free(block)
635 - else:
636 - self.cpu_allocator.free(block)
637 -
638 - def free(self, seq: Sequence) -> None:
639 - if seq.seq_id not in self.block_tables:
640 - # Already freed or haven't been scheduled yet.
641 - return
642 - block_table = self.block_tables[seq.seq_id]
643 - self._free_block_table(block_table)
644 - del self.block_tables[seq.seq_id]
645 -
646 - def free_cross(self, seq_group: SequenceGroup) -> None:
647 - if seq_group.request_id not in self.cross_block_tables:
648 - # Already freed or hasn't ben scheduled yet.
649 - return
650 - block_table = self.cross_block_tables[seq_group.request_id]
651 - self._free_block_table(block_table)
652 - del self.cross_block_tables[seq_group.request_id]
653 -
654 - def reset(self) -> None:
655 - # Free decoder block tables
656 - for block_table in self.block_tables.values():
657 - self._free_block_table(block_table)
658 - self.block_tables.clear()
659 - # Free cross-attention block tables
660 - for block_table in self.cross_block_tables.values():
661 - self._free_block_table(block_table)
662 - self.cross_block_tables.clear()
663 -
664 - def get_block_table(self, seq: Sequence) -> List[int]:
665 - return self.block_tables[seq.seq_id].ids()
666 -
667 - def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
668 - block_table = self.cross_block_tables[seq_group.request_id]
669 - return [block.block_number for block in block_table]
670 -
671 - def get_num_free_gpu_blocks(self) -> int:
672 - return self.gpu_allocator.get_num_free_blocks()
673 -
674 - def get_num_free_cpu_blocks(self) -> int:
675 - return self.cpu_allocator.get_num_free_blocks()
676 -
677 - def access_all_blocks_in_seq(
678 - self,
679 - seq: Sequence,
680 - access_time: float,
681 - ) -> None:
682 - if self.enable_caching:
683 - # Update the last accessed time of all the blocks accessed
684 - # in this step.
685 - block_table = self.block_tables[seq.seq_id]
686 - for block in block_table:
687 - block.last_accessed = access_time
688 -
689 - def compute_full_blocks_in_seq(self, seq: Sequence, token_chunk_size: int):
690 - if seq.seq_id not in self.block_tables:
691 - return
692 -
693 - # When chunked prefill is enabled, the computed full blocks
694 - # should be calculated based on the number of computed tokens.
695 - max_computed_tokens = (seq.data.get_num_computed_tokens() +
696 - token_chunk_size)
697 - computed_full_blocks = max_computed_tokens // self.block_size
698 -
699 - block_table = self.block_tables[seq.seq_id]
700 - if computed_full_blocks == 0:
701 - return
702 - for i in reversed(range(computed_full_blocks)):
703 - if block_table[i].computed:
704 - break
705 - block_table[i].computed = True
706 -
707 - def get_all_computed_blocks(self, seq: Sequence) -> List[int]:
708 - if seq.seq_id not in self.block_tables:
709 - return []
710 - block_table = self.block_tables[seq.seq_id]
711 - # NOTE We exclude the last block to avoid the case where the entire
712 - # prompt is cached. This would cause erroneous behavior in model
713 - # runner.
714 - return [
715 - b.block_number
716 - for b in takewhile(lambda b: b.computed, block_table[:-1])
717 - ]
718 -
719 - def get_common_computed_block_ids(
720 - self, seqs: List[Sequence]) -> GenericSequence[int]:
721 - """Return the block ids that are common for a given sequence group.
722 -
723 - Used in prefill (can skip prefill of some blocks).
724 - """
725 - # Can return non-empty result only with prefix caching enabled.
726 - if not self.enable_caching:
727 - return []
728 -
729 - ids_list = [self.get_all_computed_blocks(seq) for seq in seqs]
730 - return commonprefix([ids for ids in ids_list if ids != []])
731 -
732 - def mark_blocks_as_computed(self, seq_group: SequenceGroup,
733 - token_chunk_size: int):
734 - if self.enable_caching:
735 - for seq in seq_group.get_seqs():
736 - self.compute_full_blocks_in_seq(seq, token_chunk_size)
737 -
738 - def get_prefix_cache_hit_rate(self, device: Device) -> float:
739 - if device == Device.GPU:
740 - return self.gpu_allocator.get_prefix_cache_hit_rate()
741 - if device == Device.CPU:
742 - return self.cpu_allocator.get_prefix_cache_hit_rate()
743 - raise ValueError(f"Invalid device: {device}")
vllm/core/interfaces.py
+3 -7
@@ -28,13 +28,9 @@ class BlockSpaceManager(ABC):
28 def get_block_space_manager_class(version: str):
29 version = version.lower()
30
31 - if version == "v1":
32 - from vllm.core.block_manager_v1 import BlockSpaceManagerV1
33 - return BlockSpaceManagerV1
34 -
35 - if version == "v2":
36 - from vllm.core.block_manager_v2 import BlockSpaceManagerV2
37 - return BlockSpaceManagerV2
31 + if version == "selfattn":
32 + from vllm.core.block_manager import SelfAttnBlockSpaceManager
33 + return SelfAttnBlockSpaceManager
34
35 if version == "placeholder":
36 from vllm.core.placeholder_block_space_manager import (
vllm/core/scheduler.py
+1 -3
@@ -312,9 +312,7 @@ class Scheduler:
312 # LoRAs. This should be improved in the future.
313 self.lora_config = lora_config
314
315 - version = "v1"
316 - if self.scheduler_config.use_v2_block_manager:
317 - version = "v2"
315 + version = "selfattn"
316 if (self.scheduler_config.embedding_mode
317 or self.cache_config.is_attention_free):
318 version = "placeholder"
vllm/engine/arg_utils.py
+17 -21
@@ -373,12 +373,13 @@ class EngineArgs:
373 action='store_true',
374 help='Disables sliding window, '
375 'capping to sliding window size')
376 - parser.add_argument(
377 - '--use-v2-block-manager',
378 - default=EngineArgs.use_v2_block_manager,
379 - action='store_true',
380 - help='Use BlockSpaceMangerV2. By default this is set to True. '
381 - 'Set to False to use BlockSpaceManagerV1')
376 + parser.add_argument('--use-v2-block-manager',
377 + action='store_true',
378 + help='[DEPRECATED] block manager v1 has been '
379 + 'removed and SelfAttnBlockSpaceManager (i.e. '
380 + 'block manager v2) is now the default. '
381 + 'Setting this flag to True or False'
382 + ' has no effect on vLLM behavior.')
383 parser.add_argument(
384 '--num-lookahead-slots',
385 type=int,
@@ -969,12 +970,6 @@ class EngineArgs:
970 "in low performance due to small KV cache space. Consider "
971 "setting --max-model-len to a smaller value.", max_model_len)
972
972 - if self.num_scheduler_steps > 1 and not self.use_v2_block_manager:
973 - self.use_v2_block_manager = True
974 - logger.warning(
975 - "Enabled BlockSpaceManagerV2 because it is "
976 - "required for multi-step (--num-scheduler-steps > 1)")
977 -
973 speculative_config = SpeculativeConfig.maybe_create_spec_config(
974 target_model_config=model_config,
975 target_parallel_config=parallel_config,
@@ -990,7 +985,6 @@ class EngineArgs:
985 speculative_disable_by_batch_size,
986 speculative_max_model_len=self.speculative_max_model_len,
987 enable_chunked_prefill=self.enable_chunked_prefill,
993 - use_v2_block_manager=self.use_v2_block_manager,
988 disable_log_stats=self.disable_log_stats,
989 ngram_prompt_lookup_max=self.ngram_prompt_lookup_max,
990 ngram_prompt_lookup_min=self.ngram_prompt_lookup_min,
@@ -1021,11 +1015,20 @@ class EngineArgs:
1015 if speculative_config is None \
1016 else speculative_config.num_lookahead_slots
1017
1018 + if not self.use_v2_block_manager:
1019 + logger.warning(
1020 + "[DEPRECATED] Block manager v1 has been removed, "
1021 + "and setting --use-v2-block-manager to True or False has "
1022 + "no effect on vLLM behavior. Please remove "
1023 + "--use-v2-block-manager in your engine argument. "
1024 + "If your use case is not supported by "
1025 + "SelfAttnBlockSpaceManager (i.e. block manager v2),"
1026 + " please file an issue with detailed information.")
1027 +
1028 scheduler_config = SchedulerConfig(
1029 max_num_batched_tokens=self.max_num_batched_tokens,
1030 max_num_seqs=self.max_num_seqs,
1031 max_model_len=model_config.max_model_len,
1028 - use_v2_block_manager=self.use_v2_block_manager,
1032 num_lookahead_slots=num_lookahead_slots,
1033 delay_factor=self.scheduler_delay_factor,
1034 enable_chunked_prefill=self.enable_chunked_prefill,
@@ -1081,13 +1084,6 @@ class EngineArgs:
1084 or "all" in detailed_trace_modules,
1085 )
1086
1084 - if (model_config.get_sliding_window() is not None
1085 - and scheduler_config.chunked_prefill_enabled
1086 - and not scheduler_config.use_v2_block_manager):
1087 - raise ValueError(
1088 - "Chunked prefill is not supported with sliding window. "
1089 - "Set --disable-sliding-window to disable sliding window.")
1090 -
1087 return EngineConfig(
1088 model_config=model_config,
1089 cache_config=cache_config,
vllm/engine/llm_engine.py
+1 -2
@@ -247,7 +247,7 @@ class LLMEngine:
247 "enforce_eager=%s, kv_cache_dtype=%s, "
248 "quantization_param_path=%s, device_config=%s, "
249 "decoding_config=%r, observability_config=%r, "
250 - "seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
250 + "seed=%d, served_model_name=%s, "
251 "num_scheduler_steps=%d, chunked_prefill_enabled=%s "
252 "multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
253 "use_async_output_proc=%s, use_cached_outputs=%s, "
@@ -280,7 +280,6 @@ class LLMEngine:
280 observability_config,
281 model_config.seed,
282 model_config.served_model_name,
283 - scheduler_config.use_v2_block_manager,
283 scheduler_config.num_scheduler_steps,
284 scheduler_config.chunked_prefill_enabled,
285 scheduler_config.multi_step_stream_outputs,
vllm/envs.py
-6
@@ -64,7 +64,6 @@ if TYPE_CHECKING:
64 VLLM_USE_TRITON_AWQ: bool = False
65 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66 VLLM_SKIP_P2P_CHECK: bool = False
67 - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1: bool = False
67 VLLM_TORCH_COMPILE_LEVEL: int = 0
68 VLLM_DISABLED_KERNELS: List[str] = []
69
@@ -427,11 +426,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
426 "VLLM_SKIP_P2P_CHECK":
427 lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "0") == "1",
428
430 - # If set, allowing the use of deprecated block manager V1
431 - "VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1":
432 - lambda: os.environ.get("VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1", "0"
433 - ) == "1",
434 -
429 # List of quantization kernels that should be disabled, used for testing
430 # and performance comparisons. Currently only affects MPLinearKernel
431 # selection
vllm/worker/model_runner.py
+6 -11
@@ -574,17 +574,12 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
574 # paged attn. We can remove it if we make paged attn kernel
575 # to properly handle slinding window attn.
576 curr_sliding_window_block = self.sliding_window_blocks
577 - if self.scheduler_config.use_v2_block_manager:
578 - # number of elements in last block
579 - suff_len = inter_data.seq_lens[seq_idx] % self.block_size
580 - sliding_seq_len = min(
581 - inter_data.seq_lens[seq_idx],
582 - self.block_aligned_sliding_window + suff_len)
583 - if suff_len > 0:
584 - curr_sliding_window_block += 1
585 - else:
586 - sliding_seq_len = min(inter_data.seq_lens[seq_idx],
587 - self.sliding_window)
577 + # number of elements in last block
578 + suff_len = inter_data.seq_lens[seq_idx] % self.block_size
579 + sliding_seq_len = min(inter_data.seq_lens[seq_idx],
580 + self.block_aligned_sliding_window + suff_len)
581 + if suff_len > 0:
582 + curr_sliding_window_block += 1
583
584 inter_data.curr_sliding_window_blocks[
585 seq_idx] = curr_sliding_window_block