@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f3a507f1d3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Core] Add an environment variable which needs to be set explicitly to allow BlockSpaceManagerV1 (#9149)

sroy745 committed Oct 9, 2024 at 23:17 UTC f3a507f1d31e13a99c4fc8ac02738a73c3e3136f
14 files changed +94 -8
.buildkite/test-pipeline.yaml
+12 -6
@@ -77,8 +77,8 @@ steps:
77 - vllm/
78 - tests/basic_correctness/test_chunked_prefill
79 commands:
80 - - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
81 - - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
80 + - VLLM_ATTENTION_BACKEND=XFORMERS VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s basic_correctness/test_chunked_prefill.py
81 + - VLLM_ATTENTION_BACKEND=FLASH_ATTN VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s basic_correctness/test_chunked_prefill.py
82
83 - label: Core Test # 10min
84 mirror_hardwares: [amd]
@@ -88,7 +88,11 @@ steps:
88 - vllm/distributed
89 - tests/core
90 commands:
91 - - pytest -v -s core
91 + - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core/test_scheduler.py
92 + - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/test_chunked_prefill_scheduler.py
93 + - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/block/e2e/test_correctness.py
94 + - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/block/e2e/test_correctness_sliding_window.py
95 + - pytest -v -s core --ignore=core/block/e2e/test_correctness.py --ignore=core/test_scheduler.py --ignore=core/test_chunked_prefill_scheduler.py --ignore=core/block/e2e/test_correctness.py --ignore=core/block/e2e/test_correctness_sliding_window.py
96
97 - label: Entrypoints Test # 40min
98 working_dir: "/vllm-workspace/tests"
@@ -185,7 +189,8 @@ steps:
189 - vllm/
190 - tests/prefix_caching
191 commands:
188 - - pytest -v -s prefix_caching
192 + - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s prefix_caching/test_prefix_caching.py
193 + - pytest -v -s prefix_caching --ignore=prefix_caching/test_prefix_caching.py
194
195 - label: Samplers Test # 36min
196 source_file_dependencies:
@@ -209,7 +214,8 @@ steps:
214 - tests/spec_decode
215 commands:
216 - pytest -v -s spec_decode/e2e/test_multistep_correctness.py
212 - - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
217 + - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s spec_decode/e2e/test_compatibility.py
218 + - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py --ignore=spec_decode/e2e/test_compatibility.py
219
220 - label: LoRA Test %N # 15min each
221 mirror_hardwares: [amd]
@@ -391,7 +397,7 @@ steps:
397 - pytest -v -s ./compile/test_full_graph_multi_gpu.py
398 - pytest -v -s ./compile/test_wrapper.py
399 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
394 - - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
400 + - TARGET_TEST_SUITE=L4 VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest basic_correctness/ -v -s -m distributed_2_gpus
401 # Avoid importing model tests that cause CUDA reinitialization error
402 - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
403 - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
benchmarks/benchmark_latency.py
+3 -1
@@ -221,7 +221,9 @@ if __name__ == '__main__':
221 parser.add_argument("--enable-prefix-caching",
222 action='store_true',
223 help="Enable automatic prefix caching")
224 - parser.add_argument('--use-v2-block-manager', action='store_true')
224 + parser.add_argument('--use-v2-block-manager',
225 + action='store_true',
226 + default=EngineArgs.use_v2_block_manager)
227 parser.add_argument(
228 "--ray-workers-use-nsight",
229 action='store_true',
benchmarks/benchmark_prefix_caching.py
+2
@@ -33,6 +33,7 @@ from typing import List, Optional, Tuple
33 from transformers import PreTrainedTokenizerBase
34
35 from vllm import LLM, SamplingParams
36 +from vllm.engine.arg_utils import EngineArgs
37 from vllm.utils import FlexibleArgumentParser
38
39 try:
@@ -177,6 +178,7 @@ if __name__ == "__main__":
178 help='enable prefix caching')
179 parser.add_argument('--use-v2-block-manager',
180 action='store_true',
181 + default=EngineArgs.use_v2_block_manager,
182 help='Use BlockSpaceMangerV2')
183 parser.add_argument('--num-prompts',
184 type=int,
benchmarks/benchmark_throughput.py
+1
@@ -473,6 +473,7 @@ if __name__ == "__main__":
473 help="Maximum number of forward steps per scheduler call.")
474 parser.add_argument("--use-v2-block-manager",
475 action='store_true',
476 + default=EngineArgs.use_v2_block_manager,
477 help="Enable block manager v2.")
478 parser.add_argument(
479 "--enable-prefix-caching",
tests/basic_correctness/test_chunked_prefill.py
+7 -1
@@ -12,7 +12,7 @@ from contextlib import nullcontext
12 import pytest
13
14 from ..models.utils import check_logprobs_close, check_outputs_equal
15 -from ..utils import multi_gpu_test
15 +from ..utils import check_deprecated_block_manager_usage, multi_gpu_test
16
17 MODELS = [
18 "facebook/opt-125m",
@@ -20,6 +20,12 @@ MODELS = [
20 ]
21
22
23 +@pytest.fixture(scope="module", autouse=True)
24 +def check_deprecated_block_manager():
25 + check_deprecated_block_manager_usage(
26 + 'tests/basic_correctness/test_chunked_prefill.py')
27 +
28 +
29 @pytest.mark.parametrize("model", MODELS)
30 @pytest.mark.parametrize("dtype", ["half"])
31 @pytest.mark.parametrize("max_tokens", [32])
tests/core/block/e2e/test_correctness.py
+7
@@ -2,11 +2,18 @@ from itertools import cycle
2
3 import pytest
4
5 +from tests.utils import check_deprecated_block_manager_usage
6 from vllm import SamplingParams
7
8 from .conftest import get_token_ids_from_llm_generator
9
10
11 +@pytest.fixture(scope="module", autouse=True)
12 +def check_deprecated_block_manager():
13 + check_deprecated_block_manager_usage(
14 + 'tests/core/block/e2e/test_correctness.py')
15 +
16 +
17 @pytest.mark.parametrize(
18 "common_llm_kwargs",
19 [{
tests/core/block/e2e/test_correctness_sliding_window.py
+7
@@ -3,6 +3,7 @@ from typing import List
3
4 import pytest
5
6 +from tests.utils import check_deprecated_block_manager_usage
7 from vllm import LLM, SamplingParams
8
9 from .conftest import get_text_from_llm_generator
@@ -12,6 +13,12 @@ MODEL = "bigcode/starcoder2-3b"
13 BLOCK_SIZE = 16
14
15
16 +@pytest.fixture(scope="module", autouse=True)
17 +def check_deprecated_block_manager():
18 + check_deprecated_block_manager_usage(
19 + 'tests/core/block/e2e/test_correctness_sliding_window.py')
20 +
21 +
22 @pytest.mark.parametrize(
23 "common_llm_kwargs",
24 [{
tests/core/test_chunked_prefill_scheduler.py
+7
@@ -8,6 +8,7 @@ from vllm.core.interfaces import AllocStatus
8 from vllm.core.scheduler import Scheduler
9 from vllm.sequence import Logprob, SequenceGroup
10
11 +from ..utils import check_deprecated_block_manager_usage
12 from .utils import create_dummy_prompt
13
14
@@ -27,6 +28,12 @@ def schedule_and_update_computed_tokens(scheduler):
28 return metas, out
29
30
31 +@pytest.fixture(scope="module", autouse=True)
32 +def check_deprecated_block_manager():
33 + check_deprecated_block_manager_usage(
34 + 'tests/core/test_chunked_prefill_scheduler.py')
35 +
36 +
37 @pytest.mark.parametrize('use_v2_block_manager', [True, False])
38 def test_simple(use_v2_block_manager: bool):
39 """Verify basic scheduling works."""
tests/core/test_scheduler.py
+7
@@ -12,11 +12,18 @@ from vllm.core.scheduler import Scheduler, SchedulingBudget
12 from vllm.lora.request import LoRARequest
13 from vllm.sequence import SequenceGroup, SequenceStatus
14
15 +from ..utils import check_deprecated_block_manager_usage
16 from .utils import (append_new_token, append_new_token_seq_group,
17 create_dummy_prompt, get_sequence_groups,
18 schedule_and_update_computed_tokens)
19
20
21 +@pytest.fixture(scope="module", autouse=True)
22 +def check_deprecated_block_manager():
23 + check_deprecated_block_manager_usage(
24 + "tests/core/test_chunked_prefill_scheduler.py")
25 +
26 +
27 @pytest.mark.parametrize('use_v2_block_manager', [True, False])
28 def test_scheduler_add_seq_group(use_v2_block_manager: bool):
29 block_size = 4
tests/prefix_caching/test_prefix_caching.py
+7
@@ -7,6 +7,7 @@ from typing import List
7 import pytest
8
9 from tests.kernels.utils import override_backend_env_variable
10 +from tests.utils import check_deprecated_block_manager_usage
11 from vllm.block import PhysicalTokenBlock
12 from vllm.core.block_manager_v1 import CachedBlockAllocator
13 from vllm.utils import Device
@@ -18,6 +19,12 @@ MODELS = [
19 ]
20
21
22 +@pytest.fixture(scope="module", autouse=True)
23 +def check_deprecated_block_manager():
24 + check_deprecated_block_manager_usage(
25 + 'tests/prefix_caching/test_prefix_caching.py')
26 +
27 +
28 @pytest.mark.parametrize("block_size", [16])
29 @pytest.mark.parametrize("num_blocks", [16])
30 def test_block_allocator(
tests/spec_decode/e2e/test_compatibility.py
+7
@@ -1,10 +1,17 @@
1 import pytest
2
3 +from tests.utils import check_deprecated_block_manager_usage
4 from vllm import SamplingParams
5
6 from .conftest import get_output_from_llm_generator
7
8
9 +@pytest.fixture(scope="module", autouse=True)
10 +def check_deprecated_block_manager():
11 + check_deprecated_block_manager_usage(
12 + 'tests/spec_decode/e2e/test_compatibility.py')
13 +
14 +
15 @pytest.mark.parametrize(
16 "common_llm_kwargs",
17 [{
tests/utils.py
+9
@@ -678,3 +678,12 @@ def get_client_text_logprob_generations(
678 return [(text_generations, text,
679 (None if x.logprobs is None else x.logprobs.top_logprobs))
680 for completion in completions for x in completion.choices]
681 +
682 +
683 +def check_deprecated_block_manager_usage(test_name: str):
684 + assert envs.VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1 is True, (
685 + f"To allow the use of deprecated BlockSpaceManagerV1, set the "
686 + f"environment variable VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1. "
687 + f"You can run the tests with: "
688 + f"`VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest {test_name}`" #noqa
689 + )
vllm/config.py
+12
@@ -1037,6 +1037,18 @@ class SchedulerConfig:
1037 f"({self.num_scheduler_steps}) must be greater than or "
1038 "equal to 1.")
1039
1040 + if (not self.use_v2_block_manager \
1041 + and not envs.VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1):
1042 + raise ValueError(
1043 + "The use of BlockSpaceManagerV1 is deprecated and will "
1044 + "be removed in a future release. Please switch to "
1045 + "BlockSpaceManagerV2 by setting --use-v2-block-manager to "
1046 + "True. If you wish to suppress this error temporarily, "
1047 + "you can set the environment variable "
1048 + "`VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1. If your use "
1049 + "case is not supported in BlockSpaceManagerV2, please "
1050 + "file an issue with detailed information.")
1051 +
1052 @property
1053 def is_multi_step(self) -> bool:
1054 return self.num_scheduler_steps > 1
vllm/envs.py
+6
@@ -64,6 +64,7 @@ if TYPE_CHECKING:
64 VLLM_USE_TRITON_AWQ: bool = False
65 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66 VLLM_SKIP_P2P_CHECK: bool = False
67 + VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1: bool = False
68
69
70 def get_default_cache_root():
@@ -434,6 +435,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
435 # and trust the driver's peer-to-peer capability report.
436 "VLLM_SKIP_P2P_CHECK":
437 lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "0") == "1",
438 +
439 + # If set, allowing the use of deprecated block manager V1
440 + "VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1":
441 + lambda: os.environ.get("VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1", "0"
442 + ) == "1",
443 }
444
445 # end-env-vars-definition