A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Core] Add an environment variable which needs to be set explicitly to allow BlockSpaceManagerV1 (#9149)
sroy745 committed
Oct 9, 2024 at 23:17 UTC
f3a507f1d31e13a99c4fc8ac02738a73c3e3136f
14 files changed
+94
-8
.buildkite/test-pipeline.yaml
+12
-6
@@ -77,8 +77,8 @@ steps:
77
- vllm/
78
- tests/basic_correctness/test_chunked_prefill
79
commands:
80
- - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
81
- - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
80
+ - VLLM_ATTENTION_BACKEND=XFORMERS VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s basic_correctness/test_chunked_prefill.py
81
+ - VLLM_ATTENTION_BACKEND=FLASH_ATTN VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s basic_correctness/test_chunked_prefill.py
82
83
- label: Core Test # 10min
84
mirror_hardwares: [amd]
@@ -88,7 +88,11 @@ steps:
88
- vllm/distributed
89
- tests/core
90
commands:
91
- - pytest -v -s core
91
+ - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core/test_scheduler.py
92
+ - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/test_chunked_prefill_scheduler.py
93
+ - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/block/e2e/test_correctness.py
94
+ - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s core core/block/e2e/test_correctness_sliding_window.py
95
+ - pytest -v -s core --ignore=core/block/e2e/test_correctness.py --ignore=core/test_scheduler.py --ignore=core/test_chunked_prefill_scheduler.py --ignore=core/block/e2e/test_correctness.py --ignore=core/block/e2e/test_correctness_sliding_window.py
96
97
- label: Entrypoints Test # 40min
98
working_dir: "/vllm-workspace/tests"
@@ -185,7 +189,8 @@ steps:
189
- vllm/
190
- tests/prefix_caching
191
commands:
188
- - pytest -v -s prefix_caching
192
+ - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s prefix_caching/test_prefix_caching.py
193
+ - pytest -v -s prefix_caching --ignore=prefix_caching/test_prefix_caching.py
194
195
- label: Samplers Test # 36min
196
source_file_dependencies:
@@ -209,7 +214,8 @@ steps:
214
- tests/spec_decode
215
commands:
216
- pytest -v -s spec_decode/e2e/test_multistep_correctness.py
212
- - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
217
+ - VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest -v -s spec_decode/e2e/test_compatibility.py
218
+ - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py --ignore=spec_decode/e2e/test_compatibility.py
219
220
- label: LoRA Test %N # 15min each
221
mirror_hardwares: [amd]
@@ -391,7 +397,7 @@ steps:
397
- pytest -v -s ./compile/test_full_graph_multi_gpu.py
398
- pytest -v -s ./compile/test_wrapper.py
399
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
394
- - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
400
+ - TARGET_TEST_SUITE=L4 VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest basic_correctness/ -v -s -m distributed_2_gpus
401
# Avoid importing model tests that cause CUDA reinitialization error
402
- pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
403
- pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
benchmarks/benchmark_latency.py
+3
-1
@@ -221,7 +221,9 @@ if __name__ == '__main__':
221
parser.add_argument("--enable-prefix-caching",
222
action='store_true',
223
help="Enable automatic prefix caching")
224
- parser.add_argument('--use-v2-block-manager', action='store_true')
224
+ parser.add_argument('--use-v2-block-manager',
225
+ action='store_true',
226
+ default=EngineArgs.use_v2_block_manager)
227
parser.add_argument(
228
"--ray-workers-use-nsight",
229
action='store_true',
benchmarks/benchmark_prefix_caching.py
+2
@@ -33,6 +33,7 @@ from typing import List, Optional, Tuple
33
from transformers import PreTrainedTokenizerBase
34
35
from vllm import LLM, SamplingParams
36
+from vllm.engine.arg_utils import EngineArgs
37
from vllm.utils import FlexibleArgumentParser
38
39
try:
@@ -177,6 +178,7 @@ if __name__ == "__main__":
178
help='enable prefix caching')
179
parser.add_argument('--use-v2-block-manager',
180
action='store_true',
181
+ default=EngineArgs.use_v2_block_manager,
182
help='Use BlockSpaceMangerV2')
183
parser.add_argument('--num-prompts',
184
type=int,
benchmarks/benchmark_throughput.py
+1
@@ -473,6 +473,7 @@ if __name__ == "__main__":
473
help="Maximum number of forward steps per scheduler call.")
474
parser.add_argument("--use-v2-block-manager",
475
action='store_true',
476
+ default=EngineArgs.use_v2_block_manager,
477
help="Enable block manager v2.")
478
parser.add_argument(
479
"--enable-prefix-caching",
tests/basic_correctness/test_chunked_prefill.py
+7
-1
@@ -12,7 +12,7 @@ from contextlib import nullcontext
12
import pytest
13
14
from ..models.utils import check_logprobs_close, check_outputs_equal
15
-from ..utils import multi_gpu_test
15
+from ..utils import check_deprecated_block_manager_usage, multi_gpu_test
16
17
MODELS = [
18
"facebook/opt-125m",
@@ -20,6 +20,12 @@ MODELS = [
20
]
21
22
23
+@pytest.fixture(scope="module", autouse=True)
24
+def check_deprecated_block_manager():
25
+ check_deprecated_block_manager_usage(
26
+ 'tests/basic_correctness/test_chunked_prefill.py')
27
+
28
+
29
@pytest.mark.parametrize("model", MODELS)
30
@pytest.mark.parametrize("dtype", ["half"])
31
@pytest.mark.parametrize("max_tokens", [32])
tests/core/block/e2e/test_correctness.py
+7
@@ -2,11 +2,18 @@ from itertools import cycle
2
3
import pytest
4
5
+from tests.utils import check_deprecated_block_manager_usage
6
from vllm import SamplingParams
7
8
from .conftest import get_token_ids_from_llm_generator
9
10
11
+@pytest.fixture(scope="module", autouse=True)
12
+def check_deprecated_block_manager():
13
+ check_deprecated_block_manager_usage(
14
+ 'tests/core/block/e2e/test_correctness.py')
15
+
16
+
17
@pytest.mark.parametrize(
18
"common_llm_kwargs",
19
[{
tests/core/block/e2e/test_correctness_sliding_window.py
+7
@@ -3,6 +3,7 @@ from typing import List
3
4
import pytest
5
6
+from tests.utils import check_deprecated_block_manager_usage
7
from vllm import LLM, SamplingParams
8
9
from .conftest import get_text_from_llm_generator
@@ -12,6 +13,12 @@ MODEL = "bigcode/starcoder2-3b"
13
BLOCK_SIZE = 16
14
15
16
+@pytest.fixture(scope="module", autouse=True)
17
+def check_deprecated_block_manager():
18
+ check_deprecated_block_manager_usage(
19
+ 'tests/core/block/e2e/test_correctness_sliding_window.py')
20
+
21
+
22
@pytest.mark.parametrize(
23
"common_llm_kwargs",
24
[{
tests/core/test_chunked_prefill_scheduler.py
+7
@@ -8,6 +8,7 @@ from vllm.core.interfaces import AllocStatus
8
from vllm.core.scheduler import Scheduler
9
from vllm.sequence import Logprob, SequenceGroup
10
11
+from ..utils import check_deprecated_block_manager_usage
12
from .utils import create_dummy_prompt
13
14
@@ -27,6 +28,12 @@ def schedule_and_update_computed_tokens(scheduler):
28
return metas, out
29
30
31
+@pytest.fixture(scope="module", autouse=True)
32
+def check_deprecated_block_manager():
33
+ check_deprecated_block_manager_usage(
34
+ 'tests/core/test_chunked_prefill_scheduler.py')
35
+
36
+
37
@pytest.mark.parametrize('use_v2_block_manager', [True, False])
38
def test_simple(use_v2_block_manager: bool):
39
"""Verify basic scheduling works."""
tests/core/test_scheduler.py
+7
@@ -12,11 +12,18 @@ from vllm.core.scheduler import Scheduler, SchedulingBudget
12
from vllm.lora.request import LoRARequest
13
from vllm.sequence import SequenceGroup, SequenceStatus
14
15
+from ..utils import check_deprecated_block_manager_usage
16
from .utils import (append_new_token, append_new_token_seq_group,
17
create_dummy_prompt, get_sequence_groups,
18
schedule_and_update_computed_tokens)
19
20
21
+@pytest.fixture(scope="module", autouse=True)
22
+def check_deprecated_block_manager():
23
+ check_deprecated_block_manager_usage(
24
+ "tests/core/test_chunked_prefill_scheduler.py")
25
+
26
+
27
@pytest.mark.parametrize('use_v2_block_manager', [True, False])
28
def test_scheduler_add_seq_group(use_v2_block_manager: bool):
29
block_size = 4
tests/prefix_caching/test_prefix_caching.py
+7
@@ -7,6 +7,7 @@ from typing import List
7
import pytest
8
9
from tests.kernels.utils import override_backend_env_variable
10
+from tests.utils import check_deprecated_block_manager_usage
11
from vllm.block import PhysicalTokenBlock
12
from vllm.core.block_manager_v1 import CachedBlockAllocator
13
from vllm.utils import Device
@@ -18,6 +19,12 @@ MODELS = [
19
]
20
21
22
+@pytest.fixture(scope="module", autouse=True)
23
+def check_deprecated_block_manager():
24
+ check_deprecated_block_manager_usage(
25
+ 'tests/prefix_caching/test_prefix_caching.py')
26
+
27
+
28
@pytest.mark.parametrize("block_size", [16])
29
@pytest.mark.parametrize("num_blocks", [16])
30
def test_block_allocator(
tests/spec_decode/e2e/test_compatibility.py
+7
@@ -1,10 +1,17 @@
1
import pytest
2
3
+from tests.utils import check_deprecated_block_manager_usage
4
from vllm import SamplingParams
5
6
from .conftest import get_output_from_llm_generator
7
8
9
+@pytest.fixture(scope="module", autouse=True)
10
+def check_deprecated_block_manager():
11
+ check_deprecated_block_manager_usage(
12
+ 'tests/spec_decode/e2e/test_compatibility.py')
13
+
14
+
15
@pytest.mark.parametrize(
16
"common_llm_kwargs",
17
[{
tests/utils.py
+9
@@ -678,3 +678,12 @@ def get_client_text_logprob_generations(
678
return [(text_generations, text,
679
(None if x.logprobs is None else x.logprobs.top_logprobs))
680
for completion in completions for x in completion.choices]
681
+
682
+
683
+def check_deprecated_block_manager_usage(test_name: str):
684
+ assert envs.VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1 is True, (
685
+ f"To allow the use of deprecated BlockSpaceManagerV1, set the "
686
+ f"environment variable VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1. "
687
+ f"You can run the tests with: "
688
+ f"`VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest {test_name}`" #noqa
689
+ )
vllm/config.py
+12
@@ -1037,6 +1037,18 @@ class SchedulerConfig:
1037
f"({self.num_scheduler_steps}) must be greater than or "
1038
"equal to 1.")
1039
1040
+ if (not self.use_v2_block_manager \
1041
+ and not envs.VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1):
1042
+ raise ValueError(
1043
+ "The use of BlockSpaceManagerV1 is deprecated and will "
1044
+ "be removed in a future release. Please switch to "
1045
+ "BlockSpaceManagerV2 by setting --use-v2-block-manager to "
1046
+ "True. If you wish to suppress this error temporarily, "
1047
+ "you can set the environment variable "
1048
+ "`VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1. If your use "
1049
+ "case is not supported in BlockSpaceManagerV2, please "
1050
+ "file an issue with detailed information.")
1051
+
1052
@property
1053
def is_multi_step(self) -> bool:
1054
return self.num_scheduler_steps > 1
vllm/envs.py
+6
@@ -64,6 +64,7 @@ if TYPE_CHECKING:
64
VLLM_USE_TRITON_AWQ: bool = False
65
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66
VLLM_SKIP_P2P_CHECK: bool = False
67
+ VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1: bool = False
68
69
70
def get_default_cache_root():
@@ -434,6 +435,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
435
# and trust the driver's peer-to-peer capability report.
436
"VLLM_SKIP_P2P_CHECK":
437
lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "0") == "1",
438
+
439
+ # If set, allowing the use of deprecated block manager V1
440
+ "VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1":
441
+ lambda: os.environ.get("VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1", "0"
442
+ ) == "1",
443
}
444
445
# end-env-vars-definition