A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bug][CLI] Allow users to disable prefix caching explicitly (#10724)
Signed-off-by: rickyx <rickyx@anyscale.com>
Ricky Xu committed
Nov 27, 2024 at 23:59 UTC
d9b4b3f069a9f602b067a5bb3efe57b106d39c09
3 files changed
+45
-3
tests/engine/test_arg_utils.py
+19
@@ -59,6 +59,25 @@ def test_compilation_config():
59
assert args.compilation_config.level == 3
60
61
62
+def test_prefix_cache_default():
63
+ parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
64
+ args = parser.parse_args([])
65
+
66
+ engine_args = EngineArgs.from_cli_args(args=args)
67
+ assert (not engine_args.enable_prefix_caching
68
+ ), "prefix caching defaults to off."
69
+
70
+ # with flag to turn it on.
71
+ args = parser.parse_args(["--enable-prefix-caching"])
72
+ engine_args = EngineArgs.from_cli_args(args=args)
73
+ assert engine_args.enable_prefix_caching
74
+
75
+ # with disable flag to turn it off.
76
+ args = parser.parse_args(["--no-enable-prefix-caching"])
77
+ engine_args = EngineArgs.from_cli_args(args=args)
78
+ assert not engine_args.enable_prefix_caching
79
+
80
+
81
def test_valid_pooling_config():
82
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
83
args = parser.parse_args([
tests/v1/engine/test_engine_args.py
+19
@@ -4,6 +4,7 @@ from vllm import envs
4
from vllm.config import VllmConfig
5
from vllm.engine.arg_utils import EngineArgs
6
from vllm.usage.usage_lib import UsageContext
7
+from vllm.utils import FlexibleArgumentParser
8
9
if not envs.VLLM_USE_V1:
10
pytest.skip(
@@ -12,6 +13,24 @@ if not envs.VLLM_USE_V1:
13
)
14
15
16
+def test_prefix_caching_from_cli():
17
+ parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
18
+ args = parser.parse_args([])
19
+ engine_args = EngineArgs.from_cli_args(args=args)
20
+ assert (engine_args.enable_prefix_caching
21
+ ), "V1 turns on prefix caching by default."
22
+
23
+ # Turn it off possible with flag.
24
+ args = parser.parse_args(["--no-enable-prefix-caching"])
25
+ engine_args = EngineArgs.from_cli_args(args=args)
26
+ assert not engine_args.enable_prefix_caching
27
+
28
+ # Turn it on with flag.
29
+ args = parser.parse_args(["--enable-prefix-caching"])
30
+ engine_args = EngineArgs.from_cli_args(args=args)
31
+ assert engine_args.enable_prefix_caching
32
+
33
+
34
def test_defaults():
35
engine_args = EngineArgs(model="facebook/opt-125m")
36
vllm/engine/arg_utils.py
+7
-3
@@ -416,9 +416,13 @@ class EngineArgs:
416
'tokens. This is ignored on neuron devices and '
417
'set to max-model-len')
418
419
- parser.add_argument('--enable-prefix-caching',
420
- action='store_true',
421
- help='Enables automatic prefix caching.')
419
+ parser.add_argument(
420
+ "--enable-prefix-caching",
421
+ action=argparse.BooleanOptionalAction,
422
+ default=EngineArgs.enable_prefix_caching,
423
+ help="Enables automatic prefix caching. "
424
+ "Use --no-enable-prefix-caching to disable explicitly.",
425
+ )
426
parser.add_argument('--disable-sliding-window',
427
action='store_true',
428
help='Disables sliding window, '