@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d9b4b3f069
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bug][CLI] Allow users to disable prefix caching explicitly (#10724)

Signed-off-by: rickyx <rickyx@anyscale.com>

Ricky Xu committed Nov 27, 2024 at 23:59 UTC d9b4b3f069a9f602b067a5bb3efe57b106d39c09
3 files changed +45 -3
tests/engine/test_arg_utils.py
+19
@@ -59,6 +59,25 @@ def test_compilation_config():
59 assert args.compilation_config.level == 3
60
61
62 +def test_prefix_cache_default():
63 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
64 + args = parser.parse_args([])
65 +
66 + engine_args = EngineArgs.from_cli_args(args=args)
67 + assert (not engine_args.enable_prefix_caching
68 + ), "prefix caching defaults to off."
69 +
70 + # with flag to turn it on.
71 + args = parser.parse_args(["--enable-prefix-caching"])
72 + engine_args = EngineArgs.from_cli_args(args=args)
73 + assert engine_args.enable_prefix_caching
74 +
75 + # with disable flag to turn it off.
76 + args = parser.parse_args(["--no-enable-prefix-caching"])
77 + engine_args = EngineArgs.from_cli_args(args=args)
78 + assert not engine_args.enable_prefix_caching
79 +
80 +
81 def test_valid_pooling_config():
82 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
83 args = parser.parse_args([
tests/v1/engine/test_engine_args.py
+19
@@ -4,6 +4,7 @@ from vllm import envs
4 from vllm.config import VllmConfig
5 from vllm.engine.arg_utils import EngineArgs
6 from vllm.usage.usage_lib import UsageContext
7 +from vllm.utils import FlexibleArgumentParser
8
9 if not envs.VLLM_USE_V1:
10 pytest.skip(
@@ -12,6 +13,24 @@ if not envs.VLLM_USE_V1:
13 )
14
15
16 +def test_prefix_caching_from_cli():
17 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
18 + args = parser.parse_args([])
19 + engine_args = EngineArgs.from_cli_args(args=args)
20 + assert (engine_args.enable_prefix_caching
21 + ), "V1 turns on prefix caching by default."
22 +
23 + # Turn it off possible with flag.
24 + args = parser.parse_args(["--no-enable-prefix-caching"])
25 + engine_args = EngineArgs.from_cli_args(args=args)
26 + assert not engine_args.enable_prefix_caching
27 +
28 + # Turn it on with flag.
29 + args = parser.parse_args(["--enable-prefix-caching"])
30 + engine_args = EngineArgs.from_cli_args(args=args)
31 + assert engine_args.enable_prefix_caching
32 +
33 +
34 def test_defaults():
35 engine_args = EngineArgs(model="facebook/opt-125m")
36
vllm/engine/arg_utils.py
+7 -3
@@ -416,9 +416,13 @@ class EngineArgs:
416 'tokens. This is ignored on neuron devices and '
417 'set to max-model-len')
418
419 - parser.add_argument('--enable-prefix-caching',
420 - action='store_true',
421 - help='Enables automatic prefix caching.')
419 + parser.add_argument(
420 + "--enable-prefix-caching",
421 + action=argparse.BooleanOptionalAction,
422 + default=EngineArgs.enable_prefix_caching,
423 + help="Enables automatic prefix caching. "
424 + "Use --no-enable-prefix-caching to disable explicitly.",
425 + )
426 parser.add_argument('--disable-sliding-window',
427 action='store_true',
428 help='Disables sliding window, '