@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7560ae5caf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[8/N] enable cli flag without a space (#10529)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 21, 2024 at 12:30 UTC 7560ae5cafbae3af9967ac7dc979cb31a40fc572
5 files changed +43 -7
tests/compile/test_basic_correctness.py
+2 -2
@@ -103,7 +103,7 @@ def test_compile_correctness(test_setting: TestSetting):
103 CompilationLevel.NO_COMPILATION,
104 CompilationLevel.PIECEWISE,
105 ]:
106 - all_args.append(final_args + ["-O", str(level)])
106 + all_args.append(final_args + [f"-O{level}"])
107 all_envs.append({})
108
109 # inductor will change the output, so we only compare if the output
@@ -121,7 +121,7 @@ def test_compile_correctness(test_setting: TestSetting):
121 CompilationLevel.DYNAMO_AS_IS,
122 CompilationLevel.DYNAMO_ONCE,
123 ]:
124 - all_args.append(final_args + ["-O", str(level)])
124 + all_args.append(final_args + [f"-O{level}"])
125 all_envs.append({})
126 if level != CompilationLevel.DYNAMO_ONCE and not fullgraph:
127 # "DYNAMO_ONCE" will always use fullgraph
tests/engine/test_arg_utils.py
+28
@@ -31,6 +31,34 @@ def test_limit_mm_per_prompt_parser(arg, expected):
31 assert args.limit_mm_per_prompt == expected
32
33
34 +def test_compilation_config():
35 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
36 +
37 + # default value
38 + args = parser.parse_args([])
39 + assert args.compilation_config is None
40 +
41 + # set to O3
42 + args = parser.parse_args(["-O3"])
43 + assert args.compilation_config.level == 3
44 +
45 + # set to O 3 (space)
46 + args = parser.parse_args(["-O", "3"])
47 + assert args.compilation_config.level == 3
48 +
49 + # set to O 3 (equals)
50 + args = parser.parse_args(["-O=3"])
51 + assert args.compilation_config.level == 3
52 +
53 + # set to json
54 + args = parser.parse_args(["--compilation-config", '{"level": 3}'])
55 + assert args.compilation_config.level == 3
56 +
57 + # set to json
58 + args = parser.parse_args(['--compilation-config={"level": 3}'])
59 + assert args.compilation_config.level == 3
60 +
61 +
62 def test_valid_pooling_config():
63 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
64 args = parser.parse_args([
tests/tpu/test_custom_dispatcher.py
+5 -4
@@ -13,9 +13,10 @@ os.environ["VLLM_RPC_TIMEOUT"] = "30000"
13 def test_custom_dispatcher():
14 compare_two_settings(
15 "google/gemma-2b",
16 - arg1=["--enforce-eager", "-O",
17 - str(CompilationLevel.DYNAMO_ONCE)],
18 - arg2=["--enforce-eager", "-O",
19 - str(CompilationLevel.DYNAMO_AS_IS)],
16 + arg1=[
17 + "--enforce-eager",
18 + f"-O{CompilationLevel.DYNAMO_ONCE}",
19 + ],
20 + arg2=["--enforce-eager", f"-O{CompilationLevel.DYNAMO_AS_IS}"],
21 env1={},
22 env2={})
vllm/engine/arg_utils.py
+4 -1
@@ -882,7 +882,10 @@ class EngineArgs:
882 'testing only. level 3 is the recommended level '
883 'for production.\n'
884 'To specify the full compilation config, '
885 - 'use a JSON string.')
885 + 'use a JSON string.\n'
886 + 'Following the convention of traditional '
887 + 'compilers, using -O without space is also '
888 + 'supported. -O3 is equivalent to -O 3.')
889
890 return parser
891
vllm/utils.py
+4
@@ -1192,6 +1192,10 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1192 else:
1193 processed_args.append('--' +
1194 arg[len('--'):].replace('_', '-'))
1195 + elif arg.startswith('-O') and arg != '-O' and len(arg) == 2:
1196 + # allow -O flag to be used without space, e.g. -O3
1197 + processed_args.append('-O')
1198 + processed_args.append(arg[2:])
1199 else:
1200 processed_args.append(arg)
1201