A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[8/N] enable cli flag without a space (#10529)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 21, 2024 at 12:30 UTC
7560ae5cafbae3af9967ac7dc979cb31a40fc572
5 files changed
+43
-7
tests/compile/test_basic_correctness.py
+2
-2
@@ -103,7 +103,7 @@ def test_compile_correctness(test_setting: TestSetting):
103
CompilationLevel.NO_COMPILATION,
104
CompilationLevel.PIECEWISE,
105
]:
106
- all_args.append(final_args + ["-O", str(level)])
106
+ all_args.append(final_args + [f"-O{level}"])
107
all_envs.append({})
108
109
# inductor will change the output, so we only compare if the output
@@ -121,7 +121,7 @@ def test_compile_correctness(test_setting: TestSetting):
121
CompilationLevel.DYNAMO_AS_IS,
122
CompilationLevel.DYNAMO_ONCE,
123
]:
124
- all_args.append(final_args + ["-O", str(level)])
124
+ all_args.append(final_args + [f"-O{level}"])
125
all_envs.append({})
126
if level != CompilationLevel.DYNAMO_ONCE and not fullgraph:
127
# "DYNAMO_ONCE" will always use fullgraph
tests/engine/test_arg_utils.py
+28
@@ -31,6 +31,34 @@ def test_limit_mm_per_prompt_parser(arg, expected):
31
assert args.limit_mm_per_prompt == expected
32
33
34
+def test_compilation_config():
35
+ parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
36
+
37
+ # default value
38
+ args = parser.parse_args([])
39
+ assert args.compilation_config is None
40
+
41
+ # set to O3
42
+ args = parser.parse_args(["-O3"])
43
+ assert args.compilation_config.level == 3
44
+
45
+ # set to O 3 (space)
46
+ args = parser.parse_args(["-O", "3"])
47
+ assert args.compilation_config.level == 3
48
+
49
+ # set to O 3 (equals)
50
+ args = parser.parse_args(["-O=3"])
51
+ assert args.compilation_config.level == 3
52
+
53
+ # set to json
54
+ args = parser.parse_args(["--compilation-config", '{"level": 3}'])
55
+ assert args.compilation_config.level == 3
56
+
57
+ # set to json
58
+ args = parser.parse_args(['--compilation-config={"level": 3}'])
59
+ assert args.compilation_config.level == 3
60
+
61
+
62
def test_valid_pooling_config():
63
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
64
args = parser.parse_args([
tests/tpu/test_custom_dispatcher.py
+5
-4
@@ -13,9 +13,10 @@ os.environ["VLLM_RPC_TIMEOUT"] = "30000"
13
def test_custom_dispatcher():
14
compare_two_settings(
15
"google/gemma-2b",
16
- arg1=["--enforce-eager", "-O",
17
- str(CompilationLevel.DYNAMO_ONCE)],
18
- arg2=["--enforce-eager", "-O",
19
- str(CompilationLevel.DYNAMO_AS_IS)],
16
+ arg1=[
17
+ "--enforce-eager",
18
+ f"-O{CompilationLevel.DYNAMO_ONCE}",
19
+ ],
20
+ arg2=["--enforce-eager", f"-O{CompilationLevel.DYNAMO_AS_IS}"],
21
env1={},
22
env2={})
vllm/engine/arg_utils.py
+4
-1
@@ -882,7 +882,10 @@ class EngineArgs:
882
'testing only. level 3 is the recommended level '
883
'for production.\n'
884
'To specify the full compilation config, '
885
- 'use a JSON string.')
885
+ 'use a JSON string.\n'
886
+ 'Following the convention of traditional '
887
+ 'compilers, using -O without space is also '
888
+ 'supported. -O3 is equivalent to -O 3.')
889
890
return parser
891
vllm/utils.py
+4
@@ -1192,6 +1192,10 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1192
else:
1193
processed_args.append('--' +
1194
arg[len('--'):].replace('_', '-'))
1195
+ elif arg.startswith('-O') and arg != '-O' and len(arg) == 2:
1196
+ # allow -O flag to be used without space, e.g. -O3
1197
+ processed_args.append('-O')
1198
+ processed_args.append(arg[2:])
1199
else:
1200
processed_args.append(arg)
1201