@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5df1834895
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fix order of arguments matters in config.yaml (#8960)

Andy Dai committed Oct 5, 2024 at 10:35 UTC 5df183489537a155bbaad9232f25b8e57694d7b8
4 files changed +36 -9
docs/source/serving/openai_compatible_server.md
+1 -1
@@ -140,7 +140,7 @@ $ vllm serve SOME_MODEL --config config.yaml
140 ```
141 ---
142 **NOTE**
143 -In case an argument is supplied using command line and the config file, the value from the commandline will take precedence.
143 +In case an argument is supplied simultaneously using command line and the config file, the value from the commandline will take precedence.
144 The order of priorities is `command line > config file values > defaults`.
145
146 ---
tests/data/test_config.yaml
+1
@@ -1,2 +1,3 @@
1 port: 12312
2 +served_model_name: mymodel
3 tensor_parallel_size: 2
tests/test_utils.py
+23 -7
@@ -136,6 +136,8 @@ def parser():
136 def parser_with_config():
137 parser = FlexibleArgumentParser()
138 parser.add_argument('serve')
139 + parser.add_argument('model_tag')
140 + parser.add_argument('--served-model-name', type=str)
141 parser.add_argument('--config', type=str)
142 parser.add_argument('--port', type=int)
143 parser.add_argument('--tensor-parallel-size', type=int)
@@ -190,33 +192,47 @@ def test_missing_required_argument(parser):
192
193 def test_cli_override_to_config(parser_with_config):
194 args = parser_with_config.parse_args([
193 - 'serve', '--config', './data/test_config.yaml',
195 + 'serve', 'mymodel', '--config', './data/test_config.yaml',
196 '--tensor-parallel-size', '3'
197 ])
198 assert args.tensor_parallel_size == 3
199 args = parser_with_config.parse_args([
198 - 'serve', '--tensor-parallel-size', '3', '--config',
200 + 'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
201 './data/test_config.yaml'
202 ])
203 assert args.tensor_parallel_size == 3
204 + assert args.port == 12312
205 + args = parser_with_config.parse_args([
206 + 'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
207 + './data/test_config.yaml', '--port', '666'
208 + ])
209 + assert args.tensor_parallel_size == 3
210 + assert args.port == 666
211
212
213 def test_config_args(parser_with_config):
214 args = parser_with_config.parse_args(
206 - ['serve', '--config', './data/test_config.yaml'])
215 + ['serve', 'mymodel', '--config', './data/test_config.yaml'])
216 assert args.tensor_parallel_size == 2
217
218
219 def test_config_file(parser_with_config):
220 with pytest.raises(FileNotFoundError):
212 - parser_with_config.parse_args(['serve', '--config', 'test_config.yml'])
221 + parser_with_config.parse_args(
222 + ['serve', 'mymodel', '--config', 'test_config.yml'])
223
224 with pytest.raises(ValueError):
225 parser_with_config.parse_args(
216 - ['serve', '--config', './data/test_config.json'])
226 + ['serve', 'mymodel', '--config', './data/test_config.json'])
227
228 with pytest.raises(ValueError):
229 parser_with_config.parse_args([
220 - 'serve', '--tensor-parallel-size', '3', '--config', '--batch-size',
221 - '32'
230 + 'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
231 + '--batch-size', '32'
232 ])
233 +
234 +
235 +def test_no_model_tag(parser_with_config):
236 + with pytest.raises(ValueError):
237 + parser_with_config.parse_args(
238 + ['serve', '--config', './data/test_config.yaml'])
vllm/utils.py
+11 -1
@@ -1201,11 +1201,21 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1201 config_args = FlexibleArgumentParser._load_config_file(file_path)
1202
1203 # 0th index is for {serve,chat,complete}
1204 + # followed by model_tag (only for serve)
1205 # followed by config args
1206 # followed by rest of cli args.
1207 # maintaining this order will enforce the precedence
1208 # of cli > config > defaults
1208 - args = [args[0]] + config_args + args[1:index] + args[index + 2:]
1209 + if args[0] == "serve":
1210 + if index == 1:
1211 + raise ValueError(
1212 + "No model_tag specified! Please check your command-line"
1213 + " arguments.")
1214 + args = [args[0]] + [
1215 + args[1]
1216 + ] + config_args + args[2:index] + args[index + 2:]
1217 + else:
1218 + args = [args[0]] + config_args + args[1:index] + args[index + 2:]
1219
1220 return args
1221