A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix order of arguments matters in config.yaml (#8960)
Andy Dai committed
Oct 5, 2024 at 10:35 UTC
5df183489537a155bbaad9232f25b8e57694d7b8
4 files changed
+36
-9
docs/source/serving/openai_compatible_server.md
+1
-1
@@ -140,7 +140,7 @@ $ vllm serve SOME_MODEL --config config.yaml
140
```
141
---
142
**NOTE**
143
-In case an argument is supplied using command line and the config file, the value from the commandline will take precedence.
143
+In case an argument is supplied simultaneously using command line and the config file, the value from the commandline will take precedence.
144
The order of priorities is `command line > config file values > defaults`.
145
146
---
tests/data/test_config.yaml
+1
@@ -1,2 +1,3 @@
1
port: 12312
2
+served_model_name: mymodel
3
tensor_parallel_size: 2
tests/test_utils.py
+23
-7
@@ -136,6 +136,8 @@ def parser():
136
def parser_with_config():
137
parser = FlexibleArgumentParser()
138
parser.add_argument('serve')
139
+ parser.add_argument('model_tag')
140
+ parser.add_argument('--served-model-name', type=str)
141
parser.add_argument('--config', type=str)
142
parser.add_argument('--port', type=int)
143
parser.add_argument('--tensor-parallel-size', type=int)
@@ -190,33 +192,47 @@ def test_missing_required_argument(parser):
192
193
def test_cli_override_to_config(parser_with_config):
194
args = parser_with_config.parse_args([
193
- 'serve', '--config', './data/test_config.yaml',
195
+ 'serve', 'mymodel', '--config', './data/test_config.yaml',
196
'--tensor-parallel-size', '3'
197
])
198
assert args.tensor_parallel_size == 3
199
args = parser_with_config.parse_args([
198
- 'serve', '--tensor-parallel-size', '3', '--config',
200
+ 'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
201
'./data/test_config.yaml'
202
])
203
assert args.tensor_parallel_size == 3
204
+ assert args.port == 12312
205
+ args = parser_with_config.parse_args([
206
+ 'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
207
+ './data/test_config.yaml', '--port', '666'
208
+ ])
209
+ assert args.tensor_parallel_size == 3
210
+ assert args.port == 666
211
212
213
def test_config_args(parser_with_config):
214
args = parser_with_config.parse_args(
206
- ['serve', '--config', './data/test_config.yaml'])
215
+ ['serve', 'mymodel', '--config', './data/test_config.yaml'])
216
assert args.tensor_parallel_size == 2
217
218
219
def test_config_file(parser_with_config):
220
with pytest.raises(FileNotFoundError):
212
- parser_with_config.parse_args(['serve', '--config', 'test_config.yml'])
221
+ parser_with_config.parse_args(
222
+ ['serve', 'mymodel', '--config', 'test_config.yml'])
223
224
with pytest.raises(ValueError):
225
parser_with_config.parse_args(
216
- ['serve', '--config', './data/test_config.json'])
226
+ ['serve', 'mymodel', '--config', './data/test_config.json'])
227
228
with pytest.raises(ValueError):
229
parser_with_config.parse_args([
220
- 'serve', '--tensor-parallel-size', '3', '--config', '--batch-size',
221
- '32'
230
+ 'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
231
+ '--batch-size', '32'
232
])
233
+
234
+
235
+def test_no_model_tag(parser_with_config):
236
+ with pytest.raises(ValueError):
237
+ parser_with_config.parse_args(
238
+ ['serve', '--config', './data/test_config.yaml'])
vllm/utils.py
+11
-1
@@ -1201,11 +1201,21 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1201
config_args = FlexibleArgumentParser._load_config_file(file_path)
1202
1203
# 0th index is for {serve,chat,complete}
1204
+ # followed by model_tag (only for serve)
1205
# followed by config args
1206
# followed by rest of cli args.
1207
# maintaining this order will enforce the precedence
1208
# of cli > config > defaults
1208
- args = [args[0]] + config_args + args[1:index] + args[index + 2:]
1209
+ if args[0] == "serve":
1210
+ if index == 1:
1211
+ raise ValueError(
1212
+ "No model_tag specified! Please check your command-line"
1213
+ " arguments.")
1214
+ args = [args[0]] + [
1215
+ args[1]
1216
+ ] + config_args + args[2:index] + args[index + 2:]
1217
+ else:
1218
+ args = [args[0]] + config_args + args[1:index] + args[index + 2:]
1219
1220
return args
1221