A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Feature] specify model in config.yaml (#15798)
Signed-off-by: weizeng <weizeng@roblox.com>
Wei Zeng committed
Apr 1, 2025 at 01:20 UTC
30d6a015e07bc334f1d408ee343891f3aeb976aa
7 files changed
+109
-32
docs/source/serving/openai_compatible_server.md
+3
-1
@@ -188,6 +188,7 @@ For example:
188
```yaml
189
# config.yaml
190
191
+model: meta-llama/Llama-3.1-8B-Instruct
192
host: "127.0.0.1"
193
port: 6379
194
uvicorn-log-level: "info"
@@ -196,12 +197,13 @@ uvicorn-log-level: "info"
197
To use the above config file:
198
199
```bash
199
-vllm serve SOME_MODEL --config config.yaml
200
+vllm serve --config config.yaml
201
```
202
203
:::{note}
204
In case an argument is supplied simultaneously using command line and the config file, the value from the command line will take precedence.
205
The order of priorities is `command line > config file values > defaults`.
206
+e.g. `vllm serve SOME_MODEL --config config.yaml`, SOME_MODEL takes precedence over `model` in config file.
207
:::
208
209
## API Reference
tests/config/test_config.yaml
renamed
tests/config/test_config_with_model.yaml
new
+7
@@ -0,0 +1,7 @@
1
+# Same as test_config.yaml but with model specified
2
+model: config-model
3
+port: 12312
4
+served_model_name: mymodel
5
+tensor_parallel_size: 2
6
+trust_remote_code: true
7
+multi_step_stream_outputs: false
tests/conftest.py
+12
@@ -1117,3 +1117,15 @@ def pytest_collection_modifyitems(config, items):
1117
for item in items:
1118
if "optional" in item.keywords:
1119
item.add_marker(skip_optional)
1120
+
1121
+
1122
+@pytest.fixture(scope="session")
1123
+def cli_config_file():
1124
+ """Return the path to the CLI config file."""
1125
+ return os.path.join(_TEST_DIR, "config", "test_config.yaml")
1126
+
1127
+
1128
+@pytest.fixture(scope="session")
1129
+def cli_config_file_with_model():
1130
+ """Return the path to the CLI config file with model."""
1131
+ return os.path.join(_TEST_DIR, "config", "test_config_with_model.yaml")
tests/test_utils.py
+53
-11
@@ -10,7 +10,7 @@ from unittest.mock import patch
10
11
import pytest
12
import torch
13
-from vllm_test_utils import monitor
13
+from vllm_test_utils.monitor import monitor
14
15
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
16
from vllm.utils import (FlexibleArgumentParser, MemorySnapshot,
@@ -143,7 +143,8 @@ def parser():
143
def parser_with_config():
144
parser = FlexibleArgumentParser()
145
parser.add_argument('serve')
146
- parser.add_argument('model_tag')
146
+ parser.add_argument('model_tag', nargs='?')
147
+ parser.add_argument('--model', type=str)
148
parser.add_argument('--served-model-name', type=str)
149
parser.add_argument('--config', type=str)
150
parser.add_argument('--port', type=int)
@@ -199,29 +200,29 @@ def test_missing_required_argument(parser):
200
parser.parse_args([])
201
202
202
-def test_cli_override_to_config(parser_with_config):
203
+def test_cli_override_to_config(parser_with_config, cli_config_file):
204
args = parser_with_config.parse_args([
204
- 'serve', 'mymodel', '--config', './data/test_config.yaml',
205
+ 'serve', 'mymodel', '--config', cli_config_file,
206
'--tensor-parallel-size', '3'
207
])
208
assert args.tensor_parallel_size == 3
209
args = parser_with_config.parse_args([
210
'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
210
- './data/test_config.yaml'
211
+ cli_config_file
212
])
213
assert args.tensor_parallel_size == 3
214
assert args.port == 12312
215
args = parser_with_config.parse_args([
216
'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
216
- './data/test_config.yaml', '--port', '666'
217
+ cli_config_file, '--port', '666'
218
])
219
assert args.tensor_parallel_size == 3
220
assert args.port == 666
221
222
222
-def test_config_args(parser_with_config):
223
+def test_config_args(parser_with_config, cli_config_file):
224
args = parser_with_config.parse_args(
224
- ['serve', 'mymodel', '--config', './data/test_config.yaml'])
225
+ ['serve', 'mymodel', '--config', cli_config_file])
226
assert args.tensor_parallel_size == 2
227
assert args.trust_remote_code
228
assert not args.multi_step_stream_outputs
@@ -243,10 +244,9 @@ def test_config_file(parser_with_config):
244
])
245
246
246
-def test_no_model_tag(parser_with_config):
247
+def test_no_model_tag(parser_with_config, cli_config_file):
248
with pytest.raises(ValueError):
248
- parser_with_config.parse_args(
249
- ['serve', '--config', './data/test_config.yaml'])
249
+ parser_with_config.parse_args(['serve', '--config', cli_config_file])
250
251
252
# yapf: enable
@@ -480,6 +480,48 @@ def test_swap_dict_values(obj, key1, key2):
480
else:
481
assert key1 not in obj
482
483
+
484
+def test_model_specification(parser_with_config,
485
+ cli_config_file,
486
+ cli_config_file_with_model):
487
+ # Test model in CLI takes precedence over config
488
+ args = parser_with_config.parse_args([
489
+ 'serve', 'cli-model', '--config', cli_config_file_with_model
490
+ ])
491
+ assert args.model_tag == 'cli-model'
492
+ assert args.served_model_name == 'mymodel'
493
+
494
+ # Test model from config file works
495
+ args = parser_with_config.parse_args([
496
+ 'serve', '--config', cli_config_file_with_model,
497
+ ])
498
+ assert args.model == 'config-model'
499
+ assert args.served_model_name == 'mymodel'
500
+
501
+ # Test no model specified anywhere raises error
502
+ with pytest.raises(ValueError, match="No model specified!"):
503
+ parser_with_config.parse_args(['serve', '--config', cli_config_file])
504
+
505
+ # Test using --model option raises error
506
+ with pytest.raises(
507
+ ValueError,
508
+ match=(
509
+ "With `vllm serve`, you should provide the model as a positional "
510
+ "argument or in a config file instead of via the `--model` option."
511
+ ),
512
+ ):
513
+ parser_with_config.parse_args(['serve', '--model', 'my-model'])
514
+
515
+ # Test other config values are preserved
516
+ args = parser_with_config.parse_args([
517
+ 'serve', 'cli-model', '--config', cli_config_file_with_model,
518
+ ])
519
+ assert args.tensor_parallel_size == 2
520
+ assert args.trust_remote_code is True
521
+ assert args.multi_step_stream_outputs is False
522
+ assert args.port == 12312
523
+
524
+
525
@pytest.mark.parametrize("input", [(), ("abc", ), (None, ),
526
(None, bool, [1, 2, 3])])
527
@pytest.mark.parametrize("output", [0, 1, 2])
vllm/entrypoints/cli/serve.py
+7
-11
@@ -4,7 +4,6 @@ import argparse
4
5
import uvloop
6
7
-from vllm.engine.arg_utils import EngineArgs
7
from vllm.entrypoints.cli.types import CLISubcommand
8
from vllm.entrypoints.openai.api_server import run_server
9
from vllm.entrypoints.openai.cli_args import (make_arg_parser,
@@ -21,14 +20,9 @@ class ServeSubcommand(CLISubcommand):
20
21
@staticmethod
22
def cmd(args: argparse.Namespace) -> None:
24
- # The default value of `--model`
25
- if args.model != EngineArgs.model:
26
- raise ValueError(
27
- "With `vllm serve`, you should provide the model as a "
28
- "positional argument instead of via the `--model` option.")
29
-
30
- # EngineArgs expects the model name to be passed as --model.
31
- args.model = args.model_tag
23
+ # If model is specified in CLI (as positional arg), it takes precedence
24
+ if hasattr(args, 'model_tag') and args.model_tag is not None:
25
+ args.model = args.model_tag
26
27
uvloop.run(run_server(args))
28
@@ -41,10 +35,12 @@ class ServeSubcommand(CLISubcommand):
35
serve_parser = subparsers.add_parser(
36
"serve",
37
help="Start the vLLM OpenAI Compatible API server",
44
- usage="vllm serve <model_tag> [options]")
38
+ usage="vllm serve [model_tag] [options]")
39
serve_parser.add_argument("model_tag",
40
type=str,
47
- help="The model tag to serve")
41
+ nargs='?',
42
+ help="The model tag to serve "
43
+ "(optional if specified in config)")
44
serve_parser.add_argument(
45
"--config",
46
type=str,
vllm/utils.py
+27
-9
@@ -1241,6 +1241,16 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1241
if args is None:
1242
args = sys.argv[1:]
1243
1244
+ # Check for --model in command line arguments first
1245
+ if args and args[0] == "serve":
1246
+ model_in_cli_args = any(arg == '--model' for arg in args)
1247
+
1248
+ if model_in_cli_args:
1249
+ raise ValueError(
1250
+ "With `vllm serve`, you should provide the model as a "
1251
+ "positional argument or in a config file instead of via "
1252
+ "the `--model` option.")
1253
+
1254
if '--config' in args:
1255
args = self._pull_args_from_config(args)
1256
@@ -1324,19 +1334,29 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1334
config_args = self._load_config_file(file_path)
1335
1336
# 0th index is for {serve,chat,complete}
1327
- # followed by model_tag (only for serve)
1337
+ # optionally followed by model_tag (only for serve)
1338
# followed by config args
1339
# followed by rest of cli args.
1340
# maintaining this order will enforce the precedence
1341
# of cli > config > defaults
1342
if args[0] == "serve":
1333
- if index == 1:
1343
+ model_in_cli = len(args) > 1 and not args[1].startswith('-')
1344
+ model_in_config = any(arg == '--model' for arg in config_args)
1345
+
1346
+ if not model_in_cli and not model_in_config:
1347
raise ValueError(
1335
- "No model_tag specified! Please check your command-line"
1336
- " arguments.")
1337
- args = [args[0]] + [
1338
- args[1]
1339
- ] + config_args + args[2:index] + args[index + 2:]
1348
+ "No model specified! Please specify model either "
1349
+ "as a positional argument or in a config file.")
1350
+
1351
+ if model_in_cli:
1352
+ # Model specified as positional arg, keep CLI version
1353
+ args = [args[0]] + [
1354
+ args[1]
1355
+ ] + config_args + args[2:index] + args[index + 2:]
1356
+ else:
1357
+ # No model in CLI, use config if available
1358
+ args = [args[0]
1359
+ ] + config_args + args[1:index] + args[index + 2:]
1360
else:
1361
args = [args[0]] + config_args + args[1:index] + args[index + 2:]
1362
@@ -1354,9 +1374,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1374
'--port': '12323',
1375
'--tensor-parallel-size': '4'
1376
]
1357
-
1377
"""
1359
-
1378
extension: str = file_path.split('.')[-1]
1379
if extension not in ('yaml', 'yml'):
1380
raise ValueError(