A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
Revert "[Feature] specify model in config.yaml (#14855)" (#15293)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 21, 2025 at 23:30 UTC
baec0d4de9d202a4d91b8df0c3f1c53a4ddb0a09
7 files changed
+30
-102
docs/source/serving/openai_compatible_server.md
+1
-3
@@ -184,7 +184,6 @@ For example:
184
```yaml
185
# config.yaml
186
187
-model: meta-llama/Llama-3.1-8B-Instruct
187
host: "127.0.0.1"
188
port: 6379
189
uvicorn-log-level: "info"
@@ -193,13 +192,12 @@ uvicorn-log-level: "info"
192
To use the above config file:
193
194
```bash
196
-vllm serve --config config.yaml
195
+vllm serve SOME_MODEL --config config.yaml
196
```
197
198
:::{note}
199
In case an argument is supplied simultaneously using command line and the config file, the value from the command line will take precedence.
200
The order of priorities is `command line > config file values > defaults`.
202
-e.g. `vllm serve SOME_MODEL --config config.yaml`, SOME_MODEL takes precedence over `model` in config file.
201
:::
202
203
## API Reference
tests/config/test_config_with_model.yaml
deleted
-7
@@ -1,7 +0,0 @@
1
-# Same as test_config.yaml but with model specified
2
-model: config-model
3
-port: 12312
4
-served_model_name: mymodel
5
-tensor_parallel_size: 2
6
-trust_remote_code: true
7
-multi_step_stream_outputs: false
tests/conftest.py
-12
@@ -1121,15 +1121,3 @@ def pytest_collection_modifyitems(config, items):
1121
for item in items:
1122
if "optional" in item.keywords:
1123
item.add_marker(skip_optional)
1124
-
1125
-
1126
-@pytest.fixture(scope="session")
1127
-def cli_config_file():
1128
- """Return the path to the CLI config file."""
1129
- return os.path.join(_TEST_DIR, "config", "test_config.yaml")
1130
-
1131
-
1132
-@pytest.fixture(scope="session")
1133
-def cli_config_file_with_model():
1134
- """Return the path to the CLI config file with model."""
1135
- return os.path.join(_TEST_DIR, "config", "test_config_with_model.yaml")
tests/data/test_config.yaml
renamed
tests/test_utils.py
+11
-42
@@ -8,7 +8,7 @@ from unittest.mock import patch
8
9
import pytest
10
import torch
11
-from vllm_test_utils.monitor import monitor
11
+from vllm_test_utils import monitor
12
13
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
14
from vllm.utils import (FlexibleArgumentParser, MemorySnapshot,
@@ -140,8 +140,7 @@ def parser():
140
def parser_with_config():
141
parser = FlexibleArgumentParser()
142
parser.add_argument('serve')
143
- parser.add_argument('model_tag', nargs='?')
144
- parser.add_argument('--model', type=str)
143
+ parser.add_argument('model_tag')
144
parser.add_argument('--served-model-name', type=str)
145
parser.add_argument('--config', type=str)
146
parser.add_argument('--port', type=int)
@@ -197,29 +196,29 @@ def test_missing_required_argument(parser):
196
parser.parse_args([])
197
198
200
-def test_cli_override_to_config(parser_with_config, cli_config_file):
199
+def test_cli_override_to_config(parser_with_config):
200
args = parser_with_config.parse_args([
202
- 'serve', 'mymodel', '--config', cli_config_file,
201
+ 'serve', 'mymodel', '--config', './data/test_config.yaml',
202
'--tensor-parallel-size', '3'
203
])
204
assert args.tensor_parallel_size == 3
205
args = parser_with_config.parse_args([
206
'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
208
- cli_config_file
207
+ './data/test_config.yaml'
208
])
209
assert args.tensor_parallel_size == 3
210
assert args.port == 12312
211
args = parser_with_config.parse_args([
212
'serve', 'mymodel', '--tensor-parallel-size', '3', '--config',
214
- cli_config_file, '--port', '666'
213
+ './data/test_config.yaml', '--port', '666'
214
])
215
assert args.tensor_parallel_size == 3
216
assert args.port == 666
217
218
220
-def test_config_args(parser_with_config, cli_config_file):
219
+def test_config_args(parser_with_config):
220
args = parser_with_config.parse_args(
222
- ['serve', 'mymodel', '--config', cli_config_file])
221
+ ['serve', 'mymodel', '--config', './data/test_config.yaml'])
222
assert args.tensor_parallel_size == 2
223
assert args.trust_remote_code
224
assert not args.multi_step_stream_outputs
@@ -241,9 +240,10 @@ def test_config_file(parser_with_config):
240
])
241
242
244
-def test_no_model_tag(parser_with_config, cli_config_file):
243
+def test_no_model_tag(parser_with_config):
244
with pytest.raises(ValueError):
246
- parser_with_config.parse_args(['serve', '--config', cli_config_file])
245
+ parser_with_config.parse_args(
246
+ ['serve', '--config', './data/test_config.yaml'])
247
248
249
# yapf: enable
@@ -476,34 +476,3 @@ def test_swap_dict_values(obj, key1, key2):
476
assert obj[key1] == original_obj[key2]
477
else:
478
assert key1 not in obj
479
-
480
-
481
-def test_model_specification(parser_with_config,
482
- cli_config_file,
483
- cli_config_file_with_model):
484
- # Test model in CLI takes precedence over config
485
- args = parser_with_config.parse_args([
486
- 'serve', 'cli-model', '--config', cli_config_file_with_model
487
- ])
488
- assert args.model_tag == 'cli-model'
489
- assert args.served_model_name == 'mymodel'
490
-
491
- # Test model from config file works
492
- args = parser_with_config.parse_args([
493
- 'serve', '--config', cli_config_file_with_model
494
- ])
495
- assert args.model == 'config-model'
496
- assert args.served_model_name == 'mymodel'
497
-
498
- # Test no model specified anywhere raises error
499
- with pytest.raises(ValueError, match="No model specified!"):
500
- parser_with_config.parse_args(['serve', '--config', cli_config_file])
501
-
502
- # Test other config values are preserved
503
- args = parser_with_config.parse_args([
504
- 'serve', 'cli-model', '--config', cli_config_file_with_model
505
- ])
506
- assert args.tensor_parallel_size == 2
507
- assert args.trust_remote_code is True
508
- assert args.multi_step_stream_outputs is False
509
- assert args.port == 12312
vllm/entrypoints/cli/serve.py
+9
-13
@@ -21,16 +21,14 @@ class ServeSubcommand(CLISubcommand):
21
22
@staticmethod
23
def cmd(args: argparse.Namespace) -> None:
24
- # If model is specified in CLI (as positional arg), it takes precedence
25
- if hasattr(args, 'model_tag') and args.model_tag is not None:
26
- args.model = args.model_tag
27
- # Otherwise use model from config (already in args.model)
28
-
29
- # Check if we have a model specified somewhere
30
- if args.model == EngineArgs.model: # Still has default value
24
+ # The default value of `--model`
25
+ if args.model != EngineArgs.model:
26
raise ValueError(
32
- "With `vllm serve`, you should provide the model either as a "
33
- "positional argument or in config file.")
27
+ "With `vllm serve`, you should provide the model as a "
28
+ "positional argument instead of via the `--model` option.")
29
+
30
+ # EngineArgs expects the model name to be passed as --model.
31
+ args.model = args.model_tag
32
33
uvloop.run(run_server(args))
34
@@ -43,12 +41,10 @@ class ServeSubcommand(CLISubcommand):
41
serve_parser = subparsers.add_parser(
42
"serve",
43
help="Start the vLLM OpenAI Compatible API server",
46
- usage="vllm serve [model_tag] [options]")
44
+ usage="vllm serve <model_tag> [options]")
45
serve_parser.add_argument("model_tag",
46
type=str,
49
- nargs='?',
50
- help="The model tag to serve "
51
- "(optional if specified in config)")
47
+ help="The model tag to serve")
48
serve_parser.add_argument(
49
"--config",
50
type=str,
vllm/utils.py
+9
-25
@@ -1265,29 +1265,19 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1265
config_args = self._load_config_file(file_path)
1266
1267
# 0th index is for {serve,chat,complete}
1268
- # optionally followed by model_tag (only for serve)
1268
+ # followed by model_tag (only for serve)
1269
# followed by config args
1270
# followed by rest of cli args.
1271
# maintaining this order will enforce the precedence
1272
# of cli > config > defaults
1273
if args[0] == "serve":
1274
- model_in_cli = len(args) > 1 and not args[1].startswith('-')
1275
- model_in_config = any(arg == '--model' for arg in config_args)
1276
-
1277
- if not model_in_cli and not model_in_config:
1274
+ if index == 1:
1275
raise ValueError(
1279
- "No model specified! Please specify model either in "
1280
- "command-line arguments or in config file.")
1281
-
1282
- if model_in_cli:
1283
- # Model specified as positional arg, keep CLI version
1284
- args = [args[0]] + [
1285
- args[1]
1286
- ] + config_args + args[2:index] + args[index + 2:]
1287
- else:
1288
- # No model in CLI, use config if available
1289
- args = [args[0]
1290
- ] + config_args + args[1:index] + args[index + 2:]
1276
+ "No model_tag specified! Please check your command-line"
1277
+ " arguments.")
1278
+ args = [args[0]] + [
1279
+ args[1]
1280
+ ] + config_args + args[2:index] + args[index + 2:]
1281
else:
1282
args = [args[0]] + config_args + args[1:index] + args[index + 2:]
1283
@@ -1305,7 +1295,9 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1295
'--port': '12323',
1296
'--tensor-parallel-size': '4'
1297
]
1298
+
1299
"""
1300
+
1301
extension: str = file_path.split('.')[-1]
1302
if extension not in ('yaml', 'yml'):
1303
raise ValueError(
@@ -1330,15 +1322,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1322
if isinstance(action, StoreBoolean)
1323
]
1324
1333
- # Skip model from config if it's provided as positional argument
1334
- skip_model = (hasattr(self, '_parsed_args') and self._parsed_args
1335
- and len(self._parsed_args) > 1
1336
- and self._parsed_args[0] == 'serve'
1337
- and not self._parsed_args[1].startswith('-'))
1338
-
1325
for key, value in config.items():
1340
- if skip_model and key == 'model':
1341
- continue
1326
if isinstance(value, bool) and key not in store_boolean_arguments:
1327
if value:
1328
processed_args.append('--' + key)