A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Frontend]-config-cli-args (#7737)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Kaunil Dhruv <kaunil_dhruv@intuit.com>
Kaunil Dhruv committed
Aug 30, 2024 at 08:21 UTC
058344f89a6594b560e2bb4925daed3f373c3fbc
7 files changed
+185
-1
docs/requirements-docs.txt
+2
-1
@@ -11,5 +11,6 @@ pydantic >= 2.8
11
torch
12
py-cpuinfo
13
transformers
14
-mistral_common >= 1.3.4
14
openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
15
+mistral_common >= 1.3.4
16
+openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
docs/source/serving/openai_compatible_server.md
+26
@@ -111,6 +111,32 @@ directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
111
:prog: vllm serve
112
```
113
114
+### Config file
115
+
116
+The `serve` module can also accept arguments from a config file in
117
+`yaml` format. The arguments in the yaml must be specified using the
118
+long form of the argument outlined [here](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#command-line-arguments-for-the-server):
119
+
120
+For example:
121
+
122
+```yaml
123
+# config.yaml
124
+
125
+host: "127.0.0.1"
126
+port: 6379
127
+uvicorn-log-level: "info"
128
+```
129
+
130
+```bash
131
+$ vllm serve SOME_MODEL --config config.yaml
132
+```
133
+---
134
+**NOTE**
135
+In case an argument is supplied using command line and the config file, the value from the commandline will take precedence.
136
+The order of priorities is `command line > config file values > defaults`.
137
+
138
+---
139
+
140
## Tool calling in the chat completion API
141
vLLM supports only named function calling in the chat completion API. The `tool_choice` options `auto` and `required` are **not yet supported** but on the roadmap.
142
requirements-common.txt
+1
@@ -27,3 +27,4 @@ soundfile # Required for audio processing
27
gguf == 0.9.1
28
importlib_metadata
29
mistral_common >= 1.3.4
30
+pyyaml
\ No newline at end of file
tests/data/test_config.yaml
new
+2
@@ -0,0 +1,2 @@
1
+port: 12312
2
+tensor_parallel_size: 2
tests/test_utils.py
+44
@@ -132,6 +132,16 @@ def parser():
132
return parser
133
134
135
+@pytest.fixture
136
+def parser_with_config():
137
+ parser = FlexibleArgumentParser()
138
+ parser.add_argument('serve')
139
+ parser.add_argument('--config', type=str)
140
+ parser.add_argument('--port', type=int)
141
+ parser.add_argument('--tensor-parallel-size', type=int)
142
+ return parser
143
+
144
+
145
def test_underscore_to_dash(parser):
146
args = parser.parse_args(['--image_input_type', 'pixel_values'])
147
assert args.image_input_type == 'pixel_values'
@@ -176,3 +186,37 @@ def test_missing_required_argument(parser):
186
parser.add_argument('--required-arg', required=True)
187
with pytest.raises(SystemExit):
188
parser.parse_args([])
189
+
190
+
191
+def test_cli_override_to_config(parser_with_config):
192
+ args = parser_with_config.parse_args([
193
+ 'serve', '--config', './data/test_config.yaml',
194
+ '--tensor-parallel-size', '3'
195
+ ])
196
+ assert args.tensor_parallel_size == 3
197
+ args = parser_with_config.parse_args([
198
+ 'serve', '--tensor-parallel-size', '3', '--config',
199
+ './data/test_config.yaml'
200
+ ])
201
+ assert args.tensor_parallel_size == 3
202
+
203
+
204
+def test_config_args(parser_with_config):
205
+ args = parser_with_config.parse_args(
206
+ ['serve', '--config', './data/test_config.yaml'])
207
+ assert args.tensor_parallel_size == 2
208
+
209
+
210
+def test_config_file(parser_with_config):
211
+ with pytest.raises(FileNotFoundError):
212
+ parser_with_config.parse_args(['serve', '--config', 'test_config.yml'])
213
+
214
+ with pytest.raises(ValueError):
215
+ parser_with_config.parse_args(
216
+ ['serve', '--config', './data/test_config.json'])
217
+
218
+ with pytest.raises(ValueError):
219
+ parser_with_config.parse_args([
220
+ 'serve', '--tensor-parallel-size', '3', '--config', '--batch-size',
221
+ '32'
222
+ ])
vllm/scripts.py
+9
@@ -125,6 +125,15 @@ def main():
125
serve_parser.add_argument("model_tag",
126
type=str,
127
help="The model tag to serve")
128
+ serve_parser.add_argument(
129
+ "--config",
130
+ type=str,
131
+ default='',
132
+ required=False,
133
+ help="Read CLI options from a config file."
134
+ "Must be a YAML with the following options:"
135
+ "https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#command-line-arguments-for-the-server"
136
+ )
137
serve_parser = make_arg_parser(serve_parser)
138
serve_parser.set_defaults(dispatch_function=serve)
139
vllm/utils.py
+101
@@ -25,6 +25,7 @@ import numpy.typing as npt
25
import psutil
26
import torch
27
import torch.types
28
+import yaml
29
from packaging.version import Version
30
from typing_extensions import ParamSpec, TypeIs, assert_never
31
@@ -1093,6 +1094,9 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1094
if args is None:
1095
args = sys.argv[1:]
1096
1097
+ if '--config' in args:
1098
+ args = FlexibleArgumentParser._pull_args_from_config(args)
1099
+
1100
# Convert underscores to dashes and vice versa in argument names
1101
processed_args = []
1102
for arg in args:
@@ -1109,6 +1113,103 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1113
1114
return super().parse_args(processed_args, namespace)
1115
1116
+ @staticmethod
1117
+ def _pull_args_from_config(args: List[str]) -> List[str]:
1118
+ """Method to pull arguments specified in the config file
1119
+ into the command-line args variable.
1120
+
1121
+ The arguments in config file will be inserted between
1122
+ the argument list.
1123
+
1124
+ example:
1125
+ ```yaml
1126
+ port: 12323
1127
+ tensor-parallel-size: 4
1128
+ ```
1129
+ ```python
1130
+ $: vllm {serve,chat,complete} "facebook/opt-12B" \
1131
+ --config config.yaml -tp 2
1132
+ $: args = [
1133
+ "serve,chat,complete",
1134
+ "facebook/opt-12B",
1135
+ '--config', 'config.yaml',
1136
+ '-tp', '2'
1137
+ ]
1138
+ $: args = [
1139
+ "serve,chat,complete",
1140
+ "facebook/opt-12B",
1141
+ '--port', '12323',
1142
+ '--tensor-parallel-size', '4',
1143
+ '-tp', '2'
1144
+ ]
1145
+ ```
1146
+
1147
+ Please note how the config args are inserted after the sub command.
1148
+ this way the order of priorities is maintained when these are args
1149
+ parsed by super().
1150
+ """
1151
+ assert args.count(
1152
+ '--config') <= 1, "More than one config file specified!"
1153
+
1154
+ index = args.index('--config')
1155
+ if index == len(args) - 1:
1156
+ raise ValueError("No config file specified! \
1157
+ Please check your command-line arguments.")
1158
+
1159
+ file_path = args[index + 1]
1160
+
1161
+ config_args = FlexibleArgumentParser._load_config_file(file_path)
1162
+
1163
+ # 0th index is for {serve,chat,complete}
1164
+ # followed by config args
1165
+ # followed by rest of cli args.
1166
+ # maintaining this order will enforce the precedence
1167
+ # of cli > config > defaults
1168
+ args = [args[0]] + config_args + args[1:index] + args[index + 2:]
1169
+
1170
+ return args
1171
+
1172
+ @staticmethod
1173
+ def _load_config_file(file_path: str) -> List[str]:
1174
+ """Loads a yaml file and returns the key value pairs as a
1175
+ flattened list with argparse like pattern
1176
+ ```yaml
1177
+ port: 12323
1178
+ tensor-parallel-size: 4
1179
+ ```
1180
+ returns:
1181
+ processed_args: list[str] = [
1182
+ '--port': '12323',
1183
+ '--tensor-parallel-size': '4'
1184
+ ]
1185
+
1186
+ """
1187
+
1188
+ extension: str = file_path.split('.')[-1]
1189
+ if extension not in ('yaml', 'yml'):
1190
+ raise ValueError(
1191
+ "Config file must be of a yaml/yml type.\
1192
+ %s supplied", extension)
1193
+
1194
+ # only expecting a flat dictionary of atomic types
1195
+ processed_args: List[str] = []
1196
+
1197
+ config: Dict[str, Union[int, str]] = {}
1198
+ try:
1199
+ with open(file_path, 'r') as config_file:
1200
+ config = yaml.safe_load(config_file)
1201
+ except Exception as ex:
1202
+ logger.error(
1203
+ "Unable to read the config file at %s. \
1204
+ Make sure path is correct", file_path)
1205
+ raise ex
1206
+
1207
+ for key, value in config.items():
1208
+ processed_args.append('--' + key)
1209
+ processed_args.append(str(value))
1210
+
1211
+ return processed_args
1212
+
1213
1214
async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1215
**kwargs):