@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 058344f89a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Frontend]-config-cli-args (#7737)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Kaunil Dhruv <kaunil_dhruv@intuit.com>

Kaunil Dhruv committed Aug 30, 2024 at 08:21 UTC 058344f89a6594b560e2bb4925daed3f373c3fbc
7 files changed +185 -1
docs/requirements-docs.txt
+2 -1
@@ -11,5 +11,6 @@ pydantic >= 2.8
11 torch
12 py-cpuinfo
13 transformers
14 -mistral_common >= 1.3.4
14 openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
15 +mistral_common >= 1.3.4
16 +openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
docs/source/serving/openai_compatible_server.md
+26
@@ -111,6 +111,32 @@ directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
111 :prog: vllm serve
112 ```
113
114 +### Config file
115 +
116 +The `serve` module can also accept arguments from a config file in
117 +`yaml` format. The arguments in the yaml must be specified using the
118 +long form of the argument outlined [here](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#command-line-arguments-for-the-server):
119 +
120 +For example:
121 +
122 +```yaml
123 +# config.yaml
124 +
125 +host: "127.0.0.1"
126 +port: 6379
127 +uvicorn-log-level: "info"
128 +```
129 +
130 +```bash
131 +$ vllm serve SOME_MODEL --config config.yaml
132 +```
133 +---
134 +**NOTE**
135 +In case an argument is supplied using command line and the config file, the value from the commandline will take precedence.
136 +The order of priorities is `command line > config file values > defaults`.
137 +
138 +---
139 +
140 ## Tool calling in the chat completion API
141 vLLM supports only named function calling in the chat completion API. The `tool_choice` options `auto` and `required` are **not yet supported** but on the roadmap.
142
requirements-common.txt
+1
@@ -27,3 +27,4 @@ soundfile # Required for audio processing
27 gguf == 0.9.1
28 importlib_metadata
29 mistral_common >= 1.3.4
30 +pyyaml
\ No newline at end of file
tests/data/test_config.yaml new
+2
@@ -0,0 +1,2 @@
1 +port: 12312
2 +tensor_parallel_size: 2
tests/test_utils.py
+44
@@ -132,6 +132,16 @@ def parser():
132 return parser
133
134
135 +@pytest.fixture
136 +def parser_with_config():
137 + parser = FlexibleArgumentParser()
138 + parser.add_argument('serve')
139 + parser.add_argument('--config', type=str)
140 + parser.add_argument('--port', type=int)
141 + parser.add_argument('--tensor-parallel-size', type=int)
142 + return parser
143 +
144 +
145 def test_underscore_to_dash(parser):
146 args = parser.parse_args(['--image_input_type', 'pixel_values'])
147 assert args.image_input_type == 'pixel_values'
@@ -176,3 +186,37 @@ def test_missing_required_argument(parser):
186 parser.add_argument('--required-arg', required=True)
187 with pytest.raises(SystemExit):
188 parser.parse_args([])
189 +
190 +
191 +def test_cli_override_to_config(parser_with_config):
192 + args = parser_with_config.parse_args([
193 + 'serve', '--config', './data/test_config.yaml',
194 + '--tensor-parallel-size', '3'
195 + ])
196 + assert args.tensor_parallel_size == 3
197 + args = parser_with_config.parse_args([
198 + 'serve', '--tensor-parallel-size', '3', '--config',
199 + './data/test_config.yaml'
200 + ])
201 + assert args.tensor_parallel_size == 3
202 +
203 +
204 +def test_config_args(parser_with_config):
205 + args = parser_with_config.parse_args(
206 + ['serve', '--config', './data/test_config.yaml'])
207 + assert args.tensor_parallel_size == 2
208 +
209 +
210 +def test_config_file(parser_with_config):
211 + with pytest.raises(FileNotFoundError):
212 + parser_with_config.parse_args(['serve', '--config', 'test_config.yml'])
213 +
214 + with pytest.raises(ValueError):
215 + parser_with_config.parse_args(
216 + ['serve', '--config', './data/test_config.json'])
217 +
218 + with pytest.raises(ValueError):
219 + parser_with_config.parse_args([
220 + 'serve', '--tensor-parallel-size', '3', '--config', '--batch-size',
221 + '32'
222 + ])
vllm/scripts.py
+9
@@ -125,6 +125,15 @@ def main():
125 serve_parser.add_argument("model_tag",
126 type=str,
127 help="The model tag to serve")
128 + serve_parser.add_argument(
129 + "--config",
130 + type=str,
131 + default='',
132 + required=False,
133 + help="Read CLI options from a config file."
134 + "Must be a YAML with the following options:"
135 + "https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#command-line-arguments-for-the-server"
136 + )
137 serve_parser = make_arg_parser(serve_parser)
138 serve_parser.set_defaults(dispatch_function=serve)
139
vllm/utils.py
+101
@@ -25,6 +25,7 @@ import numpy.typing as npt
25 import psutil
26 import torch
27 import torch.types
28 +import yaml
29 from packaging.version import Version
30 from typing_extensions import ParamSpec, TypeIs, assert_never
31
@@ -1093,6 +1094,9 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1094 if args is None:
1095 args = sys.argv[1:]
1096
1097 + if '--config' in args:
1098 + args = FlexibleArgumentParser._pull_args_from_config(args)
1099 +
1100 # Convert underscores to dashes and vice versa in argument names
1101 processed_args = []
1102 for arg in args:
@@ -1109,6 +1113,103 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1113
1114 return super().parse_args(processed_args, namespace)
1115
1116 + @staticmethod
1117 + def _pull_args_from_config(args: List[str]) -> List[str]:
1118 + """Method to pull arguments specified in the config file
1119 + into the command-line args variable.
1120 +
1121 + The arguments in config file will be inserted between
1122 + the argument list.
1123 +
1124 + example:
1125 + ```yaml
1126 + port: 12323
1127 + tensor-parallel-size: 4
1128 + ```
1129 + ```python
1130 + $: vllm {serve,chat,complete} "facebook/opt-12B" \
1131 + --config config.yaml -tp 2
1132 + $: args = [
1133 + "serve,chat,complete",
1134 + "facebook/opt-12B",
1135 + '--config', 'config.yaml',
1136 + '-tp', '2'
1137 + ]
1138 + $: args = [
1139 + "serve,chat,complete",
1140 + "facebook/opt-12B",
1141 + '--port', '12323',
1142 + '--tensor-parallel-size', '4',
1143 + '-tp', '2'
1144 + ]
1145 + ```
1146 +
1147 + Please note how the config args are inserted after the sub command.
1148 + this way the order of priorities is maintained when these are args
1149 + parsed by super().
1150 + """
1151 + assert args.count(
1152 + '--config') <= 1, "More than one config file specified!"
1153 +
1154 + index = args.index('--config')
1155 + if index == len(args) - 1:
1156 + raise ValueError("No config file specified! \
1157 + Please check your command-line arguments.")
1158 +
1159 + file_path = args[index + 1]
1160 +
1161 + config_args = FlexibleArgumentParser._load_config_file(file_path)
1162 +
1163 + # 0th index is for {serve,chat,complete}
1164 + # followed by config args
1165 + # followed by rest of cli args.
1166 + # maintaining this order will enforce the precedence
1167 + # of cli > config > defaults
1168 + args = [args[0]] + config_args + args[1:index] + args[index + 2:]
1169 +
1170 + return args
1171 +
1172 + @staticmethod
1173 + def _load_config_file(file_path: str) -> List[str]:
1174 + """Loads a yaml file and returns the key value pairs as a
1175 + flattened list with argparse like pattern
1176 + ```yaml
1177 + port: 12323
1178 + tensor-parallel-size: 4
1179 + ```
1180 + returns:
1181 + processed_args: list[str] = [
1182 + '--port': '12323',
1183 + '--tensor-parallel-size': '4'
1184 + ]
1185 +
1186 + """
1187 +
1188 + extension: str = file_path.split('.')[-1]
1189 + if extension not in ('yaml', 'yml'):
1190 + raise ValueError(
1191 + "Config file must be of a yaml/yml type.\
1192 + %s supplied", extension)
1193 +
1194 + # only expecting a flat dictionary of atomic types
1195 + processed_args: List[str] = []
1196 +
1197 + config: Dict[str, Union[int, str]] = {}
1198 + try:
1199 + with open(file_path, 'r') as config_file:
1200 + config = yaml.safe_load(config_file)
1201 + except Exception as ex:
1202 + logger.error(
1203 + "Unable to read the config file at %s. \
1204 + Make sure path is correct", file_path)
1205 + raise ex
1206 +
1207 + for key, value in config.items():
1208 + processed_args.append('--' + key)
1209 + processed_args.append(str(value))
1210 +
1211 + return processed_args
1212 +
1213
1214 async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1215 **kwargs):