A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix load config when using bools (#9533)
madt2709 committed
Oct 27, 2024 at 10:46 UTC
34a9941620d00879599a51609225452b705bae89
4 files changed
+35
-22
tests/data/test_config.yaml
+2
@@ -1,3 +1,5 @@
1
port: 12312
2
served_model_name: mymodel
3
tensor_parallel_size: 2
4
+trust_remote_code: true
5
+multi_step_stream_outputs: false
tests/test_utils.py
+5
-1
@@ -6,7 +6,7 @@ from typing import AsyncIterator, Tuple
6
7
import pytest
8
9
-from vllm.utils import (FlexibleArgumentParser, deprecate_kwargs,
9
+from vllm.utils import (FlexibleArgumentParser, StoreBoolean, deprecate_kwargs,
10
get_open_port, merge_async_iterators, supports_kw)
11
12
from .utils import error_on_warning
@@ -141,6 +141,8 @@ def parser_with_config():
141
parser.add_argument('--config', type=str)
142
parser.add_argument('--port', type=int)
143
parser.add_argument('--tensor-parallel-size', type=int)
144
+ parser.add_argument('--trust-remote-code', action='store_true')
145
+ parser.add_argument('--multi-step-stream-outputs', action=StoreBoolean)
146
return parser
147
148
@@ -214,6 +216,8 @@ def test_config_args(parser_with_config):
216
args = parser_with_config.parse_args(
217
['serve', 'mymodel', '--config', './data/test_config.yaml'])
218
assert args.tensor_parallel_size == 2
219
+ assert args.trust_remote_code
220
+ assert not args.multi_step_stream_outputs
221
222
223
def test_config_file(parser_with_config):
vllm/engine/arg_utils.py
+1
-13
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
19
from vllm.transformers_utils.config import (
20
maybe_register_config_serialize_by_value)
21
from vllm.transformers_utils.utils import check_gguf_file
22
-from vllm.utils import FlexibleArgumentParser
22
+from vllm.utils import FlexibleArgumentParser, StoreBoolean
23
24
if TYPE_CHECKING:
25
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
@@ -1144,18 +1144,6 @@ class AsyncEngineArgs(EngineArgs):
1144
return parser
1145
1146
1147
-class StoreBoolean(argparse.Action):
1148
-
1149
- def __call__(self, parser, namespace, values, option_string=None):
1150
- if values.lower() == "true":
1151
- setattr(namespace, self.dest, True)
1152
- elif values.lower() == "false":
1153
- setattr(namespace, self.dest, False)
1154
- else:
1155
- raise ValueError(f"Invalid boolean value: {values}. "
1156
- "Expected 'true' or 'false'.")
1157
-
1158
-
1147
# These functions are used by sphinx to build the documentation
1148
def _engine_args_parser():
1149
return EngineArgs.add_cli_args(FlexibleArgumentParser())
vllm/utils.py
+27
-8
@@ -1155,6 +1155,18 @@ def run_once(f: Callable[P, None]) -> Callable[P, None]:
1155
return wrapper
1156
1157
1158
+class StoreBoolean(argparse.Action):
1159
+
1160
+ def __call__(self, parser, namespace, values, option_string=None):
1161
+ if values.lower() == "true":
1162
+ setattr(namespace, self.dest, True)
1163
+ elif values.lower() == "false":
1164
+ setattr(namespace, self.dest, False)
1165
+ else:
1166
+ raise ValueError(f"Invalid boolean value: {values}. "
1167
+ "Expected 'true' or 'false'.")
1168
+
1169
+
1170
class FlexibleArgumentParser(argparse.ArgumentParser):
1171
"""ArgumentParser that allows both underscore and dash in names."""
1172
@@ -1163,7 +1175,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1175
args = sys.argv[1:]
1176
1177
if '--config' in args:
1166
- args = FlexibleArgumentParser._pull_args_from_config(args)
1178
+ args = self._pull_args_from_config(args)
1179
1180
# Convert underscores to dashes and vice versa in argument names
1181
processed_args = []
@@ -1181,8 +1193,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1193
1194
return super().parse_args(processed_args, namespace)
1195
1184
- @staticmethod
1185
- def _pull_args_from_config(args: List[str]) -> List[str]:
1196
+ def _pull_args_from_config(self, args: List[str]) -> List[str]:
1197
"""Method to pull arguments specified in the config file
1198
into the command-line args variable.
1199
@@ -1226,7 +1237,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1237
1238
file_path = args[index + 1]
1239
1229
- config_args = FlexibleArgumentParser._load_config_file(file_path)
1240
+ config_args = self._load_config_file(file_path)
1241
1242
# 0th index is for {serve,chat,complete}
1243
# followed by model_tag (only for serve)
@@ -1247,8 +1258,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1258
1259
return args
1260
1250
- @staticmethod
1251
- def _load_config_file(file_path: str) -> List[str]:
1261
+ def _load_config_file(self, file_path: str) -> List[str]:
1262
"""Loads a yaml file and returns the key value pairs as a
1263
flattened list with argparse like pattern
1264
```yaml
@@ -1282,9 +1292,18 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1292
Make sure path is correct", file_path)
1293
raise ex
1294
1295
+ store_boolean_arguments = [
1296
+ action.dest for action in self._actions
1297
+ if isinstance(action, StoreBoolean)
1298
+ ]
1299
+
1300
for key, value in config.items():
1286
- processed_args.append('--' + key)
1287
- processed_args.append(str(value))
1301
+ if isinstance(value, bool) and key not in store_boolean_arguments:
1302
+ if value:
1303
+ processed_args.append('--' + key)
1304
+ else:
1305
+ processed_args.append('--' + key)
1306
+ processed_args.append(str(value))
1307
1308
return processed_args
1309