@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 34a9941620
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix load config when using bools (#9533)

madt2709 committed Oct 27, 2024 at 10:46 UTC 34a9941620d00879599a51609225452b705bae89
4 files changed +35 -22
tests/data/test_config.yaml
+2
@@ -1,3 +1,5 @@
1 port: 12312
2 served_model_name: mymodel
3 tensor_parallel_size: 2
4 +trust_remote_code: true
5 +multi_step_stream_outputs: false
tests/test_utils.py
+5 -1
@@ -6,7 +6,7 @@ from typing import AsyncIterator, Tuple
6
7 import pytest
8
9 -from vllm.utils import (FlexibleArgumentParser, deprecate_kwargs,
9 +from vllm.utils import (FlexibleArgumentParser, StoreBoolean, deprecate_kwargs,
10 get_open_port, merge_async_iterators, supports_kw)
11
12 from .utils import error_on_warning
@@ -141,6 +141,8 @@ def parser_with_config():
141 parser.add_argument('--config', type=str)
142 parser.add_argument('--port', type=int)
143 parser.add_argument('--tensor-parallel-size', type=int)
144 + parser.add_argument('--trust-remote-code', action='store_true')
145 + parser.add_argument('--multi-step-stream-outputs', action=StoreBoolean)
146 return parser
147
148
@@ -214,6 +216,8 @@ def test_config_args(parser_with_config):
216 args = parser_with_config.parse_args(
217 ['serve', 'mymodel', '--config', './data/test_config.yaml'])
218 assert args.tensor_parallel_size == 2
219 + assert args.trust_remote_code
220 + assert not args.multi_step_stream_outputs
221
222
223 def test_config_file(parser_with_config):
vllm/engine/arg_utils.py
+1 -13
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
19 from vllm.transformers_utils.config import (
20 maybe_register_config_serialize_by_value)
21 from vllm.transformers_utils.utils import check_gguf_file
22 -from vllm.utils import FlexibleArgumentParser
22 +from vllm.utils import FlexibleArgumentParser, StoreBoolean
23
24 if TYPE_CHECKING:
25 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
@@ -1144,18 +1144,6 @@ class AsyncEngineArgs(EngineArgs):
1144 return parser
1145
1146
1147 -class StoreBoolean(argparse.Action):
1148 -
1149 - def __call__(self, parser, namespace, values, option_string=None):
1150 - if values.lower() == "true":
1151 - setattr(namespace, self.dest, True)
1152 - elif values.lower() == "false":
1153 - setattr(namespace, self.dest, False)
1154 - else:
1155 - raise ValueError(f"Invalid boolean value: {values}. "
1156 - "Expected 'true' or 'false'.")
1157 -
1158 -
1147 # These functions are used by sphinx to build the documentation
1148 def _engine_args_parser():
1149 return EngineArgs.add_cli_args(FlexibleArgumentParser())
vllm/utils.py
+27 -8
@@ -1155,6 +1155,18 @@ def run_once(f: Callable[P, None]) -> Callable[P, None]:
1155 return wrapper
1156
1157
1158 +class StoreBoolean(argparse.Action):
1159 +
1160 + def __call__(self, parser, namespace, values, option_string=None):
1161 + if values.lower() == "true":
1162 + setattr(namespace, self.dest, True)
1163 + elif values.lower() == "false":
1164 + setattr(namespace, self.dest, False)
1165 + else:
1166 + raise ValueError(f"Invalid boolean value: {values}. "
1167 + "Expected 'true' or 'false'.")
1168 +
1169 +
1170 class FlexibleArgumentParser(argparse.ArgumentParser):
1171 """ArgumentParser that allows both underscore and dash in names."""
1172
@@ -1163,7 +1175,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1175 args = sys.argv[1:]
1176
1177 if '--config' in args:
1166 - args = FlexibleArgumentParser._pull_args_from_config(args)
1178 + args = self._pull_args_from_config(args)
1179
1180 # Convert underscores to dashes and vice versa in argument names
1181 processed_args = []
@@ -1181,8 +1193,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1193
1194 return super().parse_args(processed_args, namespace)
1195
1184 - @staticmethod
1185 - def _pull_args_from_config(args: List[str]) -> List[str]:
1196 + def _pull_args_from_config(self, args: List[str]) -> List[str]:
1197 """Method to pull arguments specified in the config file
1198 into the command-line args variable.
1199
@@ -1226,7 +1237,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1237
1238 file_path = args[index + 1]
1239
1229 - config_args = FlexibleArgumentParser._load_config_file(file_path)
1240 + config_args = self._load_config_file(file_path)
1241
1242 # 0th index is for {serve,chat,complete}
1243 # followed by model_tag (only for serve)
@@ -1247,8 +1258,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1258
1259 return args
1260
1250 - @staticmethod
1251 - def _load_config_file(file_path: str) -> List[str]:
1261 + def _load_config_file(self, file_path: str) -> List[str]:
1262 """Loads a yaml file and returns the key value pairs as a
1263 flattened list with argparse like pattern
1264 ```yaml
@@ -1282,9 +1292,18 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1292 Make sure path is correct", file_path)
1293 raise ex
1294
1295 + store_boolean_arguments = [
1296 + action.dest for action in self._actions
1297 + if isinstance(action, StoreBoolean)
1298 + ]
1299 +
1300 for key, value in config.items():
1286 - processed_args.append('--' + key)
1287 - processed_args.append(str(value))
1301 + if isinstance(value, bool) and key not in store_boolean_arguments:
1302 + if value:
1303 + processed_args.append('--' + key)
1304 + else:
1305 + processed_args.append('--' + key)
1306 + processed_args.append(str(value))
1307
1308 return processed_args
1309