A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Frontend] Improve Nullable kv Arg Parsing (#8525)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
Alex Brooks committed
Sep 16, 2024 at 22:17 UTC
1c1bb388e0d35a2d10da5c5cda2edac57bf62591
2 files changed
+40
-8
tests/engine/test_arg_utils.py
+19
-1
@@ -1,6 +1,8 @@
1
+from argparse import ArgumentTypeError
2
+
3
import pytest
4
3
-from vllm.engine.arg_utils import EngineArgs
5
+from vllm.engine.arg_utils import EngineArgs, nullable_kvs
6
from vllm.utils import FlexibleArgumentParser
7
8
@@ -13,6 +15,10 @@ from vllm.utils import FlexibleArgumentParser
15
"image": 16,
16
"video": 2
17
}),
18
+ ("Image=16, Video=2", {
19
+ "image": 16,
20
+ "video": 2
21
+ }),
22
])
23
def test_limit_mm_per_prompt_parser(arg, expected):
24
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
@@ -22,3 +28,15 @@ def test_limit_mm_per_prompt_parser(arg, expected):
28
args = parser.parse_args(["--limit-mm-per-prompt", arg])
29
30
assert args.limit_mm_per_prompt == expected
31
+
32
+
33
+@pytest.mark.parametrize(
34
+ ("arg"),
35
+ [
36
+ "image", # Missing =
37
+ "image=4,image=5", # Conflicting values
38
+ "image=video=4" # Too many = in tokenized arg
39
+ ])
40
+def test_bad_nullable_kvs(arg):
41
+ with pytest.raises(ArgumentTypeError):
42
+ nullable_kvs(arg)
vllm/engine/arg_utils.py
+21
-7
@@ -44,22 +44,36 @@ def nullable_str(val: str):
44
45
46
def nullable_kvs(val: str) -> Optional[Mapping[str, int]]:
47
+ """Parses a string containing comma separate key [str] to value [int]
48
+ pairs into a dictionary.
49
+
50
+ Args:
51
+ val: String value to be parsed.
52
+
53
+ Returns:
54
+ Dictionary with parsed values.
55
+ """
56
if len(val) == 0:
57
return None
58
59
out_dict: Dict[str, int] = {}
60
for item in val.split(","):
52
- try:
53
- key, value = item.split("=")
54
- except TypeError as exc:
55
- msg = "Each item should be in the form KEY=VALUE"
56
- raise ValueError(msg) from exc
61
+ kv_parts = [part.lower().strip() for part in item.split("=")]
62
+ if len(kv_parts) != 2:
63
+ raise argparse.ArgumentTypeError(
64
+ "Each item should be in the form KEY=VALUE")
65
+ key, value = kv_parts
66
67
try:
59
- out_dict[key] = int(value)
68
+ parsed_value = int(value)
69
except ValueError as exc:
70
msg = f"Failed to parse value of item {key}={value}"
62
- raise ValueError(msg) from exc
71
+ raise argparse.ArgumentTypeError(msg) from exc
72
+
73
+ if key in out_dict and out_dict[key] != parsed_value:
74
+ raise argparse.ArgumentTypeError(
75
+ f"Conflicting values specified for key: {key}")
76
+ out_dict[key] = parsed_value
77
78
return out_dict
79