@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1c1bb388e0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Frontend] Improve Nullable kv Arg Parsing (#8525)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Sep 16, 2024 at 22:17 UTC 1c1bb388e0d35a2d10da5c5cda2edac57bf62591
2 files changed +40 -8
tests/engine/test_arg_utils.py
+19 -1
@@ -1,6 +1,8 @@
1 +from argparse import ArgumentTypeError
2 +
3 import pytest
4
3 -from vllm.engine.arg_utils import EngineArgs
5 +from vllm.engine.arg_utils import EngineArgs, nullable_kvs
6 from vllm.utils import FlexibleArgumentParser
7
8
@@ -13,6 +15,10 @@ from vllm.utils import FlexibleArgumentParser
15 "image": 16,
16 "video": 2
17 }),
18 + ("Image=16, Video=2", {
19 + "image": 16,
20 + "video": 2
21 + }),
22 ])
23 def test_limit_mm_per_prompt_parser(arg, expected):
24 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
@@ -22,3 +28,15 @@ def test_limit_mm_per_prompt_parser(arg, expected):
28 args = parser.parse_args(["--limit-mm-per-prompt", arg])
29
30 assert args.limit_mm_per_prompt == expected
31 +
32 +
33 +@pytest.mark.parametrize(
34 + ("arg"),
35 + [
36 + "image", # Missing =
37 + "image=4,image=5", # Conflicting values
38 + "image=video=4" # Too many = in tokenized arg
39 + ])
40 +def test_bad_nullable_kvs(arg):
41 + with pytest.raises(ArgumentTypeError):
42 + nullable_kvs(arg)
vllm/engine/arg_utils.py
+21 -7
@@ -44,22 +44,36 @@ def nullable_str(val: str):
44
45
46 def nullable_kvs(val: str) -> Optional[Mapping[str, int]]:
47 + """Parses a string containing comma separate key [str] to value [int]
48 + pairs into a dictionary.
49 +
50 + Args:
51 + val: String value to be parsed.
52 +
53 + Returns:
54 + Dictionary with parsed values.
55 + """
56 if len(val) == 0:
57 return None
58
59 out_dict: Dict[str, int] = {}
60 for item in val.split(","):
52 - try:
53 - key, value = item.split("=")
54 - except TypeError as exc:
55 - msg = "Each item should be in the form KEY=VALUE"
56 - raise ValueError(msg) from exc
61 + kv_parts = [part.lower().strip() for part in item.split("=")]
62 + if len(kv_parts) != 2:
63 + raise argparse.ArgumentTypeError(
64 + "Each item should be in the form KEY=VALUE")
65 + key, value = kv_parts
66
67 try:
59 - out_dict[key] = int(value)
68 + parsed_value = int(value)
69 except ValueError as exc:
70 msg = f"Failed to parse value of item {key}={value}"
62 - raise ValueError(msg) from exc
71 + raise argparse.ArgumentTypeError(msg) from exc
72 +
73 + if key in out_dict and out_dict[key] != parsed_value:
74 + raise argparse.ArgumentTypeError(
75 + f"Conflicting values specified for key: {key}")
76 + out_dict[key] = parsed_value
77
78 return out_dict
79