A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Frontend] [Neuron] Parse literals out of override-neuron-config (#8959)
Co-authored-by: Jerzy Zagorski <jzagorsk@amazon.com>
xendo committed
Oct 3, 2024 at 20:02 UTC
63e39937f990818e2f22a9b821a4aa22387057a7
2 files changed
+37
-20
tests/engine/test_arg_utils.py
+34
-14
@@ -42,22 +42,42 @@ def test_bad_nullable_kvs(arg):
42
nullable_kvs(arg)
43
44
45
-@pytest.mark.parametrize(("arg", "expected"), [
46
- (None, None),
47
- ("{}", {}),
48
- ('{"num_crops": 4}', {
49
- "num_crops": 4
50
- }),
51
- ('{"foo": {"bar": "baz"}}', {
52
- "foo": {
53
- "bar": "baz"
54
- }
55
- }),
45
+# yapf: disable
46
+@pytest.mark.parametrize(("arg", "expected", "option"), [
47
+ (None, None, "mm-processor-kwargs"),
48
+ ("{}", {}, "mm-processor-kwargs"),
49
+ (
50
+ '{"num_crops": 4}',
51
+ {
52
+ "num_crops": 4
53
+ },
54
+ "mm-processor-kwargs"
55
+ ),
56
+ (
57
+ '{"foo": {"bar": "baz"}}',
58
+ {
59
+ "foo":
60
+ {
61
+ "bar": "baz"
62
+ }
63
+ },
64
+ "mm-processor-kwargs"
65
+ ),
66
+ (
67
+ '{"cast_logits_dtype":"bfloat16","sequence_parallel_norm":true,"sequence_parallel_norm_threshold":2048}',
68
+ {
69
+ "cast_logits_dtype": "bfloat16",
70
+ "sequence_parallel_norm": True,
71
+ "sequence_parallel_norm_threshold": 2048,
72
+ },
73
+ "override-neuron-config"
74
+ ),
75
])
57
-def test_mm_processor_kwargs_prompt_parser(arg, expected):
76
+# yapf: enable
77
+def test_composite_arg_parser(arg, expected, option):
78
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
79
if arg is None:
80
args = parser.parse_args([])
81
else:
62
- args = parser.parse_args(["--mm-processor-kwargs", arg])
63
- assert args.mm_processor_kwargs == expected
82
+ args = parser.parse_args([f"--{option}", arg])
83
+ assert getattr(args, option.replace("-", "_")) == expected
vllm/engine/arg_utils.py
+3
-6
@@ -800,13 +800,10 @@ class EngineArgs:
800
"lower performance.")
801
parser.add_argument(
802
'--override-neuron-config',
803
- type=lambda configs: {
804
- str(key): value
805
- for key, value in
806
- (config.split(':') for config in configs.split(','))
807
- },
803
+ type=json.loads,
804
default=None,
809
- help="override or set neuron device configuration.")
805
+ help="Override or set neuron device configuration. "
806
+ "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'")
807
808
parser.add_argument(
809
'--scheduling-policy',