@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 63e39937f9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Frontend] [Neuron] Parse literals out of override-neuron-config (#8959)

Co-authored-by: Jerzy Zagorski <jzagorsk@amazon.com>

xendo committed Oct 3, 2024 at 20:02 UTC 63e39937f990818e2f22a9b821a4aa22387057a7
2 files changed +37 -20
tests/engine/test_arg_utils.py
+34 -14
@@ -42,22 +42,42 @@ def test_bad_nullable_kvs(arg):
42 nullable_kvs(arg)
43
44
45 -@pytest.mark.parametrize(("arg", "expected"), [
46 - (None, None),
47 - ("{}", {}),
48 - ('{"num_crops": 4}', {
49 - "num_crops": 4
50 - }),
51 - ('{"foo": {"bar": "baz"}}', {
52 - "foo": {
53 - "bar": "baz"
54 - }
55 - }),
45 +# yapf: disable
46 +@pytest.mark.parametrize(("arg", "expected", "option"), [
47 + (None, None, "mm-processor-kwargs"),
48 + ("{}", {}, "mm-processor-kwargs"),
49 + (
50 + '{"num_crops": 4}',
51 + {
52 + "num_crops": 4
53 + },
54 + "mm-processor-kwargs"
55 + ),
56 + (
57 + '{"foo": {"bar": "baz"}}',
58 + {
59 + "foo":
60 + {
61 + "bar": "baz"
62 + }
63 + },
64 + "mm-processor-kwargs"
65 + ),
66 + (
67 + '{"cast_logits_dtype":"bfloat16","sequence_parallel_norm":true,"sequence_parallel_norm_threshold":2048}',
68 + {
69 + "cast_logits_dtype": "bfloat16",
70 + "sequence_parallel_norm": True,
71 + "sequence_parallel_norm_threshold": 2048,
72 + },
73 + "override-neuron-config"
74 + ),
75 ])
57 -def test_mm_processor_kwargs_prompt_parser(arg, expected):
76 +# yapf: enable
77 +def test_composite_arg_parser(arg, expected, option):
78 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
79 if arg is None:
80 args = parser.parse_args([])
81 else:
62 - args = parser.parse_args(["--mm-processor-kwargs", arg])
63 - assert args.mm_processor_kwargs == expected
82 + args = parser.parse_args([f"--{option}", arg])
83 + assert getattr(args, option.replace("-", "_")) == expected
vllm/engine/arg_utils.py
+3 -6
@@ -800,13 +800,10 @@ class EngineArgs:
800 "lower performance.")
801 parser.add_argument(
802 '--override-neuron-config',
803 - type=lambda configs: {
804 - str(key): value
805 - for key, value in
806 - (config.split(':') for config in configs.split(','))
807 - },
803 + type=json.loads,
804 default=None,
809 - help="override or set neuron device configuration.")
805 + help="Override or set neuron device configuration. "
806 + "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'")
807
808 parser.add_argument(
809 '--scheduling-policy',