A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Bugfix][Frontend] Fix openai CLI arg `middleware` (#21220)
Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com>
22quinn committed
Jul 19, 2025 at 02:40 UTC
b3d82108e7fdd98c781e7330335e3b4b0c7c0de5
2 files changed
+14
tests/entrypoints/openai/test_cli_args.py
+10
@@ -153,3 +153,13 @@ def test_chat_template_validation_for_sad_paths(serve_parser):
153
args = serve_parser.parse_args(args=["--chat-template", "does/not/exist"])
154
with pytest.raises(ValueError):
155
validate_parsed_serve_args(args)
156
+
157
+
158
+@pytest.mark.parametrize(
159
+ "cli_args, expected_middleware",
160
+ [(["--middleware", "middleware1", "--middleware", "middleware2"
161
+ ], ["middleware1", "middleware2"]), ([], [])])
162
+def test_middleware(serve_parser, cli_args, expected_middleware):
163
+ """Ensure multiple middleware args are parsed properly"""
164
+ args = serve_parser.parse_args(args=cli_args)
165
+ assert args.middleware == expected_middleware
vllm/entrypoints/openai/cli_args.py
+4
@@ -215,6 +215,10 @@ schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
215
216
# Special case: Middleware needs append action
217
frontend_kwargs["middleware"]["action"] = "append"
218
+ frontend_kwargs["middleware"]["type"] = str
219
+ if "nargs" in frontend_kwargs["middleware"]:
220
+ del frontend_kwargs["middleware"]["nargs"]
221
+ frontend_kwargs["middleware"]["default"] = []
222
223
# Special case: Tool call parser shows built-in options.
224
valid_tool_parsers = list(ToolParserManager.tool_parsers.keys())