| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | from types import SimpleNamespace |
| 6 | |
| 7 | import pytest |
| 8 | |
| 9 | import vllm.entrypoints.launchers.cli_args as cli_args_module |
| 10 | from tests.utils import VLLM_PATH |
| 11 | from vllm.engine.arg_utils import AsyncEngineArgs |
| 12 | from vllm.entrypoints.launchers.cli_args import ( |
| 13 | make_arg_parser, |
| 14 | validate_parsed_serve_args, |
| 15 | ) |
| 16 | from vllm.entrypoints.openai.models.protocol import LoRAModulePath |
| 17 | from vllm.exceptions import VLLMValidationError |
| 18 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 19 | |
| 20 | LORA_MODULE = { |
| 21 | "name": "module2", |
| 22 | "path": "/path/to/module2", |
| 23 | "base_model_name": "llama", |
| 24 | } |
| 25 | CHATML_JINJA_PATH = VLLM_PATH / "examples/template_chatml.jinja" |
| 26 | assert CHATML_JINJA_PATH.exists() |
| 27 | |
| 28 | |
| 29 | def _build_vllm_parsers(): |
| 30 | vllm_parser = FlexibleArgumentParser() |
| 31 | subparsers = vllm_parser.add_subparsers(dest="subparser") |
| 32 | serve_parser = subparsers.add_parser("serve") |
| 33 | make_arg_parser(serve_parser) |
| 34 | return {"vllm": vllm_parser, "vllm serve": serve_parser} |
| 35 | |
| 36 | |
| 37 | @pytest.fixture |
| 38 | def vllm_parser(): |
| 39 | return _build_vllm_parsers()["vllm"] |
| 40 | |
| 41 | |
| 42 | @pytest.fixture |
| 43 | def serve_parser(): |
| 44 | return _build_vllm_parsers()["vllm serve"] |
| 45 | |
| 46 | |
| 47 | ### Test config parsing |
| 48 | def test_config_arg_parsing(serve_parser, cli_config_file): |
| 49 | args = serve_parser.parse_args([]) |
| 50 | assert args.port == 8000 |
| 51 | args = serve_parser.parse_args(["--config", cli_config_file]) |
| 52 | assert args.port == 12312 |
| 53 | args = serve_parser.parse_args( |
| 54 | [ |
| 55 | "--config", |
| 56 | cli_config_file, |
| 57 | "--port", |
| 58 | "9000", |
| 59 | ] |
| 60 | ) |
| 61 | assert args.port == 9000 |
| 62 | args = serve_parser.parse_args( |
| 63 | [ |
| 64 | "--port", |
| 65 | "9000", |
| 66 | "--config", |
| 67 | cli_config_file, |
| 68 | ] |
| 69 | ) |
| 70 | assert args.port == 9000 |
| 71 | |
| 72 | |
| 73 | def test_logging_config_cli_args(serve_parser): |
| 74 | with pytest.warns(UserWarning, match="--log-config-file is deprecated"): |
| 75 | args = serve_parser.parse_args( |
| 76 | [ |
| 77 | "--logging-config", |
| 78 | '{"log_level":"WARNING","pylogging_config_file":"/tmp/json.json"}', |
| 79 | "--logging-config.formatter", |
| 80 | "json", |
| 81 | "--log-level", |
| 82 | "DEBUG", |
| 83 | "--log-config-file", |
| 84 | "/tmp/flat.json", |
| 85 | ] |
| 86 | ) |
| 87 | |
| 88 | config = AsyncEngineArgs.from_cli_args(args).create_logging_config() |
| 89 | assert config.log_level == "DEBUG" |
| 90 | assert config.formatter == "json" |
| 91 | assert config.pylogging_config_file == "/tmp/flat.json" |
| 92 | |
| 93 | |
| 94 | ### Tests for LoRA module parsing |
| 95 | def test_valid_key_value_format(serve_parser): |
| 96 | # Test old format: name=path |
| 97 | args = serve_parser.parse_args( |
| 98 | [ |
| 99 | "--lora-modules", |
| 100 | "module1=/path/to/module1", |
| 101 | ] |
| 102 | ) |
| 103 | expected = [LoRAModulePath(name="module1", path="/path/to/module1")] |
| 104 | assert args.lora_modules == expected |
| 105 | |
| 106 | |
| 107 | def test_valid_json_format(serve_parser): |
| 108 | # Test valid JSON format input |
| 109 | args = serve_parser.parse_args( |
| 110 | [ |
| 111 | "--lora-modules", |
| 112 | json.dumps(LORA_MODULE), |
| 113 | ] |
| 114 | ) |
| 115 | expected = [ |
| 116 | LoRAModulePath(name="module2", path="/path/to/module2", base_model_name="llama") |
| 117 | ] |
| 118 | assert args.lora_modules == expected |
| 119 | |
| 120 | |
| 121 | def test_invalid_json_format(serve_parser): |
| 122 | # Test invalid JSON format input, missing closing brace |
| 123 | with pytest.raises(SystemExit): |
| 124 | serve_parser.parse_args( |
| 125 | ["--lora-modules", '{"name": "module3", "path": "/path/to/module3"'] |
| 126 | ) |
| 127 | |
| 128 | |
| 129 | def test_invalid_type_error(serve_parser): |
| 130 | # Test type error when values are not JSON or key=value |
| 131 | with pytest.raises(SystemExit): |
| 132 | serve_parser.parse_args( |
| 133 | [ |
| 134 | "--lora-modules", |
| 135 | "invalid_format", # This is not JSON or key=value format |
| 136 | ] |
| 137 | ) |
| 138 | |
| 139 | |
| 140 | def test_invalid_json_field(serve_parser): |
| 141 | # Test valid JSON format but missing required fields |
| 142 | with pytest.raises(SystemExit): |
| 143 | serve_parser.parse_args( |
| 144 | [ |
| 145 | "--lora-modules", |
| 146 | '{"name": "module4"}', # Missing required 'path' field |
| 147 | ] |
| 148 | ) |
| 149 | |
| 150 | |
| 151 | def test_empty_values(serve_parser): |
| 152 | # Test when no LoRA modules are provided |
| 153 | args = serve_parser.parse_args(["--lora-modules", ""]) |
| 154 | assert args.lora_modules == [] |
| 155 | |
| 156 | |
| 157 | def test_multiple_valid_inputs(serve_parser): |
| 158 | # Test multiple valid inputs (both old and JSON format) |
| 159 | args = serve_parser.parse_args( |
| 160 | [ |
| 161 | "--lora-modules", |
| 162 | "module1=/path/to/module1", |
| 163 | json.dumps(LORA_MODULE), |
| 164 | ] |
| 165 | ) |
| 166 | expected = [ |
| 167 | LoRAModulePath(name="module1", path="/path/to/module1"), |
| 168 | LoRAModulePath( |
| 169 | name="module2", path="/path/to/module2", base_model_name="llama" |
| 170 | ), |
| 171 | ] |
| 172 | assert args.lora_modules == expected |
| 173 | |
| 174 | |
| 175 | ### Tests for serve argument validation that run prior to loading |
| 176 | @pytest.mark.parametrize( |
| 177 | "port, extra_args", |
| 178 | [ |
| 179 | (50051, []), |
| 180 | (0, ["--port", "0"]), |
| 181 | (50051, ["--port", "50051", "--uds", "/tmp/vllm.sock"]), |
| 182 | ], |
| 183 | ) |
| 184 | def test_rust_grpc_port_accepts_frontend_launch( |
| 185 | vllm_parser, monkeypatch, port, extra_args |
| 186 | ): |
| 187 | monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1") |
| 188 | args = vllm_parser.parse_args( |
| 189 | [ |
| 190 | "serve", |
| 191 | "--grpc-port", |
| 192 | str(port), |
| 193 | "--data-parallel-size", |
| 194 | "8", |
| 195 | "--data-parallel-size-local", |
| 196 | "4", |
| 197 | "--data-parallel-start-rank", |
| 198 | "4", |
| 199 | "--data-parallel-hybrid-lb", |
| 200 | *extra_args, |
| 201 | ] |
| 202 | ) |
| 203 | validate_parsed_serve_args(args) |
| 204 | assert args.grpc_port == port |
| 205 | |
| 206 | |
| 207 | @pytest.mark.parametrize( |
| 208 | "rust_enabled, extra_args, port", |
| 209 | [ |
| 210 | (False, [], 50051), |
| 211 | (False, ["--grpc"], 50051), |
| 212 | (True, ["--grpc"], 50051), |
| 213 | (True, ["--headless"], 50051), |
| 214 | (True, ["--api-server-count", "0"], 50051), |
| 215 | (True, ["--api-server-count", "-1"], 50051), |
| 216 | (True, ["--data-parallel-multi-port-external-lb"], 50051), |
| 217 | (True, ["--port", "50051"], 50051), |
| 218 | (True, [], -1), |
| 219 | (True, [], 65536), |
| 220 | ], |
| 221 | ) |
| 222 | def test_rust_grpc_port_rejects_incompatible_launch( |
| 223 | vllm_parser, monkeypatch, rust_enabled, extra_args, port |
| 224 | ): |
| 225 | monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1" if rust_enabled else "0") |
| 226 | args = vllm_parser.parse_args(["serve", "--grpc-port", str(port), *extra_args]) |
| 227 | error = ( |
| 228 | "--grpc and --grpc-port are mutually exclusive" |
| 229 | if "--grpc" in extra_args |
| 230 | else "--grpc-port" |
| 231 | ) |
| 232 | with pytest.raises(ValueError, match=error): |
| 233 | validate_parsed_serve_args(args) |
| 234 | |
| 235 | |
| 236 | def test_rust_grpc_port_requires_serve_subcommand(monkeypatch): |
| 237 | """Direct Python render/API entrypoints must reject a Rust-only listener.""" |
| 238 | monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1") |
| 239 | parser = make_arg_parser(FlexibleArgumentParser()) |
| 240 | args = parser.parse_args(["--grpc-port", "50051"]) |
| 241 | with pytest.raises(ValueError, match="--grpc-port requires"): |
| 242 | validate_parsed_serve_args(args) |
| 243 | |
| 244 | |
| 245 | def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser): |
| 246 | """Ensure validation fails if tool choice is enabled with no call parser.""" |
| 247 | # If we enable-auto-tool-choice, explode with no tool-call-parser |
| 248 | args = serve_parser.parse_args(args=["--enable-auto-tool-choice"]) |
| 249 | with pytest.raises(TypeError): |
| 250 | validate_parsed_serve_args(args) |
| 251 | |
| 252 | |
| 253 | def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser): |
| 254 | """Ensure validation passes with tool choice enabled with a call parser.""" |
| 255 | args = serve_parser.parse_args( |
| 256 | args=[ |
| 257 | "--enable-auto-tool-choice", |
| 258 | "--tool-call-parser", |
| 259 | "mistral", |
| 260 | ] |
| 261 | ) |
| 262 | validate_parsed_serve_args(args) |
| 263 | |
| 264 | |
| 265 | def test_enable_auto_choice_fails_with_enable_reasoning(serve_parser): |
| 266 | """Ensure validation fails if reasoning is enabled with auto tool choice.""" |
| 267 | args = serve_parser.parse_args( |
| 268 | args=[ |
| 269 | "--enable-auto-tool-choice", |
| 270 | "--reasoning-parser", |
| 271 | "deepseek_r1", |
| 272 | ] |
| 273 | ) |
| 274 | with pytest.raises(TypeError): |
| 275 | validate_parsed_serve_args(args) |
| 276 | |
| 277 | |
| 278 | def test_passes_with_reasoning_parser(serve_parser): |
| 279 | """Ensure validation passes if reasoning is enabled |
| 280 | with a reasoning parser""" |
| 281 | args = serve_parser.parse_args( |
| 282 | args=[ |
| 283 | "--reasoning-parser", |
| 284 | "deepseek_r1", |
| 285 | ] |
| 286 | ) |
| 287 | validate_parsed_serve_args(args) |
| 288 | |
| 289 | |
| 290 | def test_chat_template_validation_for_happy_paths(serve_parser): |
| 291 | """Ensure validation passes if the chat template exists.""" |
| 292 | args = serve_parser.parse_args( |
| 293 | args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()] |
| 294 | ) |
| 295 | validate_parsed_serve_args(args) |
| 296 | |
| 297 | |
| 298 | def test_chat_template_validation_for_sad_paths(serve_parser): |
| 299 | """Ensure validation fails if the chat template doesn't exist.""" |
| 300 | args = serve_parser.parse_args(args=["--chat-template", "does/not/exist"]) |
| 301 | with pytest.raises(VLLMValidationError): |
| 302 | validate_parsed_serve_args(args) |
| 303 | |
| 304 | |
| 305 | def test_per_request_metrics_requires_log_stats(serve_parser): |
| 306 | args = serve_parser.parse_args( |
| 307 | args=["--enable-per-request-metrics", "--disable-log-stats"] |
| 308 | ) |
| 309 | with pytest.raises(ValueError): |
| 310 | validate_parsed_serve_args(args) |
| 311 | |
| 312 | |
| 313 | def _build_launch_render_parser(): |
| 314 | """Mirror `vllm launch render`. |
| 315 | |
| 316 | `vllm/entrypoints/cli/main.py` parses subcommands with ``dest="subparser"``, |
| 317 | and `LaunchSubcommandBase.add_cli_args` builds the component parser with |
| 318 | `make_arg_parser`, so a launch component carries serve args under |
| 319 | ``args.subparser == "launch"``. |
| 320 | """ |
| 321 | vllm_parser = FlexibleArgumentParser() |
| 322 | subparsers = vllm_parser.add_subparsers(required=False, dest="subparser") |
| 323 | launch_parser = subparsers.add_parser("launch") |
| 324 | launch_subparsers = launch_parser.add_subparsers( |
| 325 | required=True, dest="launch_component" |
| 326 | ) |
| 327 | render_parser = launch_subparsers.add_parser("render") |
| 328 | make_arg_parser(render_parser) |
| 329 | return vllm_parser |
| 330 | |
| 331 | |
| 332 | @pytest.fixture |
| 333 | def launch_render_parser(): |
| 334 | return _build_launch_render_parser() |
| 335 | |
| 336 | |
| 337 | @pytest.mark.parametrize( |
| 338 | "cli_args, error", |
| 339 | [ |
| 340 | (["--enable-auto-tool-choice"], TypeError), |
| 341 | (["--grpc-port", "50051"], ValueError), |
| 342 | ], |
| 343 | ) |
| 344 | def test_launch_render_validates_serve_args( |
| 345 | launch_render_parser, monkeypatch, cli_args, error |
| 346 | ): |
| 347 | """`vllm launch render` reuses the serve parser, so it gets the serve checks""" |
| 348 | monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1") |
| 349 | args = launch_render_parser.parse_args(args=["launch", "render", *cli_args]) |
| 350 | assert args.subparser == "launch" |
| 351 | with pytest.raises(error): |
| 352 | validate_parsed_serve_args(args) |
| 353 | |
| 354 | |
| 355 | def test_subcommand_without_serve_args_skips_validation(): |
| 356 | """A subcommand that does not use the serve parser is still skipped""" |
| 357 | vllm_parser = FlexibleArgumentParser() |
| 358 | subparsers = vllm_parser.add_subparsers(required=False, dest="subparser") |
| 359 | subparsers.add_parser("chat") |
| 360 | args = vllm_parser.parse_args(args=["chat"]) |
| 361 | |
| 362 | validate_parsed_serve_args(args) |
| 363 | |
| 364 | |
| 365 | @pytest.mark.parametrize( |
| 366 | "cli_args, expected", |
| 367 | [ |
| 368 | ([], 0), |
| 369 | (["--sse-keep-alive-interval", "0"], 0), |
| 370 | (["--sse-keep-alive-interval", "15"], 15), |
| 371 | ], |
| 372 | ) |
| 373 | def test_sse_keep_alive_interval_valid(serve_parser, cli_args, expected): |
| 374 | """The default is 0 and zero or positive integers parse exactly.""" |
| 375 | args = serve_parser.parse_args(args=cli_args) |
| 376 | assert args.sse_keep_alive_interval == expected |
| 377 | validate_parsed_serve_args(args) |
| 378 | |
| 379 | |
| 380 | @pytest.mark.parametrize("value", ["-1", "-5"]) |
| 381 | def test_sse_keep_alive_interval_negative(serve_parser, value): |
| 382 | """Negative intervals are rejected by validation.""" |
| 383 | args = serve_parser.parse_args(args=["--sse-keep-alive-interval", value]) |
| 384 | with pytest.raises(ValueError): |
| 385 | validate_parsed_serve_args(args) |
| 386 | |
| 387 | |
| 388 | @pytest.mark.parametrize("value", ["nan", "inf", "-inf", "0.5", "abc"]) |
| 389 | def test_sse_keep_alive_interval_non_integer(serve_parser, value): |
| 390 | """Non-integer values fail argparse's int parsing with SystemExit.""" |
| 391 | with pytest.raises(SystemExit): |
| 392 | serve_parser.parse_args(args=["--sse-keep-alive-interval", value]) |
| 393 | |
| 394 | |
| 395 | @pytest.mark.parametrize( |
| 396 | "cli_args, expected_middleware", |
| 397 | [ |
| 398 | ( |
| 399 | ["--middleware", "middleware1", "--middleware", "middleware2"], |
| 400 | ["middleware1", "middleware2"], |
| 401 | ), |
| 402 | ([], []), |
| 403 | ], |
| 404 | ) |
| 405 | def test_middleware(serve_parser, cli_args, expected_middleware): |
| 406 | """Ensure multiple middleware args are parsed properly.""" |
| 407 | args = serve_parser.parse_args(args=cli_args) |
| 408 | assert args.middleware == expected_middleware |
| 409 | |
| 410 | |
| 411 | def test_default_chat_template_kwargs_parsing(serve_parser): |
| 412 | """Ensure default_chat_template_kwargs JSON is parsed correctly.""" |
| 413 | args = serve_parser.parse_args( |
| 414 | args=["--default-chat-template-kwargs", '{"enable_thinking": false}'] |
| 415 | ) |
| 416 | assert args.default_chat_template_kwargs == {"enable_thinking": False} |
| 417 | |
| 418 | |
| 419 | def test_default_chat_template_kwargs_complex(serve_parser): |
| 420 | """Ensure complex default_chat_template_kwargs JSON is parsed correctly.""" |
| 421 | kwargs_json = '{"enable_thinking": false, "custom_param": "value", "num": 42}' |
| 422 | args = serve_parser.parse_args(args=["--default-chat-template-kwargs", kwargs_json]) |
| 423 | assert args.default_chat_template_kwargs == { |
| 424 | "enable_thinking": False, |
| 425 | "custom_param": "value", |
| 426 | "num": 42, |
| 427 | } |
| 428 | |
| 429 | |
| 430 | def test_default_chat_template_kwargs_default_none(serve_parser): |
| 431 | """Ensure default_chat_template_kwargs defaults to None.""" |
| 432 | args = serve_parser.parse_args(args=[]) |
| 433 | assert args.default_chat_template_kwargs is None |
| 434 | |
| 435 | |
| 436 | @pytest.mark.parametrize( |
| 437 | "default_kwargs,cohere_format,expected", |
| 438 | [ |
| 439 | (None, "cmd3", {"cohere_format": "cmd3"}), |
| 440 | ( |
| 441 | {"cohere_format": "cmd4", "custom": True}, |
| 442 | "cmd3", |
| 443 | {"cohere_format": "cmd4", "custom": True}, |
| 444 | ), |
| 445 | ], |
| 446 | ) |
| 447 | def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, expected): |
| 448 | args = SimpleNamespace( |
| 449 | default_chat_template_kwargs=default_kwargs, |
| 450 | cohere_format=cohere_format, |
| 451 | ) |
| 452 | |
| 453 | assert cli_args_module.resolve_default_chat_template_kwargs(args) == expected |
| 454 | |
| 455 | |
| 456 | def test_default_chat_template_kwargs_invalid_json(serve_parser): |
| 457 | """Ensure invalid JSON raises an error.""" |
| 458 | with pytest.raises(SystemExit): |
| 459 | serve_parser.parse_args( |
| 460 | args=["--default-chat-template-kwargs", "not valid json"] |
| 461 | ) |
| 462 | |
| 463 | |
| 464 | @pytest.mark.parametrize( |
| 465 | "args, raises", |
| 466 | [ |
| 467 | (["user/model"], None), |
| 468 | (["user/model", "--served-model-name", "model"], None), |
| 469 | (["--served-model-name", "model", "user/model"], ValueError), |
| 470 | (["--served-model-name", "model", "--config", "config.yaml"], None), |
| 471 | (["--served-model-name", "model", "--config", "config.yaml"], ValueError), |
| 472 | ], |
| 473 | ids=[ |
| 474 | "model_tag_only", |
| 475 | "model_tag_with_served_model_name", |
| 476 | "served_model_name_before_model_tag", |
| 477 | "served_model_name_with_model_in_config", |
| 478 | "served_model_name_with_no_model_in_config", |
| 479 | ], |
| 480 | ) |
| 481 | def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises): |
| 482 | """Ensure that users don't misuse --served-model-name and end up with the default |
| 483 | model tag instead of the one they intended to serve.""" |
| 484 | # Call the serve subparser |
| 485 | args.insert(0, "serve") |
| 486 | # Create a dummy config file if the test case includes it |
| 487 | if "config.yaml" in args: |
| 488 | # Create a dummy config file if the test case includes it |
| 489 | config_path = tmp_path / "config.yaml" |
| 490 | config_path.write_text("model: user/model" if raises is None else "port: 8000") |
| 491 | args[args.index("config.yaml")] = config_path.as_posix() |
| 492 | # Do the parsing and check for expected exceptions or values |
| 493 | if raises is None: |
| 494 | parsed_args = vllm_parser.parse_args(args=args) |
| 495 | expected = "user/model" |
| 496 | assert parsed_args.model_tag == expected or parsed_args.model == expected |
| 497 | else: |
| 498 | with pytest.raises(raises): |
| 499 | vllm_parser.parse_args(args=args) |
| 500 | |
| 501 | |
| 502 | ### Tests for LoRA target modules parsing |
| 503 | def test_lora_target_modules_single(serve_parser): |
| 504 | """Test parsing single lora-target-modules argument.""" |
| 505 | args = serve_parser.parse_args( |
| 506 | args=["--enable-lora", "--lora-target-modules", "o_proj"] |
| 507 | ) |
| 508 | assert args.lora_target_modules == ["o_proj"] |
| 509 | |
| 510 | |
| 511 | def test_lora_target_modules_multiple(serve_parser): |
| 512 | """Test parsing multiple lora-target-modules arguments.""" |
| 513 | args = serve_parser.parse_args( |
| 514 | args=[ |
| 515 | "--enable-lora", |
| 516 | "--lora-target-modules", |
| 517 | "o_proj", |
| 518 | "qkv_proj", |
| 519 | "down_proj", |
| 520 | ] |
| 521 | ) |
| 522 | assert args.lora_target_modules == ["o_proj", "qkv_proj", "down_proj"] |
| 523 | |
| 524 | |
| 525 | def test_lora_target_modules_default_none(serve_parser): |
| 526 | """Test that lora-target-modules defaults to None.""" |
| 527 | args = serve_parser.parse_args(args=[]) |
| 528 | assert args.lora_target_modules is None |