| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # ruff: noqa |
| 4 | |
| 5 | import json |
| 6 | import os |
| 7 | from argparse import BooleanOptionalAction |
| 8 | |
| 9 | import pytest |
| 10 | import yaml |
| 11 | from transformers import AutoTokenizer |
| 12 | from pydantic import ValidationError |
| 13 | |
| 14 | from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens |
| 15 | |
| 16 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 17 | from ..utils import flat_product |
| 18 | |
| 19 | |
| 20 | # Tests for FlexibleArgumentParser |
| 21 | @pytest.fixture |
| 22 | def parser(): |
| 23 | parser = FlexibleArgumentParser() |
| 24 | parser.add_argument( |
| 25 | "--image-input-type", choices=["pixel_values", "image_features"] |
| 26 | ) |
| 27 | parser.add_argument("--model-name") |
| 28 | parser.add_argument("--batch-size", type=int) |
| 29 | parser.add_argument("--enable-feature", action="store_true") |
| 30 | parser.add_argument("--hf-overrides", type=json.loads) |
| 31 | parser.add_argument("-cc", "--compilation-config", type=json.loads) |
| 32 | parser.add_argument("--optimization-level", type=int) |
| 33 | return parser |
| 34 | |
| 35 | |
| 36 | @pytest.fixture |
| 37 | def parser_with_config(): |
| 38 | parser = FlexibleArgumentParser() |
| 39 | parser.add_argument("serve") |
| 40 | parser.add_argument("model_tag", nargs="?") |
| 41 | parser.add_argument("--model", type=str) |
| 42 | parser.add_argument("--served-model-name", type=str) |
| 43 | parser.add_argument("--config", type=str) |
| 44 | parser.add_argument("--port", type=int) |
| 45 | parser.add_argument("--tensor-parallel-size", type=int) |
| 46 | parser.add_argument("--trust-remote-code", action="store_true") |
| 47 | return parser |
| 48 | |
| 49 | |
| 50 | def test_underscore_to_dash(parser): |
| 51 | args = parser.parse_args(["--image_input_type", "pixel_values"]) |
| 52 | assert args.image_input_type == "pixel_values" |
| 53 | |
| 54 | |
| 55 | def test_mixed_usage(parser): |
| 56 | args = parser.parse_args( |
| 57 | ["--image_input_type", "image_features", "--model-name", "facebook/opt-125m"] |
| 58 | ) |
| 59 | assert args.image_input_type == "image_features" |
| 60 | assert args.model_name == "facebook/opt-125m" |
| 61 | |
| 62 | |
| 63 | def test_with_equals_sign(parser): |
| 64 | args = parser.parse_args( |
| 65 | ["--image_input_type=pixel_values", "--model-name=facebook/opt-125m"] |
| 66 | ) |
| 67 | assert args.image_input_type == "pixel_values" |
| 68 | assert args.model_name == "facebook/opt-125m" |
| 69 | |
| 70 | |
| 71 | def test_with_int_value(parser): |
| 72 | args = parser.parse_args(["--batch_size", "32"]) |
| 73 | assert args.batch_size == 32 |
| 74 | args = parser.parse_args(["--batch-size", "32"]) |
| 75 | assert args.batch_size == 32 |
| 76 | |
| 77 | |
| 78 | def test_with_bool_flag(parser): |
| 79 | args = parser.parse_args(["--enable_feature"]) |
| 80 | assert args.enable_feature is True |
| 81 | args = parser.parse_args(["--enable-feature"]) |
| 82 | assert args.enable_feature is True |
| 83 | |
| 84 | |
| 85 | def test_invalid_choice(parser): |
| 86 | with pytest.raises(SystemExit): |
| 87 | parser.parse_args(["--image_input_type", "invalid_choice"]) |
| 88 | |
| 89 | |
| 90 | def test_missing_required_argument(parser): |
| 91 | parser.add_argument("--required-arg", required=True) |
| 92 | with pytest.raises(SystemExit): |
| 93 | parser.parse_args([]) |
| 94 | |
| 95 | |
| 96 | def test_cli_override_to_config(parser_with_config, cli_config_file): |
| 97 | args = parser_with_config.parse_args( |
| 98 | ["serve", "mymodel", "--config", cli_config_file, "--tensor-parallel-size", "3"] |
| 99 | ) |
| 100 | assert args.tensor_parallel_size == 3 |
| 101 | args = parser_with_config.parse_args( |
| 102 | ["serve", "mymodel", "--tensor-parallel-size", "3", "--config", cli_config_file] |
| 103 | ) |
| 104 | assert args.tensor_parallel_size == 3 |
| 105 | assert args.port == 12312 |
| 106 | args = parser_with_config.parse_args( |
| 107 | [ |
| 108 | "serve", |
| 109 | "mymodel", |
| 110 | "--tensor-parallel-size", |
| 111 | "3", |
| 112 | "--config", |
| 113 | cli_config_file, |
| 114 | "--port", |
| 115 | "666", |
| 116 | ] |
| 117 | ) |
| 118 | assert args.tensor_parallel_size == 3 |
| 119 | assert args.port == 666 |
| 120 | |
| 121 | |
| 122 | def test_config_args(parser_with_config, cli_config_file): |
| 123 | args = parser_with_config.parse_args( |
| 124 | ["serve", "mymodel", "--config", cli_config_file] |
| 125 | ) |
| 126 | assert args.tensor_parallel_size == 2 |
| 127 | assert args.trust_remote_code |
| 128 | |
| 129 | |
| 130 | def test_config_file(parser_with_config): |
| 131 | with pytest.raises(FileNotFoundError): |
| 132 | parser_with_config.parse_args( |
| 133 | ["serve", "mymodel", "--config", "test_config.yml"] |
| 134 | ) |
| 135 | |
| 136 | with pytest.raises(ValueError): |
| 137 | parser_with_config.parse_args( |
| 138 | ["serve", "mymodel", "--config", "./data/test_config.json"] |
| 139 | ) |
| 140 | |
| 141 | with pytest.raises(ValueError): |
| 142 | parser_with_config.parse_args( |
| 143 | [ |
| 144 | "serve", |
| 145 | "mymodel", |
| 146 | "--tensor-parallel-size", |
| 147 | "3", |
| 148 | "--config", |
| 149 | "--batch-size", |
| 150 | "32", |
| 151 | ] |
| 152 | ) |
| 153 | |
| 154 | |
| 155 | def test_no_model_tag(parser_with_config, cli_config_file): |
| 156 | with pytest.raises(ValueError): |
| 157 | parser_with_config.parse_args(["serve", "--config", cli_config_file]) |
| 158 | |
| 159 | |
| 160 | def test_dict_args(parser): |
| 161 | args = [ |
| 162 | "--model-name=something.something", |
| 163 | "--hf-overrides.key1", |
| 164 | "val1", |
| 165 | # Test nesting |
| 166 | "--hf-overrides.key2.key3", |
| 167 | "val2", |
| 168 | "--hf-overrides.key2.key4", |
| 169 | "val3", |
| 170 | # Test compile config and compilation mode |
| 171 | "-cc.use_inductor_graph_partition=true", |
| 172 | "-cc.backend", |
| 173 | "custom", |
| 174 | "-O1", |
| 175 | # Test = sign |
| 176 | "--hf-overrides.key5=val4", |
| 177 | # Test underscore to dash conversion |
| 178 | "--hf_overrides.key_6", |
| 179 | "val5", |
| 180 | "--hf_overrides.key-7.key_8", |
| 181 | "val6", |
| 182 | # Test data type detection |
| 183 | "--hf_overrides.key9", |
| 184 | "100", |
| 185 | "--hf_overrides.key10", |
| 186 | "100.0", |
| 187 | "--hf_overrides.key11", |
| 188 | "true", |
| 189 | "--hf_overrides.key12.key13", |
| 190 | "null", |
| 191 | # Test '-' and '.' in value |
| 192 | "--hf_overrides.key14.key15", |
| 193 | "-minus.and.dot", |
| 194 | # Test array values |
| 195 | "-cc.custom_ops+", |
| 196 | "-quant_fp8", |
| 197 | "-cc.custom_ops+=+silu_mul,-rms_norm", |
| 198 | ] |
| 199 | parsed_args = parser.parse_args(args) |
| 200 | assert parsed_args.model_name == "something.something" |
| 201 | assert parsed_args.hf_overrides == { |
| 202 | "key1": "val1", |
| 203 | "key2": { |
| 204 | "key3": "val2", |
| 205 | "key4": "val3", |
| 206 | }, |
| 207 | "key5": "val4", |
| 208 | "key_6": "val5", |
| 209 | "key-7": { |
| 210 | "key_8": "val6", |
| 211 | }, |
| 212 | "key9": 100, |
| 213 | "key10": 100.0, |
| 214 | "key11": True, |
| 215 | "key12": { |
| 216 | "key13": None, |
| 217 | }, |
| 218 | "key14": { |
| 219 | "key15": "-minus.and.dot", |
| 220 | }, |
| 221 | } |
| 222 | assert parsed_args.optimization_level == 1 |
| 223 | assert parsed_args.compilation_config == { |
| 224 | "use_inductor_graph_partition": True, |
| 225 | "backend": "custom", |
| 226 | "custom_ops": ["-quant_fp8", "+silu_mul", "-rms_norm"], |
| 227 | } |
| 228 | |
| 229 | |
| 230 | def test_duplicate_dict_args(caplog_vllm, parser): |
| 231 | args = [ |
| 232 | "--model-name=something.something", |
| 233 | "--hf-overrides.key1", |
| 234 | "val1", |
| 235 | "--hf-overrides.key1", |
| 236 | "val2", |
| 237 | "-O1", |
| 238 | "-cc.mode", |
| 239 | "2", |
| 240 | "-O3", |
| 241 | ] |
| 242 | |
| 243 | parsed_args = parser.parse_args(args) |
| 244 | # Should be the last value |
| 245 | assert parsed_args.hf_overrides == {"key1": "val2"} |
| 246 | assert parsed_args.optimization_level == 3 |
| 247 | assert parsed_args.compilation_config == {"mode": 2} |
| 248 | |
| 249 | assert len(caplog_vllm.records) == 1 |
| 250 | assert "duplicate" in caplog_vllm.text |
| 251 | assert "--hf-overrides.key1" in caplog_vllm.text |
| 252 | assert "--optimization-level" in caplog_vllm.text |
| 253 | |
| 254 | |
| 255 | def test_model_specification( |
| 256 | parser_with_config, cli_config_file, cli_config_file_with_model |
| 257 | ): |
| 258 | # Test model in CLI takes precedence over config |
| 259 | args = parser_with_config.parse_args( |
| 260 | ["serve", "cli-model", "--config", cli_config_file_with_model] |
| 261 | ) |
| 262 | assert args.model_tag == "cli-model" |
| 263 | assert args.served_model_name == "mymodel" |
| 264 | |
| 265 | # Test model from config file works |
| 266 | args = parser_with_config.parse_args( |
| 267 | [ |
| 268 | "serve", |
| 269 | "--config", |
| 270 | cli_config_file_with_model, |
| 271 | ] |
| 272 | ) |
| 273 | assert args.model == "config-model" |
| 274 | assert args.served_model_name == "mymodel" |
| 275 | |
| 276 | # Test no model specified anywhere raises error |
| 277 | with pytest.raises(ValueError, match="No model specified!"): |
| 278 | parser_with_config.parse_args(["serve", "--config", cli_config_file]) |
| 279 | |
| 280 | # Test using --model option raises error |
| 281 | # with pytest.raises( |
| 282 | # ValueError, |
| 283 | # match= |
| 284 | # ("With `vllm serve`, you should provide the model as a positional " |
| 285 | # "argument or in a config file instead of via the `--model` option."), |
| 286 | # ): |
| 287 | # parser_with_config.parse_args(['serve', '--model', 'my-model']) |
| 288 | |
| 289 | # Test using --model option back-compatibility |
| 290 | # (when back-compatibility ends, the above test should be uncommented |
| 291 | # and the below test should be removed) |
| 292 | args = parser_with_config.parse_args( |
| 293 | [ |
| 294 | "serve", |
| 295 | "--tensor-parallel-size", |
| 296 | "2", |
| 297 | "--model", |
| 298 | "my-model", |
| 299 | "--trust-remote-code", |
| 300 | "--port", |
| 301 | "8001", |
| 302 | ] |
| 303 | ) |
| 304 | assert args.model is None |
| 305 | assert args.tensor_parallel_size == 2 |
| 306 | assert args.trust_remote_code is True |
| 307 | assert args.port == 8001 |
| 308 | |
| 309 | args = parser_with_config.parse_args( |
| 310 | [ |
| 311 | "serve", |
| 312 | "--tensor-parallel-size=2", |
| 313 | "--model=my-model", |
| 314 | "--trust-remote-code", |
| 315 | "--port=8001", |
| 316 | ] |
| 317 | ) |
| 318 | assert args.model is None |
| 319 | assert args.tensor_parallel_size == 2 |
| 320 | assert args.trust_remote_code is True |
| 321 | assert args.port == 8001 |
| 322 | |
| 323 | # Test other config values are preserved |
| 324 | args = parser_with_config.parse_args( |
| 325 | [ |
| 326 | "serve", |
| 327 | "cli-model", |
| 328 | "--config", |
| 329 | cli_config_file_with_model, |
| 330 | ] |
| 331 | ) |
| 332 | assert args.tensor_parallel_size == 2 |
| 333 | assert args.trust_remote_code is True |
| 334 | assert args.port == 12312 |
| 335 | |
| 336 | |
| 337 | def test_convert_ids_list_to_tokens(): |
| 338 | tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-1.5B-Instruct") |
| 339 | token_ids = tokenizer.encode("Hello, world!") |
| 340 | # token_ids = [9707, 11, 1879, 0] |
| 341 | assert tokenizer.convert_ids_to_tokens(token_ids) == ["Hello", ",", "Ġworld", "!"] |
| 342 | tokens = convert_ids_list_to_tokens(tokenizer, token_ids) |
| 343 | assert tokens == ["Hello", ",", " world", "!"] |
| 344 | |
| 345 | |
| 346 | def test_load_config_file(tmp_path): |
| 347 | # Define the configuration data |
| 348 | config_data = { |
| 349 | "enable-logging": True, |
| 350 | "list-arg": ["item1", "item2"], |
| 351 | "port": 12323, |
| 352 | "tensor-parallel-size": 4, |
| 353 | } |
| 354 | |
| 355 | # Write the configuration data to a temporary YAML file |
| 356 | config_file_path = tmp_path / "config.yaml" |
| 357 | with open(config_file_path, "w") as config_file: |
| 358 | yaml.dump(config_data, config_file) |
| 359 | |
| 360 | # Initialize the parser |
| 361 | parser = FlexibleArgumentParser() |
| 362 | |
| 363 | # Call the function with the temporary file path |
| 364 | processed_args = parser.load_config_file(str(config_file_path)) |
| 365 | |
| 366 | # Expected output |
| 367 | expected_args = [ |
| 368 | "--enable-logging", |
| 369 | "--list-arg", |
| 370 | "item1", |
| 371 | "item2", |
| 372 | "--port", |
| 373 | "12323", |
| 374 | "--tensor-parallel-size", |
| 375 | "4", |
| 376 | ] |
| 377 | |
| 378 | # Assert that the processed arguments match the expected output |
| 379 | assert processed_args == expected_args |
| 380 | os.remove(str(config_file_path)) |
| 381 | |
| 382 | |
| 383 | def test_load_config_file_false_store_true_dropped(tmp_path): |
| 384 | """False values for store_true flags are silently dropped (correct).""" |
| 385 | config_data = { |
| 386 | "enable-feature": False, |
| 387 | "port": 8000, |
| 388 | } |
| 389 | |
| 390 | config_file_path = tmp_path / "config.yaml" |
| 391 | with open(config_file_path, "w") as config_file: |
| 392 | yaml.dump(config_data, config_file) |
| 393 | |
| 394 | parser = FlexibleArgumentParser() |
| 395 | parser.add_argument("--enable-feature", action=BooleanOptionalAction) |
| 396 | parser.add_argument("--port", type=int) |
| 397 | |
| 398 | processed_args = parser.load_config_file(str(config_file_path)) |
| 399 | |
| 400 | assert "--enable-feature" not in processed_args |
| 401 | assert "--no-enable-feature" in processed_args |
| 402 | |
| 403 | |
| 404 | def test_load_config_file_nested(tmp_path): |
| 405 | """Test that nested dicts in YAML config are converted to JSON strings.""" |
| 406 | config_data = { |
| 407 | "port": 8000, |
| 408 | "compilation-config": { |
| 409 | "pass_config": {"fuse_allreduce_rms": True}, |
| 410 | }, |
| 411 | } |
| 412 | config_file_path = tmp_path / "nested_config.yaml" |
| 413 | with open(config_file_path, "w") as f: |
| 414 | yaml.dump(config_data, f) |
| 415 | |
| 416 | parser = FlexibleArgumentParser() |
| 417 | processed_args = parser.load_config_file(str(config_file_path)) |
| 418 | |
| 419 | assert processed_args[processed_args.index("--port") + 1] == "8000" |
| 420 | cc_value = json.loads( |
| 421 | processed_args[processed_args.index("--compilation-config") + 1] |
| 422 | ) |
| 423 | assert cc_value == {"pass_config": {"fuse_allreduce_rms": True}} |
| 424 | |
| 425 | |
| 426 | def test_nested_config_end_to_end(tmp_path): |
| 427 | """Test end-to-end parsing of nested configs in YAML files.""" |
| 428 | config_data = { |
| 429 | "compilation-config": { |
| 430 | "mode": 3, |
| 431 | "pass_config": {"fuse_allreduce_rms": True}, |
| 432 | }, |
| 433 | } |
| 434 | config_file_path = tmp_path / "nested_config.yaml" |
| 435 | with open(config_file_path, "w") as f: |
| 436 | yaml.dump(config_data, f) |
| 437 | |
| 438 | parser = FlexibleArgumentParser() |
| 439 | parser.add_argument("-cc", "--compilation-config", type=json.loads) |
| 440 | args = parser.parse_args(["--config", str(config_file_path)]) |
| 441 | |
| 442 | assert args.compilation_config == { |
| 443 | "mode": 3, |
| 444 | "pass_config": {"fuse_allreduce_rms": True}, |
| 445 | } |
| 446 | |
| 447 | |
| 448 | def test_compilation_mode_string_values(parser): |
| 449 | """Test that -cc.mode accepts both integer and string mode values.""" |
| 450 | args = parser.parse_args(["-cc.mode", "0"]) |
| 451 | assert args.compilation_config == {"mode": 0} |
| 452 | |
| 453 | args = parser.parse_args(["-O3"]) |
| 454 | assert args.optimization_level == 3 |
| 455 | |
| 456 | args = parser.parse_args(["-cc.mode=NONE"]) |
| 457 | assert args.compilation_config == {"mode": "NONE"} |
| 458 | |
| 459 | args = parser.parse_args(["-cc.mode", "STOCK_TORCH_COMPILE"]) |
| 460 | assert args.compilation_config == {"mode": "STOCK_TORCH_COMPILE"} |
| 461 | |
| 462 | args = parser.parse_args(["-cc.mode=DYNAMO_TRACE_ONCE"]) |
| 463 | assert args.compilation_config == {"mode": "DYNAMO_TRACE_ONCE"} |
| 464 | |
| 465 | args = parser.parse_args(["-cc.mode", "VLLM_COMPILE"]) |
| 466 | assert args.compilation_config == {"mode": "VLLM_COMPILE"} |
| 467 | |
| 468 | args = parser.parse_args(["-cc.mode=none"]) |
| 469 | assert args.compilation_config == {"mode": "none"} |
| 470 | |
| 471 | args = parser.parse_args(["-cc.mode=vllm_compile"]) |
| 472 | assert args.compilation_config == {"mode": "vllm_compile"} |
| 473 | |
| 474 | |
| 475 | def test_compilation_config_mode_validator(): |
| 476 | """Test that CompilationConfig.mode field validator converts strings to integers.""" |
| 477 | from vllm.config.compilation import CompilationConfig, CompilationMode |
| 478 | |
| 479 | config = CompilationConfig(mode=0) |
| 480 | assert config.mode == CompilationMode.NONE |
| 481 | |
| 482 | config = CompilationConfig(mode=3) |
| 483 | assert config.mode == CompilationMode.VLLM_COMPILE |
| 484 | |
| 485 | config = CompilationConfig(mode="NONE") |
| 486 | assert config.mode == CompilationMode.NONE |
| 487 | |
| 488 | config = CompilationConfig(mode="STOCK_TORCH_COMPILE") |
| 489 | assert config.mode == CompilationMode.STOCK_TORCH_COMPILE |
| 490 | |
| 491 | config = CompilationConfig(mode="DYNAMO_TRACE_ONCE") |
| 492 | assert config.mode == CompilationMode.DYNAMO_TRACE_ONCE |
| 493 | |
| 494 | config = CompilationConfig(mode="VLLM_COMPILE") |
| 495 | assert config.mode == CompilationMode.VLLM_COMPILE |
| 496 | |
| 497 | config = CompilationConfig(mode="none") |
| 498 | assert config.mode == CompilationMode.NONE |
| 499 | |
| 500 | config = CompilationConfig(mode="vllm_compile") |
| 501 | assert config.mode == CompilationMode.VLLM_COMPILE |
| 502 | |
| 503 | with pytest.raises(ValidationError, match="Invalid compilation mode"): |
| 504 | CompilationConfig(mode="INVALID_MODE") |
| 505 | |
| 506 | |
| 507 | def test_flat_product(): |
| 508 | # Check regular itertools.product behavior |
| 509 | result1 = list(flat_product([1, 2, 3], ["a", "b"])) |
| 510 | assert result1 == [ |
| 511 | (1, "a"), |
| 512 | (1, "b"), |
| 513 | (2, "a"), |
| 514 | (2, "b"), |
| 515 | (3, "a"), |
| 516 | (3, "b"), |
| 517 | ] |
| 518 | |
| 519 | # check that the tuples get flattened |
| 520 | result2 = list(flat_product([(1, 2), (3, 4)], ["a", "b"], [(5, 6)])) |
| 521 | assert result2 == [ |
| 522 | (1, 2, "a", 5, 6), |
| 523 | (1, 2, "b", 5, 6), |
| 524 | (3, 4, "a", 5, 6), |
| 525 | (3, 4, "b", 5, 6), |
| 526 | ] |
| 527 | |
| 528 | |
| 529 | def test_group_description_is_summary_only(): |
| 530 | """Group descriptions are config docstrings, too long for the terminal.""" |
| 531 | parser = FlexibleArgumentParser() |
| 532 | group = parser.add_argument_group( |
| 533 | title="MyConfig", |
| 534 | description="Summary line.\n\nDetails which only belong in the docs.", |
| 535 | ) |
| 536 | group.add_argument("--my-arg") |
| 537 | |
| 538 | assert "Summary line." in parser.format_help() |
| 539 | assert "only belong in the docs" not in parser.format_help() |
| 540 | |
| 541 | parser._search_keyword = "myconfig" |
| 542 | assert "Summary line." in parser.format_help() |
| 543 | assert "only belong in the docs" not in parser.format_help() |