@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d1e21a979b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[CI/Build] Split up VLM tests (#11083)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 12, 2024 at 06:18 UTC d1e21a979bba4712f48dac1bbf410e0b57c92e7a
4 files changed +94 -50
.buildkite/test-pipeline.yaml
+21 -11
@@ -321,7 +321,7 @@ steps:
321
322 ##### models test #####
323
324 -- label: Basic Models Test # 30min
324 +- label: Basic Models Test # 24min
325 source_file_dependencies:
326 - vllm/
327 - tests/models
@@ -331,7 +331,7 @@ steps:
331 - pytest -v -s models/test_registry.py
332 - pytest -v -s models/test_initialization.py
333
334 -- label: Language Models Test (Standard) # 42min
334 +- label: Language Models Test (Standard) # 32min
335 #mirror_hardwares: [amd]
336 source_file_dependencies:
337 - vllm/
@@ -342,7 +342,7 @@ steps:
342 - pytest -v -s models/decoder_only/language -m 'core_model or quant_model'
343 - pytest -v -s models/embedding/language -m core_model
344
345 -- label: Language Models Test (Extended) # 50min
345 +- label: Language Models Test (Extended) # 1h10min
346 optional: true
347 source_file_dependencies:
348 - vllm/
@@ -353,7 +353,7 @@ steps:
353 - pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
354 - pytest -v -s models/embedding/language -m 'not core_model'
355
356 -- label: Multi-Modal Models Test (Standard) # 26min
356 +- label: Multi-Modal Models Test (Standard) # 28min
357 #mirror_hardwares: [amd]
358 source_file_dependencies:
359 - vllm/
@@ -369,7 +369,7 @@ steps:
369 - pytest -v -s models/encoder_decoder/language -m core_model
370 - pytest -v -s models/encoder_decoder/vision_language -m core_model
371
372 -- label: Multi-Modal Models Test (Extended) # 1h15m
372 +- label: Multi-Modal Models Test (Extended) 1 # 1h16m
373 optional: true
374 source_file_dependencies:
375 - vllm/
@@ -380,14 +380,24 @@ steps:
380 commands:
381 - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
382 - pytest -v -s models/decoder_only/audio_language -m 'not core_model and not quant_model'
383 + - pytest -v -s models/decoder_only/vision_language/test_models.py -m 'split(group=0) and not core_model and not quant_model'
384 # HACK - run phi3v tests separately to sidestep this transformers bug
385 # https://github.com/huggingface/transformers/issues/34307
386 - pytest -v -s models/decoder_only/vision_language/test_phi3v.py
386 - - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
387 + - pytest -v -s --ignore models/decoder_only/vision_language/test_models.py --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
388 - pytest -v -s models/embedding/vision_language -m 'not core_model'
389 - pytest -v -s models/encoder_decoder/language -m 'not core_model'
390 - pytest -v -s models/encoder_decoder/vision_language -m 'not core_model'
391
392 +- label: Multi-Modal Models Test (Extended) 2 # 38m
393 + optional: true
394 + source_file_dependencies:
395 + - vllm/
396 + - tests/models/decoder_only/vision_language
397 + commands:
398 + - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
399 + - pytest -v -s models/decoder_only/vision_language/test_models.py -m 'split(group=1) and not core_model and not quant_model'
400 +
401 # This test is used only in PR development phase to test individual models and should never run on main
402 - label: Custom Models Test
403 optional: true
@@ -446,11 +456,11 @@ steps:
456 - pytest -v -s ./compile/test_basic_correctness.py
457 - pytest -v -s ./compile/test_wrapper.py
458 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
449 - - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
459 + - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
460 # Avoid importing model tests that cause CUDA reinitialization error
451 - - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
452 - - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
453 - - pytest models/decoder_only/vision_language/test_models.py -v -s -m distributed_2_gpus
461 + - pytest models/encoder_decoder/language/test_bart.py -v -s -m 'distributed(num_gpus=2)'
462 + - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m 'distributed(num_gpus=2)'
463 + - pytest models/decoder_only/vision_language/test_models.py -v -s -m 'distributed(num_gpus=2)'
464 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
465 - pip install -e ./plugins/vllm_add_dummy_model
466 - pytest -v -s distributed/test_distributed_oot.py
@@ -540,7 +550,7 @@ steps:
550 # see https://github.com/vllm-project/vllm/pull/5689 for details
551 - pytest -v -s distributed/test_custom_all_reduce.py
552 - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
543 - - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m distributed_2_gpus
553 + - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
554 - pytest -v -s -x lora/test_mixtral.py
555
556 - label: LM Eval Large Models # optional
pyproject.toml
+2 -1
@@ -96,7 +96,8 @@ markers = [
96 "core_model: enable this model test in each PR instead of only nightly",
97 "cpu_model: enable this model test in CPU tests",
98 "quant_model: run this model test under Quantized category",
99 - "distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
99 + "split: run this test as part of a split",
100 + "distributed: run this test only in distributed GPU tests",
101 "skip_v1: do not run this test with v1",
102 "optional: optional tests that are automatically skipped, include --optional to run them",
103 ]
tests/models/decoder_only/vision_language/test_models.py
+46 -26
@@ -1,7 +1,9 @@
1 """Common tests for testing .generate() functionality for single / multiple
2 image, embedding, and video support for different VLMs in vLLM.
3 """
4 +import math
5 import os
6 +from collections import defaultdict
7 from pathlib import PosixPath
8 from typing import Type
9
@@ -10,11 +12,12 @@ from transformers import AutoModelForVision2Seq
12 from transformers.utils import is_flash_attn_2_available
13
14 from vllm.platforms import current_platform
13 -from vllm.utils import cuda_device_count_stateless, identity
15 +from vllm.utils import identity
16
17 from ....conftest import (IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets,
18 _VideoAssets)
17 -from ....utils import fork_new_process_for_each_test, large_gpu_mark
19 +from ....utils import (fork_new_process_for_each_test, large_gpu_mark,
20 + multi_gpu_marks)
21 from ...utils import check_outputs_equal
22 from .vlm_utils import custom_inputs, model_utils, runners
23 from .vlm_utils.case_filtering import get_parametrized_options
@@ -382,7 +385,7 @@ VLM_TEST_SETTINGS = {
385 prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
386 ),
387 ### Tensor parallel / multi-gpu broadcast tests
385 - "broadcast-chameleon": VLMTestInfo(
388 + "chameleon-broadcast": VLMTestInfo(
389 models=["facebook/chameleon-7b"],
390 prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
391 max_model_len=4096,
@@ -393,43 +396,25 @@ VLM_TEST_SETTINGS = {
396 vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
397 hf_output_post_proc = lambda hf_output, model: hf_output[:2],
398 comparator=check_outputs_equal,
396 - marks=[
397 - pytest.mark.distributed_2_gpus,
398 - pytest.mark.skipif(
399 - cuda_device_count_stateless() < 2,
400 - reason="Need at least 2 GPUs to run the test.",
401 - ),
402 - ],
399 + marks=multi_gpu_marks(num_gpus=2),
400 **COMMON_BROADCAST_SETTINGS # type: ignore
401 ),
405 - "broadcast-llava": VLMTestInfo(
402 + "llava-broadcast": VLMTestInfo(
403 models=["llava-hf/llava-1.5-7b-hf"],
404 prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
405 max_model_len=4096,
406 auto_cls=AutoModelForVision2Seq,
407 vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
411 - marks=[
412 - pytest.mark.distributed_2_gpus,
413 - pytest.mark.skipif(
414 - cuda_device_count_stateless() < 2,
415 - reason="Need at least 2 GPUs to run the test.",
416 - )
417 - ],
408 + marks=multi_gpu_marks(num_gpus=2),
409 **COMMON_BROADCAST_SETTINGS # type: ignore
410 ),
420 - "broadcast-llava_next": VLMTestInfo(
411 + "llava_next-broadcast": VLMTestInfo(
412 models=["llava-hf/llava-v1.6-mistral-7b-hf"],
413 prompt_formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]",
414 max_model_len=10240,
415 auto_cls=AutoModelForVision2Seq,
416 vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
426 - marks=[
427 - pytest.mark.distributed_2_gpus,
428 - pytest.mark.skipif(
429 - cuda_device_count_stateless() < 2,
430 - reason="Need at least 2 GPUs to run the test.",
431 - )
432 - ],
417 + marks=multi_gpu_marks(num_gpus=2),
418 **COMMON_BROADCAST_SETTINGS # type: ignore
419 ),
420 ### Custom input edge-cases for specific models
@@ -468,6 +453,41 @@ VLM_TEST_SETTINGS = {
453 # yapf: enable
454
455
456 +def _mark_splits(
457 + test_settings: dict[str, VLMTestInfo],
458 + *,
459 + num_groups: int,
460 +) -> dict[str, VLMTestInfo]:
461 + name_by_test_info_id = {id(v): k for k, v in test_settings.items()}
462 + test_infos_by_model = defaultdict[str, list[VLMTestInfo]](list)
463 +
464 + for info in test_settings.values():
465 + for model in info.models:
466 + test_infos_by_model[model].append(info)
467 +
468 + models = sorted(test_infos_by_model.keys())
469 + split_size = math.ceil(len(models) / num_groups)
470 +
471 + new_test_settings = dict[str, VLMTestInfo]()
472 +
473 + for i in range(num_groups):
474 + models_in_group = models[i * split_size:(i + 1) * split_size]
475 +
476 + for model in models_in_group:
477 + for info in test_infos_by_model[model]:
478 + new_marks = (info.marks or []) + [pytest.mark.split(group=i)]
479 + new_info = info._replace(marks=new_marks)
480 + new_test_settings[name_by_test_info_id[id(info)]] = new_info
481 +
482 + missing_keys = test_settings.keys() - new_test_settings.keys()
483 + assert not missing_keys, f"Missing keys: {missing_keys}"
484 +
485 + return new_test_settings
486 +
487 +
488 +VLM_TEST_SETTINGS = _mark_splits(VLM_TEST_SETTINGS, num_groups=2)
489 +
490 +
491 ### Test wrappers
492 # Wrappers around the core test running func for:
493 # - single image
tests/utils.py
+25 -12
@@ -682,10 +682,12 @@ def fork_new_process_for_each_test(
682
683
684 def large_gpu_mark(min_gb: int) -> pytest.MarkDecorator:
685 - """Gets a pytest skipif mark, which triggers ig the the device doesn't have
686 - meet a minimum memory requirement in gb; can be leveraged via
687 - @large_gpu_test to skip tests in environments without enough resources, or
688 - called when filtering tests to run directly.
685 + """
686 + Get a pytest mark, which skips the test if the GPU doesn't meet
687 + a minimum memory requirement in GB.
688 +
689 + This can be leveraged via `@large_gpu_test` to skip tests in environments
690 + without enough resources, or called when filtering tests to run directly.
691 """
692 try:
693 if current_platform.is_cpu():
@@ -712,26 +714,37 @@ def large_gpu_test(*, min_gb: int):
714
715 Currently, the CI machine uses L4 GPU which has 24 GB VRAM.
716 """
715 - test_skipif = large_gpu_mark(min_gb)
717 + mark = large_gpu_mark(min_gb)
718
719 def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
718 - return test_skipif(f)
720 + return mark(f)
721
722 return wrapper
723
724
723 -def multi_gpu_test(*, num_gpus: int):
724 - """
725 - Decorate a test to be run only when multiple GPUs are available.
726 - """
727 - test_selector = getattr(pytest.mark, f"distributed_{num_gpus}_gpus")
725 +def multi_gpu_marks(*, num_gpus: int):
726 + """Get a collection of pytest marks to apply for `@multi_gpu_test`."""
727 + test_selector = pytest.mark.distributed(num_gpus=num_gpus)
728 test_skipif = pytest.mark.skipif(
729 cuda_device_count_stateless() < num_gpus,
730 reason=f"Need at least {num_gpus} GPUs to run the test.",
731 )
732
733 + return [test_selector, test_skipif]
734 +
735 +
736 +def multi_gpu_test(*, num_gpus: int):
737 + """
738 + Decorate a test to be run only when multiple GPUs are available.
739 + """
740 + marks = multi_gpu_marks(num_gpus=num_gpus)
741 +
742 def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
734 - return test_selector(test_skipif(fork_new_process_for_each_test(f)))
743 + func = fork_new_process_for_each_test(f)
744 + for mark in reversed(marks):
745 + func = mark(func)
746 +
747 + return func
748
749 return wrapper
750