A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[CI/Build] Split up VLM tests (#11083)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 12, 2024 at 06:18 UTC
d1e21a979bba4712f48dac1bbf410e0b57c92e7a
4 files changed
+94
-50
.buildkite/test-pipeline.yaml
+21
-11
@@ -321,7 +321,7 @@ steps:
321
322
##### models test #####
323
324
-- label: Basic Models Test # 30min
324
+- label: Basic Models Test # 24min
325
source_file_dependencies:
326
- vllm/
327
- tests/models
@@ -331,7 +331,7 @@ steps:
331
- pytest -v -s models/test_registry.py
332
- pytest -v -s models/test_initialization.py
333
334
-- label: Language Models Test (Standard) # 42min
334
+- label: Language Models Test (Standard) # 32min
335
#mirror_hardwares: [amd]
336
source_file_dependencies:
337
- vllm/
@@ -342,7 +342,7 @@ steps:
342
- pytest -v -s models/decoder_only/language -m 'core_model or quant_model'
343
- pytest -v -s models/embedding/language -m core_model
344
345
-- label: Language Models Test (Extended) # 50min
345
+- label: Language Models Test (Extended) # 1h10min
346
optional: true
347
source_file_dependencies:
348
- vllm/
@@ -353,7 +353,7 @@ steps:
353
- pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
354
- pytest -v -s models/embedding/language -m 'not core_model'
355
356
-- label: Multi-Modal Models Test (Standard) # 26min
356
+- label: Multi-Modal Models Test (Standard) # 28min
357
#mirror_hardwares: [amd]
358
source_file_dependencies:
359
- vllm/
@@ -369,7 +369,7 @@ steps:
369
- pytest -v -s models/encoder_decoder/language -m core_model
370
- pytest -v -s models/encoder_decoder/vision_language -m core_model
371
372
-- label: Multi-Modal Models Test (Extended) # 1h15m
372
+- label: Multi-Modal Models Test (Extended) 1 # 1h16m
373
optional: true
374
source_file_dependencies:
375
- vllm/
@@ -380,14 +380,24 @@ steps:
380
commands:
381
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
382
- pytest -v -s models/decoder_only/audio_language -m 'not core_model and not quant_model'
383
+ - pytest -v -s models/decoder_only/vision_language/test_models.py -m 'split(group=0) and not core_model and not quant_model'
384
# HACK - run phi3v tests separately to sidestep this transformers bug
385
# https://github.com/huggingface/transformers/issues/34307
386
- pytest -v -s models/decoder_only/vision_language/test_phi3v.py
386
- - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
387
+ - pytest -v -s --ignore models/decoder_only/vision_language/test_models.py --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
388
- pytest -v -s models/embedding/vision_language -m 'not core_model'
389
- pytest -v -s models/encoder_decoder/language -m 'not core_model'
390
- pytest -v -s models/encoder_decoder/vision_language -m 'not core_model'
391
392
+- label: Multi-Modal Models Test (Extended) 2 # 38m
393
+ optional: true
394
+ source_file_dependencies:
395
+ - vllm/
396
+ - tests/models/decoder_only/vision_language
397
+ commands:
398
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
399
+ - pytest -v -s models/decoder_only/vision_language/test_models.py -m 'split(group=1) and not core_model and not quant_model'
400
+
401
# This test is used only in PR development phase to test individual models and should never run on main
402
- label: Custom Models Test
403
optional: true
@@ -446,11 +456,11 @@ steps:
456
- pytest -v -s ./compile/test_basic_correctness.py
457
- pytest -v -s ./compile/test_wrapper.py
458
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
449
- - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
459
+ - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
460
# Avoid importing model tests that cause CUDA reinitialization error
451
- - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
452
- - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
453
- - pytest models/decoder_only/vision_language/test_models.py -v -s -m distributed_2_gpus
461
+ - pytest models/encoder_decoder/language/test_bart.py -v -s -m 'distributed(num_gpus=2)'
462
+ - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m 'distributed(num_gpus=2)'
463
+ - pytest models/decoder_only/vision_language/test_models.py -v -s -m 'distributed(num_gpus=2)'
464
- pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
465
- pip install -e ./plugins/vllm_add_dummy_model
466
- pytest -v -s distributed/test_distributed_oot.py
@@ -540,7 +550,7 @@ steps:
550
# see https://github.com/vllm-project/vllm/pull/5689 for details
551
- pytest -v -s distributed/test_custom_all_reduce.py
552
- torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
543
- - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m distributed_2_gpus
553
+ - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
554
- pytest -v -s -x lora/test_mixtral.py
555
556
- label: LM Eval Large Models # optional
pyproject.toml
+2
-1
@@ -96,7 +96,8 @@ markers = [
96
"core_model: enable this model test in each PR instead of only nightly",
97
"cpu_model: enable this model test in CPU tests",
98
"quant_model: run this model test under Quantized category",
99
- "distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
99
+ "split: run this test as part of a split",
100
+ "distributed: run this test only in distributed GPU tests",
101
"skip_v1: do not run this test with v1",
102
"optional: optional tests that are automatically skipped, include --optional to run them",
103
]
tests/models/decoder_only/vision_language/test_models.py
+46
-26
@@ -1,7 +1,9 @@
1
"""Common tests for testing .generate() functionality for single / multiple
2
image, embedding, and video support for different VLMs in vLLM.
3
"""
4
+import math
5
import os
6
+from collections import defaultdict
7
from pathlib import PosixPath
8
from typing import Type
9
@@ -10,11 +12,12 @@ from transformers import AutoModelForVision2Seq
12
from transformers.utils import is_flash_attn_2_available
13
14
from vllm.platforms import current_platform
13
-from vllm.utils import cuda_device_count_stateless, identity
15
+from vllm.utils import identity
16
17
from ....conftest import (IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets,
18
_VideoAssets)
17
-from ....utils import fork_new_process_for_each_test, large_gpu_mark
19
+from ....utils import (fork_new_process_for_each_test, large_gpu_mark,
20
+ multi_gpu_marks)
21
from ...utils import check_outputs_equal
22
from .vlm_utils import custom_inputs, model_utils, runners
23
from .vlm_utils.case_filtering import get_parametrized_options
@@ -382,7 +385,7 @@ VLM_TEST_SETTINGS = {
385
prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
386
),
387
### Tensor parallel / multi-gpu broadcast tests
385
- "broadcast-chameleon": VLMTestInfo(
388
+ "chameleon-broadcast": VLMTestInfo(
389
models=["facebook/chameleon-7b"],
390
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
391
max_model_len=4096,
@@ -393,43 +396,25 @@ VLM_TEST_SETTINGS = {
396
vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
397
hf_output_post_proc = lambda hf_output, model: hf_output[:2],
398
comparator=check_outputs_equal,
396
- marks=[
397
- pytest.mark.distributed_2_gpus,
398
- pytest.mark.skipif(
399
- cuda_device_count_stateless() < 2,
400
- reason="Need at least 2 GPUs to run the test.",
401
- ),
402
- ],
399
+ marks=multi_gpu_marks(num_gpus=2),
400
**COMMON_BROADCAST_SETTINGS # type: ignore
401
),
405
- "broadcast-llava": VLMTestInfo(
402
+ "llava-broadcast": VLMTestInfo(
403
models=["llava-hf/llava-1.5-7b-hf"],
404
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
405
max_model_len=4096,
406
auto_cls=AutoModelForVision2Seq,
407
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
411
- marks=[
412
- pytest.mark.distributed_2_gpus,
413
- pytest.mark.skipif(
414
- cuda_device_count_stateless() < 2,
415
- reason="Need at least 2 GPUs to run the test.",
416
- )
417
- ],
408
+ marks=multi_gpu_marks(num_gpus=2),
409
**COMMON_BROADCAST_SETTINGS # type: ignore
410
),
420
- "broadcast-llava_next": VLMTestInfo(
411
+ "llava_next-broadcast": VLMTestInfo(
412
models=["llava-hf/llava-v1.6-mistral-7b-hf"],
413
prompt_formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]",
414
max_model_len=10240,
415
auto_cls=AutoModelForVision2Seq,
416
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
426
- marks=[
427
- pytest.mark.distributed_2_gpus,
428
- pytest.mark.skipif(
429
- cuda_device_count_stateless() < 2,
430
- reason="Need at least 2 GPUs to run the test.",
431
- )
432
- ],
417
+ marks=multi_gpu_marks(num_gpus=2),
418
**COMMON_BROADCAST_SETTINGS # type: ignore
419
),
420
### Custom input edge-cases for specific models
@@ -468,6 +453,41 @@ VLM_TEST_SETTINGS = {
453
# yapf: enable
454
455
456
+def _mark_splits(
457
+ test_settings: dict[str, VLMTestInfo],
458
+ *,
459
+ num_groups: int,
460
+) -> dict[str, VLMTestInfo]:
461
+ name_by_test_info_id = {id(v): k for k, v in test_settings.items()}
462
+ test_infos_by_model = defaultdict[str, list[VLMTestInfo]](list)
463
+
464
+ for info in test_settings.values():
465
+ for model in info.models:
466
+ test_infos_by_model[model].append(info)
467
+
468
+ models = sorted(test_infos_by_model.keys())
469
+ split_size = math.ceil(len(models) / num_groups)
470
+
471
+ new_test_settings = dict[str, VLMTestInfo]()
472
+
473
+ for i in range(num_groups):
474
+ models_in_group = models[i * split_size:(i + 1) * split_size]
475
+
476
+ for model in models_in_group:
477
+ for info in test_infos_by_model[model]:
478
+ new_marks = (info.marks or []) + [pytest.mark.split(group=i)]
479
+ new_info = info._replace(marks=new_marks)
480
+ new_test_settings[name_by_test_info_id[id(info)]] = new_info
481
+
482
+ missing_keys = test_settings.keys() - new_test_settings.keys()
483
+ assert not missing_keys, f"Missing keys: {missing_keys}"
484
+
485
+ return new_test_settings
486
+
487
+
488
+VLM_TEST_SETTINGS = _mark_splits(VLM_TEST_SETTINGS, num_groups=2)
489
+
490
+
491
### Test wrappers
492
# Wrappers around the core test running func for:
493
# - single image
tests/utils.py
+25
-12
@@ -682,10 +682,12 @@ def fork_new_process_for_each_test(
682
683
684
def large_gpu_mark(min_gb: int) -> pytest.MarkDecorator:
685
- """Gets a pytest skipif mark, which triggers ig the the device doesn't have
686
- meet a minimum memory requirement in gb; can be leveraged via
687
- @large_gpu_test to skip tests in environments without enough resources, or
688
- called when filtering tests to run directly.
685
+ """
686
+ Get a pytest mark, which skips the test if the GPU doesn't meet
687
+ a minimum memory requirement in GB.
688
+
689
+ This can be leveraged via `@large_gpu_test` to skip tests in environments
690
+ without enough resources, or called when filtering tests to run directly.
691
"""
692
try:
693
if current_platform.is_cpu():
@@ -712,26 +714,37 @@ def large_gpu_test(*, min_gb: int):
714
715
Currently, the CI machine uses L4 GPU which has 24 GB VRAM.
716
"""
715
- test_skipif = large_gpu_mark(min_gb)
717
+ mark = large_gpu_mark(min_gb)
718
719
def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
718
- return test_skipif(f)
720
+ return mark(f)
721
722
return wrapper
723
724
723
-def multi_gpu_test(*, num_gpus: int):
724
- """
725
- Decorate a test to be run only when multiple GPUs are available.
726
- """
727
- test_selector = getattr(pytest.mark, f"distributed_{num_gpus}_gpus")
725
+def multi_gpu_marks(*, num_gpus: int):
726
+ """Get a collection of pytest marks to apply for `@multi_gpu_test`."""
727
+ test_selector = pytest.mark.distributed(num_gpus=num_gpus)
728
test_skipif = pytest.mark.skipif(
729
cuda_device_count_stateless() < num_gpus,
730
reason=f"Need at least {num_gpus} GPUs to run the test.",
731
)
732
733
+ return [test_selector, test_skipif]
734
+
735
+
736
+def multi_gpu_test(*, num_gpus: int):
737
+ """
738
+ Decorate a test to be run only when multiple GPUs are available.
739
+ """
740
+ marks = multi_gpu_marks(num_gpus=num_gpus)
741
+
742
def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
734
- return test_selector(test_skipif(fork_new_process_for_each_test(f)))
743
+ func = fork_new_process_for_each_test(f)
744
+ for mark in reversed(marks):
745
+ func = mark(func)
746
+
747
+ return func
748
749
return wrapper
750