A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[CI/Build] Reorganize models tests (#7820)
Cyrus Leung committed
Sep 14, 2024 at 01:20 UTC
a84e598e2125960d3b4f716b78863f24ac562947
55 files changed
+415
-498
.buildkite/run-cpu-test.sh
+4
-6
@@ -23,12 +23,10 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23
# Run basic model test
24
docker exec cpu-test bash -c "
25
pip install pytest matplotlib einops transformers_stream_generator
26
- pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py \
27
- --ignore=tests/models/test_oot_registration.py \
28
- --ignore=tests/models/test_registry.py \
29
- --ignore=tests/models/test_fp8.py \
30
- --ignore=tests/models/test_jamba.py \
31
- --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
26
+ pytest -v -s tests/models/decoder_only/language \
27
+ --ignore=tests/models/test_fp8.py \
28
+ --ignore=tests/models/decoder_only/language/test_jamba.py \
29
+ --ignore=tests/models/decoder_only/language/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
30
31
# Run compressed-tensor test
32
docker exec cpu-test bash -c "
.buildkite/test-pipeline.yaml
+45
-25
@@ -94,7 +94,6 @@ steps:
94
- pytest -v -s entrypoints/test_chat_utils.py
95
- pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
96
97
-
97
- label: Distributed Tests (4 GPUs) # 10min
98
working_dir: "/vllm-workspace/tests"
99
num_gpus: 4
@@ -164,15 +163,6 @@ steps:
163
- python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
164
- python3 offline_inference_encoder_decoder.py
165
167
-- label: Models Test # 1hr10min
168
- source_file_dependencies:
169
- - vllm/
170
- - tests/models
171
- commands:
172
- - pip install -e ./plugins/vllm_add_dummy_model
173
- - pytest -v -s models/test_oot_registration.py # it needs a clean process
174
- - pytest -v -s models -m \"not vlm\" --ignore=models/test_oot_registration.py
175
-
166
- label: torch compile integration test
167
source_file_dependencies:
168
- vllm/
@@ -180,14 +170,6 @@ steps:
170
- pytest -v -s ./compile/test_full_graph.py
171
- pytest -v -s ./compile/test_wrapper.py
172
183
-
184
-- label: Vision Language Models Test # 42min
185
- #mirror_hardwares: [amd]
186
- source_file_dependencies:
187
- - vllm/
188
- commands:
189
- - pytest -v -s models -m vlm
190
-
173
- label: Prefix Caching Test # 7min
174
#mirror_hardwares: [amd]
175
source_file_dependencies:
@@ -286,6 +268,45 @@ steps:
268
commands:
269
- pytest -v -s tool_use
270
271
+##### models test #####
272
+
273
+- label: Basic Models Test # 3min
274
+ source_file_dependencies:
275
+ - vllm/
276
+ - tests/models
277
+ commands:
278
+ - pip install -e ./plugins/vllm_add_dummy_model
279
+ - pytest -v -s models/test_oot_registration.py # it needs a clean process
280
+ - pytest -v -s models/*.py --ignore=models/test_oot_registration.py
281
+
282
+- label: Decoder-only Language Models Test # 1h3min
283
+ #mirror_hardwares: [amd]
284
+ source_file_dependencies:
285
+ - vllm/
286
+ - tests/models/decoder_only/language
287
+ commands:
288
+ - pytest -v -s models/decoder_only/language
289
+
290
+- label: Decoder-only Multi-Modal Models Test # 56min
291
+ #mirror_hardwares: [amd]
292
+ source_file_dependencies:
293
+ - vllm/
294
+ - tests/models/decoder_only/audio_language
295
+ - tests/models/decoder_only/vision_language
296
+ commands:
297
+ - pytest -v -s models/decoder_only/audio_language
298
+ - pytest -v -s models/decoder_only/vision_language
299
+
300
+- label: Other Models Test # 5min
301
+ #mirror_hardwares: [amd]
302
+ source_file_dependencies:
303
+ - vllm/
304
+ - tests/models/embedding/language
305
+ - tests/models/encoder_decoder/language
306
+ commands:
307
+ - pytest -v -s models/embedding/language
308
+ - pytest -v -s models/encoder_decoder/language
309
+
310
##### 1 GPU test #####
311
##### multi gpus test #####
312
@@ -311,11 +332,11 @@ steps:
332
- tests/distributed/
333
commands:
334
- # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
314
- - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
335
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep -q 'Same node test passed'
336
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py
337
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
338
- # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
318
- - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
339
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep -q 'Same node test passed'
340
341
- label: Distributed Tests (2 GPUs) # 28min
342
#mirror_hardwares: [amd]
@@ -328,11 +349,10 @@ steps:
349
- vllm/model_executor/models/
350
- tests/distributed/
351
commands:
331
- - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
332
- - TARGET_TEST_SUITE=L4 pytest -v -s distributed/test_basic_distributed_correctness.py
333
- - pytest -v -s distributed/test_basic_distributed_correctness_enc_dec.py
334
- - pytest -v -s distributed/test_chunked_prefill_distributed.py
335
- - pytest -v -s distributed/test_multimodal_broadcast.py
352
+ - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
353
+ - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
354
+ # Avoid importing model tests that cause CUDA reinitialization error
355
+ - pytest models/encoder_decoder/language/test_bart.py models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
356
- pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
357
- pip install -e ./plugins/vllm_add_dummy_model
358
- pytest -v -s distributed/test_distributed_oot.py
docs/source/models/supported_models.rst
+1
-1
@@ -342,7 +342,7 @@ Note that, as an inference engine, vLLM does not introduce new models. Therefore
342
343
We have the following levels of testing for models:
344
345
-1. **Strict Consistency**: We compare the output of the model with the output of the model in the HuggingFace Transformers library under greedy decoding. This is the most stringent test. Please refer to `test_models.py <https://github.com/vllm-project/vllm/blob/main/tests/models/test_models.py>`_ and `test_big_models.py <https://github.com/vllm-project/vllm/blob/main/tests/models/test_big_models.py>`_ for the models that have passed this test.
345
+1. **Strict Consistency**: We compare the output of the model with the output of the model in the HuggingFace Transformers library under greedy decoding. This is the most stringent test. Please refer to `models tests <https://github.com/vllm-project/vllm/blob/main/tests/models>`_ for the models that have passed this test.
346
2. **Output Sensibility**: We check if the output of the model is sensible and coherent, by measuring the perplexity of the output and checking for any obvious errors. This is a less stringent test.
347
3. **Runtime Functionality**: We check if the model can be loaded and run without errors. This is the least stringent test. Please refer to `functionality tests <https://github.com/vllm-project/vllm/tree/main/tests>`_ and `examples <https://github.com/vllm-project/vllm/tree/main/examples>`_ for the models that have passed this test.
348
4. **Community Feedback**: We rely on the community to provide feedback on the models. If a model is broken or not working as expected, we encourage users to raise issues to report it or open pull requests to fix it. The rest of the models fall under this category.
pyproject.toml
+2
-1
@@ -85,5 +85,6 @@ skip_gitignore = true
85
[tool.pytest.ini_options]
86
markers = [
87
"skip_global_cleanup",
88
- "vlm: run tests for vision language models only",
88
+ "core_model: run this model test in each PR instead of just daily",
89
+ "distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
90
]
tests/basic_correctness/test_basic_correctness.py
+62
@@ -15,12 +15,15 @@ from vllm.utils import is_hip
15
from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
16
17
from ..models.utils import check_outputs_equal
18
+from ..utils import multi_gpu_test
19
20
MODELS = [
21
"facebook/opt-125m",
22
"meta-llama/Llama-2-7b-hf",
23
]
24
25
+TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
26
+
27
28
def test_vllm_gc_ed():
29
"""Verify vllm instance is GC'ed when it is deleted"""
@@ -70,6 +73,65 @@ def test_models(
73
)
74
75
76
+@multi_gpu_test(num_gpus=2)
77
+@pytest.mark.parametrize(
78
+ "model, distributed_executor_backend, attention_backend, "
79
+ "test_suite", [
80
+ ("facebook/opt-125m", "ray", "", "L4"),
81
+ ("facebook/opt-125m", "mp", "", "L4"),
82
+ ("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
83
+ ("meta-llama/Llama-2-7b-hf", "mp", "", "L4"),
84
+ ("facebook/opt-125m", "ray", "", "A100"),
85
+ ("facebook/opt-125m", "mp", "", "A100"),
86
+ ("facebook/opt-125m", "mp", "FLASHINFER", "A100"),
87
+ ("meta-llama/Meta-Llama-3-8B", "ray", "FLASHINFER", "A100"),
88
+ ])
89
+def test_models_distributed(
90
+ hf_runner,
91
+ vllm_runner,
92
+ example_prompts,
93
+ model: str,
94
+ distributed_executor_backend: str,
95
+ attention_backend: str,
96
+ test_suite: str,
97
+) -> None:
98
+
99
+ if test_suite != TARGET_TEST_SUITE:
100
+ pytest.skip(f"Skip test for {test_suite}")
101
+
102
+ if model == "meta-llama/Llama-2-7b-hf" and distributed_executor_backend == "ray" and attention_backend == "" and test_suite == "L4": # noqa
103
+ # test ray adag
104
+ os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
105
+ os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
106
+
107
+ if attention_backend:
108
+ os.environ["VLLM_ATTENTION_BACKEND"] = attention_backend
109
+
110
+ dtype = "half"
111
+ max_tokens = 5
112
+
113
+ # NOTE: take care of the order. run vLLM first, and then run HF.
114
+ # vLLM needs a fresh new process without cuda initialization.
115
+ # if we run HF first, the cuda initialization will be done and it
116
+ # will hurt multiprocessing backend with fork method (the default method).
117
+ with vllm_runner(model,
118
+ dtype=dtype,
119
+ tensor_parallel_size=2,
120
+ distributed_executor_backend=distributed_executor_backend
121
+ ) as vllm_model:
122
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
123
+
124
+ with hf_runner(model, dtype=dtype) as hf_model:
125
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
126
+
127
+ check_outputs_equal(
128
+ outputs_0_lst=hf_outputs,
129
+ outputs_1_lst=vllm_outputs,
130
+ name_0="hf",
131
+ name_1="vllm",
132
+ )
133
+
134
+
135
def test_model_with_failure(vllm_runner) -> None:
136
try:
137
with patch("vllm.model_executor.models.opt.OPTForCausalLM.forward",
tests/basic_correctness/test_chunked_prefill.py
+55
@@ -6,11 +6,13 @@ prefill requests are chunked.
6
7
Run `pytest tests/models/test_chunked_prefill.py`.
8
"""
9
+import os
10
from contextlib import nullcontext
11
12
import pytest
13
14
from ..models.utils import check_logprobs_close, check_outputs_equal
15
+from ..utils import multi_gpu_test
16
17
MODELS = [
18
"facebook/opt-125m",
@@ -66,6 +68,59 @@ def test_models(
68
)
69
70
71
+@multi_gpu_test(num_gpus=2)
72
+@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
73
+@pytest.mark.parametrize("model", MODELS)
74
+def test_models_distributed(
75
+ hf_runner,
76
+ vllm_runner,
77
+ example_prompts,
78
+ model: str,
79
+ distributed_executor_backend: str,
80
+) -> None:
81
+ if (model == "meta-llama/Llama-2-7b-hf"
82
+ and distributed_executor_backend == "ray"):
83
+ # test ray adag
84
+ os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
85
+ os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
86
+
87
+ dtype = "half"
88
+ max_tokens = 5
89
+ chunked_prefill_token_size = 16
90
+
91
+ # Add a chunked prefill config.
92
+ max_num_seqs = min(chunked_prefill_token_size, 256)
93
+ assert chunked_prefill_token_size != -1
94
+ enable_chunked_prefill = True
95
+ max_num_batched_tokens = chunked_prefill_token_size
96
+
97
+ # NOTE: take care of the order. run vLLM first, and then run HF.
98
+ # vLLM needs a fresh new process without cuda initialization.
99
+ # if we run HF first, the cuda initialization will be done and it
100
+ # will hurt multiprocessing backend with fork method (the default method).
101
+
102
+ with vllm_runner(
103
+ model,
104
+ dtype=dtype,
105
+ tensor_parallel_size=2,
106
+ max_num_seqs=max_num_seqs,
107
+ enable_chunked_prefill=enable_chunked_prefill,
108
+ max_num_batched_tokens=max_num_batched_tokens,
109
+ distributed_executor_backend=distributed_executor_backend,
110
+ ) as vllm_model:
111
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
112
+
113
+ with hf_runner(model, dtype=dtype) as hf_model:
114
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
115
+
116
+ check_outputs_equal(
117
+ outputs_0_lst=hf_outputs,
118
+ outputs_1_lst=vllm_outputs,
119
+ name_0="hf",
120
+ name_1="vllm",
121
+ )
122
+
123
+
124
@pytest.mark.parametrize(
125
"kv_cache_dtype,model",
126
[("fp8_e4m3",
tests/basic_correctness/test_preemption.py
+7
-4
@@ -19,10 +19,13 @@ MODELS = [
19
"facebook/opt-125m",
20
]
21
22
-assert ENABLE_ARTIFICIAL_PREEMPT is True, (
23
- "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1. "
24
- "`VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest "
25
- "tests/basic_correctness/test_preemption.py`")
22
+
23
+@pytest.fixture(scope="module", autouse=True)
24
+def check_settings():
25
+ assert ENABLE_ARTIFICIAL_PREEMPT is True, (
26
+ "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1. "
27
+ "`VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest "
28
+ "tests/basic_correctness/test_preemption.py`")
29
30
31
@pytest.fixture
tests/conftest.py
+12
-17
@@ -6,8 +6,8 @@ import sys
6
import tempfile
7
from collections import UserList
8
from enum import Enum
9
-from typing import (Any, Callable, Dict, List, Optional, Tuple, TypedDict,
10
- TypeVar, Union)
9
+from typing import (Any, Callable, Dict, List, Optional, Tuple, Type,
10
+ TypedDict, TypeVar, Union)
11
12
import numpy as np
13
import pytest
@@ -18,6 +18,7 @@ from huggingface_hub import snapshot_download
18
from PIL import Image
19
from transformers import (AutoModelForCausalLM, AutoTokenizer, BatchEncoding,
20
BatchFeature)
21
+from transformers.models.auto.auto_factory import _BaseAutoModelClass
22
23
from vllm import LLM, SamplingParams
24
from vllm.assets.image import ImageAsset
@@ -260,7 +261,7 @@ class HfRunner:
261
*,
262
model_kwargs: Optional[Dict[str, Any]] = None,
263
is_embedding_model: bool = False,
263
- auto_cls=AutoModelForCausalLM,
264
+ auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM,
265
postprocess_inputs: Callable[[BatchEncoding],
266
BatchEncoding] = identity,
267
) -> None:
@@ -292,20 +293,14 @@ class HfRunner:
293
trust_remote_code=True,
294
)
295
295
- try:
296
- # don't put this import at the top level
297
- # it will call torch.cuda.device_count()
298
- from transformers import AutoProcessor # noqa: F401
299
- self.processor = AutoProcessor.from_pretrained(
300
- model_name,
301
- torch_dtype=torch_dtype,
302
- trust_remote_code=True,
303
- )
304
- except Exception as exc:
305
- logger.warning(
306
- "Unable to auto-load HuggingFace processor for model (%s). "
307
- "Using tokenizer instead. Reason: %s", model_name, exc)
308
- self.processor = self.tokenizer
296
+ # don't put this import at the top level
297
+ # it will call torch.cuda.device_count()
298
+ from transformers import AutoProcessor # noqa: F401
299
+ self.processor = AutoProcessor.from_pretrained(
300
+ model_name,
301
+ torch_dtype=torch_dtype,
302
+ trust_remote_code=True,
303
+ )
304
305
self.postprocess_inputs = postprocess_inputs
306
tests/distributed/test_basic_distributed_correctness.py
deleted
-80
@@ -1,80 +0,0 @@
1
-"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2
-
3
-Run:
4
-```sh
5
-cd $VLLM_PATH/tests
6
-
7
-pytest distributed/test_basic_distributed_correctness.py
8
-```
9
-"""
10
-import os
11
-
12
-import pytest
13
-
14
-from vllm.utils import cuda_device_count_stateless
15
-
16
-from ..models.utils import check_outputs_equal
17
-from ..utils import fork_new_process_for_each_test
18
-
19
-TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
20
-
21
-
22
-@pytest.mark.skipif(cuda_device_count_stateless() < 2,
23
- reason="Need at least 2 GPUs to run the test.")
24
-@pytest.mark.parametrize(
25
- "model, distributed_executor_backend, attention_backend, "
26
- "test_suite", [
27
- ("facebook/opt-125m", "ray", "", "L4"),
28
- ("facebook/opt-125m", "mp", "", "L4"),
29
- ("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
30
- ("meta-llama/Llama-2-7b-hf", "mp", "", "L4"),
31
- ("facebook/opt-125m", "ray", "", "A100"),
32
- ("facebook/opt-125m", "mp", "", "A100"),
33
- ("facebook/opt-125m", "mp", "FLASHINFER", "A100"),
34
- ("meta-llama/Meta-Llama-3-8B", "ray", "FLASHINFER", "A100"),
35
- ])
36
-@fork_new_process_for_each_test
37
-def test_models(
38
- hf_runner,
39
- vllm_runner,
40
- example_prompts,
41
- model: str,
42
- distributed_executor_backend: str,
43
- attention_backend: str,
44
- test_suite: str,
45
-) -> None:
46
-
47
- if test_suite != TARGET_TEST_SUITE:
48
- pytest.skip(f"Skip test for {test_suite}")
49
-
50
- if model == "meta-llama/Llama-2-7b-hf" and distributed_executor_backend == "ray" and attention_backend == "" and test_suite == "L4": # noqa
51
- # test ray adag
52
- os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
53
- os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
54
-
55
- if attention_backend:
56
- os.environ["VLLM_ATTENTION_BACKEND"] = attention_backend
57
-
58
- dtype = "half"
59
- max_tokens = 5
60
-
61
- # NOTE: take care of the order. run vLLM first, and then run HF.
62
- # vLLM needs a fresh new process without cuda initialization.
63
- # if we run HF first, the cuda initialization will be done and it
64
- # will hurt multiprocessing backend with fork method (the default method).
65
- with vllm_runner(model,
66
- dtype=dtype,
67
- tensor_parallel_size=2,
68
- distributed_executor_backend=distributed_executor_backend
69
- ) as vllm_model:
70
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
71
-
72
- with hf_runner(model, dtype=dtype) as hf_model:
73
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
74
-
75
- check_outputs_equal(
76
- outputs_0_lst=hf_outputs,
77
- outputs_1_lst=vllm_outputs,
78
- name_0="hf",
79
- name_1="vllm",
80
- )
tests/distributed/test_basic_distributed_correctness_enc_dec.py
deleted
-102
@@ -1,102 +0,0 @@
1
-"""For encoder/decoder models only:
2
-Compare the outputs of HF and distributed vLLM when using greedy sampling.
3
-
4
-Run:
5
-```sh
6
-cd $VLLM_PATH/tests
7
-
8
-pytest distributed/test_basic_distributed_correctness_enc_dec.py
9
-```
10
-"""
11
-
12
-import pytest
13
-from transformers import AutoModelForSeq2SeqLM
14
-
15
-from vllm.utils import cuda_device_count_stateless
16
-
17
-from ..conftest import DecoderPromptType
18
-from ..models.utils import check_logprobs_close
19
-from ..utils import fork_new_process_for_each_test
20
-
21
-
22
-@pytest.mark.skipif(cuda_device_count_stateless() < 2,
23
- reason="Need at least 2 GPUs to run the test.")
24
-@pytest.mark.parametrize("model, distributed_executor_backend", [
25
- ("facebook/bart-large-cnn", "ray"),
26
- ("facebook/bart-large-cnn", "mp"),
27
-])
28
-@fork_new_process_for_each_test
29
-def test_models(
30
- model: str,
31
- distributed_executor_backend: str,
32
- hf_runner,
33
- vllm_runner,
34
- example_encoder_decoder_prompts,
35
-) -> None:
36
- '''
37
- Test vLLM BART inference on more than one GPU, comparing
38
- outputs against HF as a baseline.
39
-
40
- Fork a new process for each test, to prevent CUDA from
41
- being re-initialized by successive tests within the same
42
- process.
43
-
44
- Arguments:
45
-
46
- * model: the HF ID of the specific BART variant under test
47
- * distributed_executor_backend
48
- * hf_runner: HuggingFace (HF) test model runner
49
- * vllm_runner: vLLM test model runner
50
- * example_encoder_decoder_prompts: test fixture which provides a
51
- dictionary of dummy prompts
52
- '''
53
-
54
- dtype = "float"
55
- max_tokens = 64
56
- num_logprobs = 5
57
-
58
- # Example inputs with non-trivial (i.e. not None/empty) encoder &
59
- # decoder prompts.
60
- test_prompts = example_encoder_decoder_prompts[DecoderPromptType.CUSTOM]
61
-
62
- # NOTE: take care of the order. run vLLM first, and then run HF.
63
- # vLLM needs a fresh new process without cuda initialization.
64
- # if we run HF first, the cuda initialization will be done and it
65
- # will hurt multiprocessing backend with fork method (the default method).
66
- with vllm_runner(
67
- model,
68
- dtype=dtype,
69
- tensor_parallel_size=2,
70
- distributed_executor_backend=distributed_executor_backend,
71
- enforce_eager=True,
72
- ) as vllm_model:
73
- vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
74
- test_prompts, max_tokens, num_logprobs)
75
-
76
- # Configuration settings for HF baseline
77
- hf_kwargs = {
78
- "top_k": None,
79
- "num_beams": 1,
80
- "repetition_penalty": 1.0,
81
- "top_p": 1.0,
82
- "length_penalty": 1.0,
83
- "early_stopping": False,
84
- "no_repeat_ngram_size": None,
85
- "min_length": 0
86
- }
87
-
88
- with hf_runner(model, dtype=dtype,
89
- auto_cls=AutoModelForSeq2SeqLM) as hf_model:
90
- hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
91
- test_prompts,
92
- max_tokens,
93
- num_logprobs,
94
- **hf_kwargs,
95
- ))
96
-
97
- check_logprobs_close(
98
- outputs_0_lst=hf_outputs,
99
- outputs_1_lst=vllm_outputs,
100
- name_0="hf",
101
- name_1="vllm",
102
- )
tests/distributed/test_chunked_prefill_distributed.py
deleted
-75
@@ -1,75 +0,0 @@
1
-"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2
-
3
-Run:
4
-```sh
5
-pytest test_chunked_prefill_distributed.py
6
-```
7
-"""
8
-
9
-import os
10
-
11
-import pytest
12
-
13
-from vllm.utils import cuda_device_count_stateless
14
-
15
-from ..models.utils import check_outputs_equal
16
-from ..utils import fork_new_process_for_each_test
17
-
18
-
19
-@pytest.mark.skipif(cuda_device_count_stateless() < 2,
20
- reason="Need at least 2 GPUs to run the test.")
21
-@pytest.mark.parametrize("model, distributed_executor_backend", [
22
- ("facebook/opt-125m", "ray"),
23
- ("meta-llama/Llama-2-7b-hf", "ray"),
24
- ("facebook/opt-125m", "mp"),
25
- ("meta-llama/Llama-2-7b-hf", "mp"),
26
-])
27
-@fork_new_process_for_each_test
28
-def test_models(
29
- hf_runner,
30
- vllm_runner,
31
- example_prompts,
32
- model: str,
33
- distributed_executor_backend: str,
34
-) -> None:
35
- if model == "meta-llama/Llama-2-7b-hf" and distributed_executor_backend == "ray": # noqa
36
- assert distributed_executor_backend == "ray"
37
- # test ray adag
38
- os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
39
- os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
40
-
41
- dtype = "half"
42
- max_tokens = 5
43
- chunked_prefill_token_size = 16
44
-
45
- # Add a chunked prefill config.
46
- max_num_seqs = min(chunked_prefill_token_size, 256)
47
- assert chunked_prefill_token_size != -1
48
- enable_chunked_prefill = True
49
- max_num_batched_tokens = chunked_prefill_token_size
50
-
51
- # NOTE: take care of the order. run vLLM first, and then run HF.
52
- # vLLM needs a fresh new process without cuda initialization.
53
- # if we run HF first, the cuda initialization will be done and it
54
- # will hurt multiprocessing backend with fork method (the default method).
55
-
56
- with vllm_runner(
57
- model,
58
- dtype=dtype,
59
- tensor_parallel_size=2,
60
- max_num_seqs=max_num_seqs,
61
- enable_chunked_prefill=enable_chunked_prefill,
62
- max_num_batched_tokens=max_num_batched_tokens,
63
- distributed_executor_backend=distributed_executor_backend,
64
- ) as vllm_model:
65
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
66
-
67
- with hf_runner(model, dtype=dtype) as hf_model:
68
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
69
-
70
- check_outputs_equal(
71
- outputs_0_lst=hf_outputs,
72
- outputs_1_lst=vllm_outputs,
73
- name_0="hf",
74
- name_1="vllm",
75
- )
tests/distributed/test_multimodal_broadcast.py
deleted
-58
@@ -1,58 +0,0 @@
1
-"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2
-
3
-Run:
4
-```sh
5
-pytest -s -v test_multimodal_broadcast.py
6
-```
7
-"""
8
-
9
-import pytest
10
-
11
-from vllm.utils import cuda_device_count_stateless
12
-
13
-from ..utils import fork_new_process_for_each_test
14
-
15
-
16
-@pytest.mark.skipif(cuda_device_count_stateless() < 2,
17
- reason="Need at least 2 GPUs to run the test.")
18
-@pytest.mark.parametrize("model, distributed_executor_backend", [
19
- ("llava-hf/llava-1.5-7b-hf", "ray"),
20
- ("llava-hf/llava-v1.6-mistral-7b-hf", "ray"),
21
- ("facebook/chameleon-7b", "ray"),
22
- ("llava-hf/llava-1.5-7b-hf", "mp"),
23
- ("llava-hf/llava-v1.6-mistral-7b-hf", "mp"),
24
- ("facebook/chameleon-7b", "mp"),
25
-])
26
-@fork_new_process_for_each_test
27
-def test_models(hf_runner, vllm_runner, image_assets, model: str,
28
- distributed_executor_backend: str) -> None:
29
-
30
- dtype = "half"
31
- max_tokens = 5
32
- num_logprobs = 5
33
- tensor_parallel_size = 2
34
-
35
- if model.startswith("llava-hf/llava-1.5"):
36
- from ..models.test_llava import models, run_test
37
- elif model.startswith("llava-hf/llava-v1.6"):
38
- from ..models.test_llava_next import run_test # type: ignore[no-redef]
39
- from ..models.test_llava_next import models
40
- elif model.startswith("facebook/chameleon"):
41
- from ..models.test_chameleon import run_test # type: ignore[no-redef]
42
- from ..models.test_chameleon import models
43
- else:
44
- raise NotImplementedError(f"Unsupported model: {model}")
45
-
46
- run_test(
47
- hf_runner,
48
- vllm_runner,
49
- image_assets,
50
- model=models[0],
51
- # So that LLaVA-NeXT processor may return nested list
52
- size_factors=[0.25, 0.5, 1.0],
53
- dtype=dtype,
54
- max_tokens=max_tokens,
55
- num_logprobs=num_logprobs,
56
- tensor_parallel_size=tensor_parallel_size,
57
- distributed_executor_backend=distributed_executor_backend,
58
- )
tests/distributed/test_same_node.py
+7
-7
@@ -1,13 +1,13 @@
1
import os
2
3
-import torch
3
+import torch.distributed as dist
4
5
from vllm.distributed.parallel_state import in_the_same_node_as
6
7
-torch.distributed.init_process_group(backend="gloo")
8
-test_result = all(
9
- in_the_same_node_as(torch.distributed.group.WORLD, source_rank=0))
7
+if __name__ == "__main__":
8
+ dist.init_process_group(backend="gloo")
9
+ test_result = all(in_the_same_node_as(dist.group.WORLD, source_rank=0))
10
11
-expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
12
-assert test_result == expected, f"Expected {expected}, got {test_result}"
13
-print("Same node test passed!")
11
+ expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
12
+ assert test_result == expected, f"Expected {expected}, got {test_result}"
13
+ print("Same node test passed!")
tests/kernels/utils.py
+3
-1
@@ -10,7 +10,6 @@ import pytest
10
import torch
11
12
from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
13
-from vllm.attention.backends.xformers import XFormersBackend
13
from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
14
make_tensor_with_pad)
15
@@ -521,6 +520,9 @@ def make_backend(backend_name: str) -> AttentionBackend:
520
* Backend instance
521
'''
522
if backend_name == STR_XFORMERS_ATTN_VAL:
523
+ # NOTE: xFormers backend cannot be imported for CPU and AMD GPUs.
524
+ from vllm.attention.backends.xformers import XFormersBackend
525
+
526
return XFormersBackend()
527
raise AssertionError(
528
f"Unrecognized backend_name {backend_name} for unit test")
tests/models/decoder_only/__init__.py
tests/models/decoder_only/audio_language/__init__.py
tests/models/decoder_only/audio_language/test_ultravox.py
renamed
+2
-4
@@ -7,10 +7,8 @@ from transformers import AutoModel, AutoTokenizer, BatchEncoding
7
from vllm.sequence import SampleLogprobs
8
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
9
10
-from ..conftest import HfRunner, VllmRunner
11
-from .utils import check_logprobs_close
12
-
13
-pytestmark = pytest.mark.vlm
10
+from ....conftest import HfRunner, VllmRunner
11
+from ...utils import check_logprobs_close
12
13
MODEL_NAME = "fixie-ai/ultravox-v0_3"
14
tests/models/decoder_only/language/__init__.py
tests/models/decoder_only/language/test_aqlm.py
renamed
tests/models/decoder_only/language/test_big_models.py
renamed
+1
-1
@@ -7,7 +7,7 @@ Run `pytest tests/models/test_big_models.py`.
7
import pytest
8
import torch
9
10
-from .utils import check_outputs_equal
10
+from ...utils import check_outputs_equal
11
12
MODELS = [
13
"meta-llama/Llama-2-7b-hf",
tests/models/decoder_only/language/test_danube3_4b.py
renamed
+1
-1
@@ -6,7 +6,7 @@ Run `pytest tests/models/test_danube3_4b.py`.
6
"""
7
import pytest
8
9
-from .utils import check_outputs_equal
9
+from ...utils import check_outputs_equal
10
11
MODELS = ["h2oai/h2o-danube3-4b-base"]
12
tests/models/decoder_only/language/test_fp8.py
renamed
+1
-1
@@ -10,7 +10,7 @@ import pytest
10
from tests.kernels.utils import override_backend_env_variable
11
from tests.quantization.utils import is_quant_method_supported
12
13
-from ..models.utils import check_logprobs_close
13
+from ...utils import check_logprobs_close
14
15
os.environ["TOKENIZERS_PARALLELISM"] = "true"
16
tests/models/decoder_only/language/test_gguf.py
renamed
+1
-1
@@ -11,7 +11,7 @@ from transformers import AutoTokenizer
11
12
from tests.quantization.utils import is_quant_method_supported
13
14
-from .utils import check_logprobs_close
14
+from ...utils import check_logprobs_close
15
16
os.environ["TOKENIZERS_PARALLELISM"] = "true"
17
tests/models/decoder_only/language/test_gptq_marlin.py
renamed
+1
-1
@@ -15,7 +15,7 @@ import pytest
15
from tests.quantization.utils import is_quant_method_supported
16
from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
17
18
-from .utils import check_logprobs_close
18
+from ...utils import check_logprobs_close
19
20
os.environ["TOKENIZERS_PARALLELISM"] = "true"
21
tests/models/decoder_only/language/test_gptq_marlin_24.py
renamed
+2
-1
@@ -10,9 +10,10 @@ from dataclasses import dataclass
10
11
import pytest
12
13
-from tests.models.utils import check_logprobs_close
13
from tests.quantization.utils import is_quant_method_supported
14
15
+from ...utils import check_logprobs_close
16
+
17
18
@dataclass
19
class ModelPair:
tests/models/decoder_only/language/test_granite.py
renamed
+1
-1
@@ -6,7 +6,7 @@ import importlib.metadata
6
7
import pytest
8
9
-from .utils import check_logprobs_close
9
+from ...utils import check_logprobs_close
10
11
TRANSFORMERS_VERSION = tuple(
12
map(int,
tests/models/decoder_only/language/test_jamba.py
renamed
+2
-1
@@ -1,8 +1,9 @@
1
import pytest
2
3
-from tests.models.utils import check_outputs_equal
3
from vllm.worker.model_runner import _get_graph_batch_size
4
5
+from ...utils import check_outputs_equal
6
+
7
MODELS = ["ai21labs/Jamba-tiny-random"]
8
9
tests/models/decoder_only/language/test_marlin.py
renamed
+1
-1
@@ -16,7 +16,7 @@ import pytest
16
17
from tests.quantization.utils import is_quant_method_supported
18
19
-from .utils import check_logprobs_close
19
+from ...utils import check_logprobs_close
20
21
22
@dataclass
tests/models/decoder_only/language/test_mistral.py
renamed
+1
-1
@@ -4,7 +4,7 @@ Run `pytest tests/models/test_mistral.py`.
4
"""
5
import pytest
6
7
-from .utils import check_logprobs_close
7
+from ...utils import check_logprobs_close
8
9
MODELS = [
10
"mistralai/Mistral-7B-Instruct-v0.1",
tests/models/decoder_only/language/test_modelopt.py
renamed
tests/models/decoder_only/language/test_models.py
renamed
+1
-1
@@ -7,7 +7,7 @@ Run `pytest tests/models/test_models.py`.
7
"""
8
import pytest
9
10
-from .utils import check_outputs_equal
10
+from ...utils import check_outputs_equal
11
12
MODELS = [
13
"facebook/opt-125m",
tests/models/decoder_only/language/test_phimoe.py
renamed
+1
-1
@@ -7,7 +7,7 @@ import torch
7
8
from vllm.utils import is_cpu
9
10
-from .utils import check_logprobs_close
10
+from ...utils import check_logprobs_close
11
12
MODELS = [
13
"microsoft/Phi-3.5-MoE-instruct",
tests/models/decoder_only/vision_language/__init__.py
tests/models/decoder_only/vision_language/test_blip2.py
renamed
+3
-5
@@ -6,10 +6,8 @@ from transformers import AutoModelForVision2Seq, AutoTokenizer
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
8
9
-from ..conftest import IMAGE_ASSETS
10
-from .utils import check_logprobs_close
11
-
12
-pytestmark = pytest.mark.vlm
9
+from ....conftest import IMAGE_ASSETS
10
+from ...utils import check_logprobs_close
11
12
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13
"stop_sign":
@@ -56,7 +54,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
54
dtype: str, max_tokens: int, num_logprobs: int) -> None:
55
"""Inference result should be the same between hf and vllm.
56
59
- All the image fixtures for the test is under tests/images.
57
+ All the image fixtures for the test are from IMAGE_ASSETS.
58
For huggingface runner, we provide the PIL images as input.
59
For vllm runner, we provide MultiModalData objects and corresponding
60
MultiModalConfig as input.
tests/models/decoder_only/vision_language/test_broadcast.py
new
+42
@@ -0,0 +1,42 @@
1
+import pytest
2
+
3
+from ....utils import multi_gpu_test
4
+
5
+
6
+@multi_gpu_test(num_gpus=2)
7
+@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
8
+@pytest.mark.parametrize("model", [
9
+ "llava-hf/llava-1.5-7b-hf",
10
+ "llava-hf/llava-v1.6-mistral-7b-hf",
11
+ "facebook/chameleon-7b",
12
+])
13
+def test_models(hf_runner, vllm_runner, image_assets,
14
+ distributed_executor_backend, model) -> None:
15
+
16
+ dtype = "half"
17
+ max_tokens = 5
18
+ num_logprobs = 5
19
+ tensor_parallel_size = 2
20
+
21
+ if model.startswith("llava-hf/llava-1.5"):
22
+ from .test_llava import models, run_test
23
+ elif model.startswith("llava-hf/llava-v1.6"):
24
+ from .test_llava_next import models, run_test # type: ignore[no-redef]
25
+ elif model.startswith("facebook/chameleon"):
26
+ from .test_chameleon import models, run_test # type: ignore[no-redef]
27
+ else:
28
+ raise NotImplementedError(f"Unsupported model: {model}")
29
+
30
+ run_test(
31
+ hf_runner,
32
+ vllm_runner,
33
+ image_assets,
34
+ model=models[0],
35
+ # So that LLaVA-NeXT processor may return nested list
36
+ size_factors=[0.25, 0.5, 1.0],
37
+ dtype=dtype,
38
+ max_tokens=max_tokens,
39
+ num_logprobs=num_logprobs,
40
+ tensor_parallel_size=tensor_parallel_size,
41
+ distributed_executor_backend=distributed_executor_backend,
42
+ )
tests/models/decoder_only/vision_language/test_chameleon.py
renamed
+3
-5
@@ -6,10 +6,8 @@ from transformers import AutoModelForVision2Seq, BatchEncoding
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
8
9
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
-from .utils import check_outputs_equal
11
-
12
-pytestmark = pytest.mark.vlm
9
+from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
+from ...utils import check_outputs_equal
11
12
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13
"stop_sign":
@@ -36,7 +34,7 @@ def run_test(
34
):
35
"""Inference result should be the same between hf and vllm.
36
39
- All the image fixtures for the test is under tests/images.
37
+ All the image fixtures for the test are from IMAGE_ASSETS.
38
For huggingface runner, we provide the PIL images as input.
39
For vllm runner, we provide MultiModalDataDict objects
40
and corresponding vision language config as input.
tests/models/decoder_only/vision_language/test_fuyu.py
renamed
+3
-5
@@ -6,10 +6,8 @@ from vllm.multimodal.utils import rescale_image_size
6
from vllm.sequence import SampleLogprobs
7
from vllm.utils import is_cpu
8
9
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
-from .utils import check_logprobs_close
11
-
12
-pytestmark = pytest.mark.vlm
9
+from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
+from ...utils import check_logprobs_close
11
12
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13
"stop_sign":
@@ -46,7 +44,7 @@ def run_test(
44
):
45
"""Inference result should be the same between hf and vllm.
46
49
- All the image fixtures for the test is under tests/images.
47
+ All the image fixtures for the test are from IMAGE_ASSETS.
48
For huggingface runner, we provide the PIL images as input.
49
For vllm runner, we provide MultiModalDataDict objects
50
and corresponding MultiModalConfig as input.
tests/models/decoder_only/vision_language/test_intern_vit.py
renamed
+1
-3
@@ -6,9 +6,7 @@ import torch.nn as nn
6
from huggingface_hub import snapshot_download
7
from transformers import AutoConfig, AutoModel, CLIPImageProcessor
8
9
-from ..conftest import _ImageAssets, cleanup
10
-
11
-pytestmark = pytest.mark.vlm
9
+from ....conftest import _ImageAssets, cleanup
10
11
# we use snapshot_download to prevent conflicts between
12
# dynamic_module and trust_remote_code for hf_runner
tests/models/decoder_only/vision_language/test_internvl.py
renamed
+4
-6
@@ -9,11 +9,9 @@ from transformers import AutoConfig
9
from vllm.multimodal.utils import rescale_image_size
10
from vllm.utils import is_cpu
11
12
-from ..conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13
- _ImageAssets)
14
-from .utils import check_logprobs_close
15
-
16
-pytestmark = pytest.mark.vlm
12
+from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13
+ _ImageAssets)
14
+from ...utils import check_logprobs_close
15
16
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
17
"stop_sign":
@@ -78,7 +76,7 @@ def run_test(
76
):
77
"""Inference result should be the same between hf and vllm.
78
81
- All the image fixtures for the test is under tests/images.
79
+ All the image fixtures for the test are from IMAGE_ASSETS.
80
For huggingface runner, we provide the PIL images as input.
81
For vllm runner, we provide MultiModalDataDict objects
82
and corresponding MultiModalConfig as input.
tests/models/decoder_only/vision_language/test_llava.py
renamed
+5
-7
@@ -8,11 +8,9 @@ from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
9
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
10
11
-from ..conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12
- _ImageAssets)
13
-from .utils import check_logprobs_close
14
-
15
-pytestmark = pytest.mark.vlm
11
+from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12
+ _ImageAssets)
13
+from ...utils import check_logprobs_close
14
15
_LIMIT_IMAGE_PER_PROMPT = 4
16
@@ -143,7 +141,7 @@ def _run_test(
141
):
142
"""Inference result should be the same between hf and vllm.
143
146
- All the image fixtures for the test is under tests/images.
144
+ All the image fixtures for the test are from IMAGE_ASSETS.
145
For huggingface runner, we provide the PIL images as input.
146
For vllm runner, we provide MultiModalDataDict objects
147
and corresponding MultiModalConfig as input.
@@ -239,7 +237,7 @@ def _run_test(
237
@pytest.mark.parametrize("max_tokens", [128])
238
@pytest.mark.parametrize("num_logprobs", [5])
239
def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
242
- dtype: str, max_tokens: int, num_logprobs: int) -> None:
240
+ dtype, max_tokens, num_logprobs) -> None:
241
run_test(
242
hf_runner,
243
vllm_runner,
tests/models/decoder_only/vision_language/test_llava_image_embeds.py
renamed
+3
-5
@@ -5,10 +5,8 @@ from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5
6
from vllm.sequence import SampleLogprobs
7
8
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9
-from .utils import check_logprobs_close
10
-
11
-pytestmark = pytest.mark.vlm
8
+from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9
+from ...utils import check_logprobs_close
10
11
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
12
"stop_sign":
@@ -62,7 +60,7 @@ def run_test(
60
):
61
"""Inference result should be the same between hf and vllm.
62
65
- All the image fixtures for the test is under tests/images.
63
+ All the image fixtures for the test are from IMAGE_ASSETS.
64
For huggingface runner, we provide the PIL images as input.
65
For vllm runner, we provide MultiModalDataDict objects
66
and corresponding vision language config as input.
tests/models/decoder_only/vision_language/test_llava_next.py
renamed
+4
-6
@@ -6,11 +6,9 @@ from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
8
9
-from ..conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
10
- _ImageAssets)
11
-from .utils import check_logprobs_close
12
-
13
-pytestmark = pytest.mark.vlm
9
+from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
10
+ _ImageAssets)
11
+from ...utils import check_logprobs_close
12
13
_LIMIT_IMAGE_PER_PROMPT = 4
14
@@ -197,7 +195,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
195
dtype, max_tokens, num_logprobs) -> None:
196
"""Inference result should be the same between hf and vllm.
197
200
- All the image fixtures for the test is under tests/images.
198
+ All the image fixtures for the test are from IMAGE_ASSETS.
199
For huggingface runner, we provide the PIL images as input.
200
For vllm runner, we provide MultiModalDataDict objects
201
and corresponding MultiModalConfig as input.
tests/models/decoder_only/vision_language/test_llava_next_video.py
renamed
+2
-4
@@ -8,10 +8,8 @@ from vllm.multimodal.utils import (rescale_video_size, resize_video,
8
sample_frames_from_video)
9
from vllm.sequence import SampleLogprobs
10
11
-from ..conftest import VIDEO_ASSETS, HfRunner, VllmRunner, _VideoAssets
12
-from .utils import check_logprobs_close
13
-
14
-pytestmark = pytest.mark.vlm
11
+from ....conftest import VIDEO_ASSETS, HfRunner, VllmRunner, _VideoAssets
12
+from ...utils import check_logprobs_close
13
14
_PREFACE = (
15
"A chat between a curious human and an artificial intelligence assistant. "
tests/models/decoder_only/vision_language/test_minicpmv.py
renamed
+3
-5
@@ -9,10 +9,8 @@ from transformers import BatchEncoding
9
from vllm.multimodal.utils import rescale_image_size
10
from vllm.sequence import SampleLogprobs
11
12
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner
13
-from .utils import check_logprobs_close
14
-
15
-pytestmark = pytest.mark.vlm
12
+from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner
13
+from ...utils import check_logprobs_close
14
15
# The image token is placed before "user" on purpose so that the test can pass
16
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
@@ -65,7 +63,7 @@ def run_test(
63
):
64
"""Inference result should be the same between hf and vllm.
65
68
- All the image fixtures for the test is under tests/images.
66
+ All the image fixtures for the test are from IMAGE_ASSETS.
67
For huggingface runner, we provide the PIL images as input.
68
For vllm runner, we provide MultiModalDataDict objects
69
and corresponding MultiModalConfig as input.
tests/models/decoder_only/vision_language/test_paligemma.py
renamed
+3
-5
@@ -8,10 +8,8 @@ from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
9
from vllm.utils import is_hip
10
11
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
12
-from .utils import check_logprobs_close
13
-
14
-pytestmark = pytest.mark.vlm
11
+from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
12
+from ...utils import check_logprobs_close
13
14
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
15
"stop_sign":
@@ -69,7 +67,7 @@ def run_test(
67
):
68
"""Inference result should be the same between hf and vllm.
69
72
- All the image fixtures for the test is under tests/images.
70
+ All the image fixtures for the test are from IMAGE_ASSETS.
71
For huggingface runner, we provide the PIL images as input.
72
For vllm runner, we provide MultiModalDataDict objects
73
and corresponding MultiModalConfig as input.
tests/models/decoder_only/vision_language/test_phi3v.py
renamed
+3
-5
@@ -9,10 +9,8 @@ from vllm.multimodal.utils import rescale_image_size
9
from vllm.sequence import SampleLogprobs
10
from vllm.utils import is_cpu, is_hip
11
12
-from ..conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
13
-from .utils import check_logprobs_close
14
-
15
-pytestmark = pytest.mark.vlm
12
+from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
13
+from ...utils import check_logprobs_close
14
15
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
16
"stop_sign":
@@ -71,7 +69,7 @@ def run_test(
69
):
70
"""Inference result should be the same between hf and vllm.
71
74
- All the image fixtures for the test is under tests/images.
72
+ All the image fixtures for the test are from IMAGE_ASSETS.
73
For huggingface runner, we provide the PIL images as input.
74
For vllm runner, we provide MultiModalDataDict objects
75
and corresponding MultiModalConfig as input.
tests/models/decoder_only/vision_language/test_pixtral.py
renamed
+16
-7
@@ -5,7 +5,7 @@ Run `pytest tests/models/test_mistral.py`.
5
import json
6
import uuid
7
from dataclasses import asdict
8
-from typing import Any, Dict, List, Optional, Tuple
8
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
9
10
import pytest
11
from mistral_common.protocol.instruct.messages import ImageURLChunk
@@ -17,9 +17,11 @@ from vllm import EngineArgs, LLMEngine, SamplingParams, TokensPrompt
17
from vllm.multimodal import MultiModalDataBuiltins
18
from vllm.sequence import Logprob, SampleLogprobs
19
20
-from .utils import check_logprobs_close
20
+from ....utils import VLLM_PATH
21
+from ...utils import check_logprobs_close
22
22
-pytestmark = pytest.mark.vlm
23
+if TYPE_CHECKING:
24
+ from _typeshed import StrPath
25
26
MODELS = ["mistralai/Pixtral-12B-2409"]
27
IMG_URLS = [
@@ -83,14 +85,21 @@ SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
85
LIMIT_MM_PER_PROMPT = dict(image=4)
86
87
MAX_MODEL_LEN = [8192, 65536]
86
-FIXTURE_LOGPROBS_CHAT = "tests/models/fixtures/pixtral_chat.json"
87
-FIXTURE_LOGPROBS_ENGINE = "tests/models/fixtures/pixtral_chat_engine.json"
88
+
89
+FIXTURES_PATH = VLLM_PATH / "tests/models/fixtures"
90
+assert FIXTURES_PATH.exists()
91
+
92
+FIXTURE_LOGPROBS_CHAT = FIXTURES_PATH / "pixtral_chat.json"
93
+FIXTURE_LOGPROBS_ENGINE = FIXTURES_PATH / "pixtral_chat_engine.json"
94
95
OutputsLogprobs = List[Tuple[List[int], str, Optional[SampleLogprobs]]]
96
97
98
# For the test author to store golden output in JSON
93
-def _dump_outputs_w_logprobs(outputs: OutputsLogprobs, filename: str) -> None:
99
+def _dump_outputs_w_logprobs(
100
+ outputs: OutputsLogprobs,
101
+ filename: "StrPath",
102
+) -> None:
103
json_data = [(tokens, text,
104
[{k: asdict(v)
105
for k, v in token_logprobs.items()}
@@ -101,7 +110,7 @@ def _dump_outputs_w_logprobs(outputs: OutputsLogprobs, filename: str) -> None:
110
json.dump(json_data, f)
111
112
104
-def load_outputs_w_logprobs(filename: str) -> OutputsLogprobs:
113
+def load_outputs_w_logprobs(filename: "StrPath") -> OutputsLogprobs:
114
with open(filename, "rb") as f:
115
json_data = json.load(f)
116
tests/models/decoder_only/vision_language/test_qwen.py
renamed
+3
-5
@@ -10,11 +10,9 @@ from vllm.inputs import InputContext, LLMInputs
10
from vllm.multimodal.base import MultiModalInputs
11
from vllm.multimodal.utils import cached_get_tokenizer, rescale_image_size
12
13
-from ..conftest import (IMAGE_ASSETS, HfRunner, ImageAsset, PromptImageInput,
14
- VllmRunner, _ImageAssets)
15
-from .utils import check_logprobs_close
16
-
17
-pytestmark = pytest.mark.vlm
13
+from ....conftest import (IMAGE_ASSETS, HfRunner, ImageAsset, PromptImageInput,
14
+ VllmRunner, _ImageAssets)
15
+from ...utils import check_logprobs_close
16
17
text_only_models = [
18
"Qwen/Qwen-7B-Chat" # Has no visual component
tests/models/embedding/__init__.py
tests/models/embedding/language/__init__.py
tests/models/embedding/language/test_embedding.py
renamed
tests/models/encoder_decoder/__init__.py
tests/models/encoder_decoder/language/__init__.py
tests/models/encoder_decoder/language/test_bart.py
renamed
+84
-31
@@ -1,8 +1,8 @@
1
"""Compare the outputs of HF and vLLM for BART models using greedy sampling.
2
3
-Run `pytest tests/models/test_bart.py`.
3
+Run `pytest tests/models/encoder_decoder/language/test_bart.py`.
4
"""
5
-from typing import List, Optional, Tuple
5
+from typing import List, Optional, Tuple, Type
6
7
from vllm.utils import is_cpu
8
@@ -16,8 +16,10 @@ if not is_cpu():
16
17
from vllm.sequence import SampleLogprobs
18
19
- from ..conftest import DecoderPromptType
20
- from .utils import check_logprobs_close
19
+ from ....conftest import (DecoderPromptType, ExplicitEncoderDecoderPrompt,
20
+ HfRunner, VllmRunner)
21
+ from ....utils import multi_gpu_test
22
+ from ...utils import check_logprobs_close
23
24
MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
25
@@ -34,20 +36,18 @@ if not is_cpu():
36
37
return output_ids, hf_output_str, out_logprobs
38
37
- @pytest.mark.parametrize("model", MODELS)
38
- @pytest.mark.parametrize("dtype", ["float", "bfloat16"])
39
- @pytest.mark.parametrize("max_tokens", [64])
40
- @pytest.mark.parametrize("num_logprobs", [5])
41
- @pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
42
- def test_models(
43
- hf_runner,
44
- vllm_runner,
45
- example_encoder_decoder_prompts,
39
+ def run_test(
40
+ hf_runner: Type[HfRunner],
41
+ vllm_runner: Type[VllmRunner],
42
+ prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
43
+ decoder_prompt_type: DecoderPromptType,
44
model: str,
45
+ *,
46
dtype: str,
47
max_tokens: int,
48
num_logprobs: int,
50
- decoder_prompt_type: DecoderPromptType,
49
+ tensor_parallel_size: int,
50
+ distributed_executor_backend: Optional[str] = None,
51
) -> None:
52
'''
53
Test the vLLM BART model for a variety of encoder/decoder input prompts,
@@ -116,8 +116,29 @@ if not is_cpu():
116
token during the process of validating the vLLM decoded output.
117
'''
118
119
- test_case_prompts = example_encoder_decoder_prompts[
120
- decoder_prompt_type]
119
+ # NOTE: take care of the order. run vLLM first, and then run HF.
120
+ # vLLM needs a fresh new process without cuda initialization.
121
+ # if we run HF first, the cuda initialization will be done and it
122
+ # will hurt multiprocessing backend with fork method (the default).
123
+
124
+ # Note: currently encoder/decoder models are only compatible with
125
+ # enforce_eager=True. Normally this is not a problem because
126
+ # for encoder/decoder models vLLM will
127
+ # default to enforce_eager=True if enforce_eager
128
+ # is left unspecified. However, the
129
+ # VllmRunner test fixture (which wraps around the LLM class) defaults to
130
+ # enforce_eager=False (a behavior which a number of already-exisitng
131
+ # decoder-only unit tests expect), so when testing an encoder/decoder
132
+ # model we must explicitly specify enforce_eager=True in the VllmRunner
133
+ # constructor.
134
+ with vllm_runner(
135
+ model,
136
+ dtype=dtype,
137
+ tensor_parallel_size=tensor_parallel_size,
138
+ distributed_executor_backend=distributed_executor_backend,
139
+ enforce_eager=True) as vllm_model:
140
+ vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
141
+ prompts, max_tokens, num_logprobs)
142
143
# Configuration settings for HF baseline
144
hf_kwargs = {
@@ -135,26 +156,12 @@ if not is_cpu():
156
auto_cls=AutoModelForSeq2SeqLM) as hf_model:
157
hf_outputs = (
158
hf_model.generate_encoder_decoder_greedy_logprobs_limit(
138
- test_case_prompts,
159
+ prompts,
160
max_tokens,
161
num_logprobs,
162
**hf_kwargs,
163
))
164
144
- # Note: currently encoder/decoder models are only compatible with
145
- # enforce_eager=True. Normally this is not a problem because
146
- # for encoder/decoder models vLLM will
147
- # default to enforce_eager=True if enforce_eager
148
- # is left unspecified. However, the
149
- # VllmRunner test fixture (which wraps around the LLM class) defaults to
150
- # enforce_eager=False (a behavior which a number of already-exisitng
151
- # decoder-only unit tests expect), so when testing an encoder/decoder
152
- # model we must explicitly specify enforce_eager=True in the VllmRunner
153
- # constructor.
154
- with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
155
- vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
156
- test_case_prompts, max_tokens, num_logprobs)
157
-
165
hf_skip_tokens = (1 if decoder_prompt_type == DecoderPromptType.NONE
166
else 0)
167
@@ -168,3 +175,49 @@ if not is_cpu():
175
name_1="vllm",
176
num_outputs_0_skip_tokens=hf_skip_tokens,
177
)
178
+
179
+ @pytest.mark.parametrize("model", MODELS)
180
+ @pytest.mark.parametrize("dtype", ["float", "bfloat16"])
181
+ @pytest.mark.parametrize("max_tokens", [64])
182
+ @pytest.mark.parametrize("num_logprobs", [5])
183
+ @pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
184
+ def test_models(hf_runner, vllm_runner, example_encoder_decoder_prompts,
185
+ model, dtype, max_tokens, num_logprobs,
186
+ decoder_prompt_type) -> None:
187
+
188
+ run_test(
189
+ hf_runner,
190
+ vllm_runner,
191
+ example_encoder_decoder_prompts[decoder_prompt_type],
192
+ decoder_prompt_type,
193
+ model,
194
+ dtype=dtype,
195
+ max_tokens=max_tokens,
196
+ num_logprobs=num_logprobs,
197
+ tensor_parallel_size=1,
198
+ )
199
+
200
+ @multi_gpu_test(num_gpus=2)
201
+ @pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
202
+ @pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
203
+ @pytest.mark.parametrize("dtype", ["float"])
204
+ @pytest.mark.parametrize("max_tokens", [64])
205
+ @pytest.mark.parametrize("num_logprobs", [5])
206
+ @pytest.mark.parametrize("decoder_prompt_type", [DecoderPromptType.CUSTOM])
207
+ def test_models_distributed(hf_runner, vllm_runner,
208
+ example_encoder_decoder_prompts,
209
+ distributed_executor_backend, model, dtype,
210
+ max_tokens, num_logprobs,
211
+ decoder_prompt_type) -> None:
212
+ run_test(
213
+ hf_runner,
214
+ vllm_runner,
215
+ example_encoder_decoder_prompts[decoder_prompt_type],
216
+ decoder_prompt_type,
217
+ model,
218
+ dtype=dtype,
219
+ max_tokens=max_tokens,
220
+ num_logprobs=num_logprobs,
221
+ tensor_parallel_size=2,
222
+ distributed_executor_backend=distributed_executor_backend,
223
+ )
tests/utils.py
+19
-1
@@ -10,6 +10,7 @@ from pathlib import Path
10
from typing import Any, Callable, Dict, List, Optional
11
12
import openai
13
+import pytest
14
import requests
15
from openai.types.completion import Completion
16
from transformers import AutoTokenizer
@@ -22,7 +23,8 @@ from vllm.engine.arg_utils import AsyncEngineArgs
23
from vllm.entrypoints.openai.cli_args import make_arg_parser
24
from vllm.model_executor.model_loader.loader import get_model_loader
25
from vllm.platforms import current_platform
25
-from vllm.utils import FlexibleArgumentParser, get_open_port, is_hip
26
+from vllm.utils import (FlexibleArgumentParser, cuda_device_count_stateless,
27
+ get_open_port, is_hip)
28
29
if current_platform.is_rocm():
30
from amdsmi import (amdsmi_get_gpu_vram_usage,
@@ -452,6 +454,22 @@ def fork_new_process_for_each_test(
454
return wrapper
455
456
457
+def multi_gpu_test(*, num_gpus: int):
458
+ """
459
+ Decorate a test to be run only when multiple GPUs are available.
460
+ """
461
+ test_selector = getattr(pytest.mark, f"distributed_{num_gpus}_gpus")
462
+ test_skipif = pytest.mark.skipif(
463
+ cuda_device_count_stateless() < num_gpus,
464
+ reason=f"Need at least {num_gpus} GPUs to run the test.",
465
+ )
466
+
467
+ def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
468
+ return test_selector(test_skipif(fork_new_process_for_each_test(f)))
469
+
470
+ return wrapper
471
+
472
+
473
async def completions_with_server_args(
474
prompts: List[str],
475
model_name: str,