[CI] And PPL test for multimodal generation models (#50839)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Aug 3, 2026 at 19:25 UTC
9acb7b3699b2c9e76db380983850c7e4c12941ab
4 files changed
+198
.buildkite/test_areas/models_multimodal.yaml
+17
@@ -145,6 +145,23 @@ steps:
145
depends_on:
146
- image-build-amd
147
148
+- label: Multi-Modal Models (PPL)
149
+ device: h200_35gb
150
+ key: multi-modal-models-extended-ppl
151
+ optional: true
152
+ source_file_dependencies:
153
+ - vllm/
154
+ - tests/models/multimodal/generation_ppl_test/
155
+ commands:
156
+ - pytest -v -s models/multimodal/generation_ppl_test/
157
+ mirror:
158
+ amd:
159
+ dind: false
160
+ device: mi300_1
161
+ timeout_in_minutes: 90
162
+ depends_on:
163
+ - image-build-amd
164
+
165
- label: Multi-Modal Models (Extended Generation 2)
166
device: h200_35gb
167
key: multi-modal-models-extended-generation-2
tests/models/multimodal/generation_ppl_test/__init__.py
tests/models/multimodal/generation_ppl_test/ppl_utils.py
new
+155
@@ -0,0 +1,155 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import cast
4
+
5
+import torch
6
+from datasets import load_dataset
7
+from transformers import AutoModelForImageTextToText, AutoProcessor
8
+
9
+import tests.ci_envs as ci_envs
10
+from tests.models.utils import (
11
+ GenerateModelInfo,
12
+ TokensTextLogprobsPromptLogprobs,
13
+ get_vllm_extra_kwargs,
14
+)
15
+from vllm.logprobs import Logprob
16
+
17
+PPL_TOL = 0.01
18
+
19
+
20
+@torch.inference_mode
21
+def vqa_ppl_test(
22
+ hf_runner,
23
+ vllm_runner,
24
+ model_info: GenerateModelInfo,
25
+ mm_processor_kwargs=None,
26
+ vllm_extra_kwargs=None,
27
+ tol=PPL_TOL,
28
+):
29
+ dataset = load_dataset("lmms-lab-encoder/llava-bench-in-the-wild", split="train")
30
+
31
+ vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
32
+
33
+ if mm_processor_kwargs is None:
34
+ mm_processor_kwargs = {}
35
+
36
+ processor = AutoProcessor.from_pretrained(model_info.name)
37
+
38
+ images = []
39
+ prompts = []
40
+ for row in dataset:
41
+ image = row["image"]
42
+ images.append(image)
43
+
44
+ question = row["question"]
45
+ answer = row["gpt_answer"]
46
+
47
+ conversation = [
48
+ {
49
+ "role": "user",
50
+ "content": [
51
+ {
52
+ "type": "image",
53
+ },
54
+ {"type": "text", "text": question},
55
+ ],
56
+ },
57
+ {"role": "assistant", "content": answer},
58
+ ]
59
+
60
+ prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
61
+ prompts.append(prompt)
62
+
63
+ with vllm_runner(
64
+ model_info.name,
65
+ gpu_memory_utilization=0.7,
66
+ max_num_seqs=1,
67
+ max_model_len=4096,
68
+ mm_processor_kwargs=mm_processor_kwargs,
69
+ **vllm_extra_kwargs,
70
+ ) as vllm_model:
71
+ # Use max_num_seqs=1 to avoid OOM,
72
+ # and avoid batch different requests together.
73
+
74
+ model_config = vllm_model.llm.llm_engine.model_config
75
+
76
+ # Confirm whether vllm is using the correct architecture
77
+ if model_info.architecture:
78
+ assert model_info.architecture in model_config.architectures
79
+
80
+ nll_sum = torch.tensor(0.0, dtype=torch.float32, device="cpu")
81
+ n_tokens = 0
82
+
83
+ outputs = vllm_model.generate_greedy_logprobs(
84
+ prompts=prompts,
85
+ images=images,
86
+ max_tokens=1,
87
+ num_logprobs=None,
88
+ num_prompt_logprobs=0,
89
+ use_tqdm=False,
90
+ )
91
+
92
+ for output in outputs:
93
+ output = cast(TokensTextLogprobsPromptLogprobs, output)
94
+ token_datas = cast(list[dict[int, Logprob] | None], output[3])
95
+
96
+ assert token_datas[0] is None
97
+ token_log_probs = []
98
+ for token_data in token_datas[1:]:
99
+ assert token_data is not None
100
+ assert len(token_data) == 1
101
+ token_log_prob = list(token_data.values())[0].logprob
102
+ token_log_probs.append(token_log_prob)
103
+
104
+ neg_log_likelihood = -torch.tensor(
105
+ token_log_probs, dtype=torch.float32, device="cpu"
106
+ ).sum()
107
+ nll_sum += neg_log_likelihood
108
+ n_tokens += len(token_log_probs)
109
+ vllm_ppl = float(torch.exp(nll_sum / n_tokens))
110
+ vllm_dtype = model_config.dtype
111
+ head_dtype = model_config.head_dtype
112
+
113
+ # Accelerate ppl test by setting Transformers ppl score to a constant
114
+ if model_info.hf_ppl is None:
115
+ with hf_runner(
116
+ model_info.name,
117
+ dtype=ci_envs.VLLM_CI_HF_DTYPE or model_info.hf_dtype,
118
+ auto_cls=AutoModelForImageTextToText,
119
+ ) as hf_model:
120
+ nll_sum = torch.tensor(0.0, dtype=torch.float32, device="cpu")
121
+ n_tokens = 0
122
+
123
+ for prompt, image in zip(prompts, images):
124
+ inputs = processor(
125
+ text=[prompt],
126
+ images=[image],
127
+ padding=True,
128
+ return_tensors="pt",
129
+ **mm_processor_kwargs,
130
+ )
131
+ inputs = inputs.to("cuda")
132
+ input_ids = inputs["input_ids"]
133
+
134
+ outputs = hf_model.model(**inputs, labels=input_ids)
135
+ neg_log_likelihood = outputs.loss
136
+ neg_log_likelihood = neg_log_likelihood.to(torch.float32).cpu()
137
+
138
+ num_loss_tokens = input_ids.shape[1] - 1
139
+ nll_sum += neg_log_likelihood * num_loss_tokens
140
+ n_tokens += num_loss_tokens
141
+
142
+ hf_ppl = float(torch.exp(nll_sum / n_tokens))
143
+ hf_dtype = next(hf_model.model.parameters()).dtype
144
+ else:
145
+ hf_ppl = model_info.hf_ppl
146
+ hf_dtype = "Constant"
147
+
148
+ differ = (vllm_ppl - hf_ppl) / hf_ppl
149
+ print("Model:", model_info.name)
150
+ print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_ppl)
151
+ print("Transformers:", hf_dtype, hf_ppl)
152
+ print("Difference (%):", differ * 100)
153
+
154
+ assert abs(differ) < tol
155
+ return vllm_ppl
tests/models/multimodal/generation_ppl_test/test_qwen.py
new
+26
@@ -0,0 +1,26 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+
6
+from tests.models.utils import GenerateModelInfo
7
+
8
+from .ppl_utils import vqa_ppl_test
9
+
10
+MODELS = [
11
+ GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct"),
12
+ GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct"),
13
+]
14
+
15
+
16
+mm_processor_kwargs = {
17
+ "min_pixels": 28 * 28,
18
+ "max_pixels": 1280 * 28 * 28,
19
+}
20
+
21
+
22
+@pytest.mark.parametrize("model_info", MODELS)
23
+def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
24
+ vqa_ppl_test(
25
+ hf_runner, vllm_runner, model_info, mm_processor_kwargs=mm_processor_kwargs
26
+ )