A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[CI/Build] Fix VLM test failures when using transformers v4.46 (#9666)
Cyrus Leung committed
Oct 25, 2024 at 01:40 UTC
c866e0079de05cf6aee5931f3b9e200e8cbcf26c
4 files changed
+28
-12
tests/conftest.py
+9
-7
@@ -232,20 +232,22 @@ def video_assets() -> _VideoAssets:
232
return VIDEO_ASSETS
233
234
235
-_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature)
235
+_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
236
237
238
class HfRunner:
239
240
- def wrap_device(self, input: _T, device: Optional[str] = None) -> _T:
240
+ def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
241
if device is None:
242
- return self.wrap_device(
243
- input, "cpu" if current_platform.is_cpu() else "cuda")
242
+ device = "cpu" if current_platform.is_cpu() else "cuda"
243
245
- if hasattr(input, "device") and input.device.type == device:
246
- return input
244
+ if isinstance(x, dict):
245
+ return {k: self.wrap_device(v, device) for k, v in x.items()}
246
248
- return input.to(device)
247
+ if hasattr(x, "device") and x.device.type == device:
248
+ return x
249
+
250
+ return x.to(device)
251
252
def __init__(
253
self,
tests/models/decoder_only/vision_language/test_chameleon.py
+5
@@ -1,6 +1,7 @@
1
from typing import List, Optional, Type
2
3
import pytest
4
+import transformers
5
from transformers import AutoModelForVision2Seq, BatchEncoding
6
7
from vllm.multimodal.utils import rescale_image_size
@@ -93,6 +94,10 @@ def run_test(
94
)
95
96
97
+@pytest.mark.skipif(
98
+ transformers.__version__.startswith("4.46.0"),
99
+ reason="Model broken in HF, see huggingface/transformers#34379",
100
+)
101
@pytest.mark.parametrize("model", models)
102
@pytest.mark.parametrize(
103
"size_factors",
tests/models/decoder_only/vision_language/test_minicpmv.py
+2
-2
@@ -32,8 +32,8 @@ HF_MULTIIMAGE_IMAGE_PROMPT = \
32
models = ["openbmb/MiniCPM-Llama3-V-2_5"]
33
34
35
-def _wrap_inputs(hf_inputs: BatchEncoding) -> BatchEncoding:
36
- return BatchEncoding({"model_inputs": hf_inputs})
35
+def _wrap_inputs(hf_inputs: BatchEncoding):
36
+ return {"model_inputs": hf_inputs}
37
38
39
def trunc_hf_output(hf_output: Tuple[List[int], str,
tests/models/decoder_only/vision_language/test_paligemma.py
+12
-3
@@ -2,11 +2,12 @@ import os
2
from typing import List, Optional, Tuple, Type
3
4
import pytest
5
-from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5
+from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
6
+ BatchEncoding)
7
8
from vllm.multimodal.utils import rescale_image_size
9
from vllm.sequence import SampleLogprobs
9
-from vllm.utils import is_hip
10
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, is_hip
11
12
from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13
from ...utils import check_logprobs_close
@@ -74,6 +75,7 @@ def run_test(
75
Note, the text input is also adjusted to abide by vllm contract.
76
The text output is sanitized to be able to compare with hf.
77
"""
78
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
79
images = [asset.pil_image for asset in image_assets]
80
81
inputs_per_image = [(
@@ -100,7 +102,14 @@ def run_test(
102
for prompts, images in inputs_per_image
103
]
104
103
- with hf_runner(model, dtype=dtype,
105
+ def process(hf_inputs: BatchEncoding):
106
+ hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
107
+ .to(torch_dtype) # type: ignore
108
+ return hf_inputs
109
+
110
+ with hf_runner(model,
111
+ dtype=dtype,
112
+ postprocess_inputs=process,
113
auto_cls=AutoModelForVision2Seq) as hf_model:
114
hf_outputs_per_image = [
115
hf_model.generate_greedy_logprobs_limit(prompts,