@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c866e0079d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[CI/Build] Fix VLM test failures when using transformers v4.46 (#9666)

Cyrus Leung committed Oct 25, 2024 at 01:40 UTC c866e0079de05cf6aee5931f3b9e200e8cbcf26c
4 files changed +28 -12
tests/conftest.py
+9 -7
@@ -232,20 +232,22 @@ def video_assets() -> _VideoAssets:
232 return VIDEO_ASSETS
233
234
235 -_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature)
235 +_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
236
237
238 class HfRunner:
239
240 - def wrap_device(self, input: _T, device: Optional[str] = None) -> _T:
240 + def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
241 if device is None:
242 - return self.wrap_device(
243 - input, "cpu" if current_platform.is_cpu() else "cuda")
242 + device = "cpu" if current_platform.is_cpu() else "cuda"
243
245 - if hasattr(input, "device") and input.device.type == device:
246 - return input
244 + if isinstance(x, dict):
245 + return {k: self.wrap_device(v, device) for k, v in x.items()}
246
248 - return input.to(device)
247 + if hasattr(x, "device") and x.device.type == device:
248 + return x
249 +
250 + return x.to(device)
251
252 def __init__(
253 self,
tests/models/decoder_only/vision_language/test_chameleon.py
+5
@@ -1,6 +1,7 @@
1 from typing import List, Optional, Type
2
3 import pytest
4 +import transformers
5 from transformers import AutoModelForVision2Seq, BatchEncoding
6
7 from vllm.multimodal.utils import rescale_image_size
@@ -93,6 +94,10 @@ def run_test(
94 )
95
96
97 +@pytest.mark.skipif(
98 + transformers.__version__.startswith("4.46.0"),
99 + reason="Model broken in HF, see huggingface/transformers#34379",
100 +)
101 @pytest.mark.parametrize("model", models)
102 @pytest.mark.parametrize(
103 "size_factors",
tests/models/decoder_only/vision_language/test_minicpmv.py
+2 -2
@@ -32,8 +32,8 @@ HF_MULTIIMAGE_IMAGE_PROMPT = \
32 models = ["openbmb/MiniCPM-Llama3-V-2_5"]
33
34
35 -def _wrap_inputs(hf_inputs: BatchEncoding) -> BatchEncoding:
36 - return BatchEncoding({"model_inputs": hf_inputs})
35 +def _wrap_inputs(hf_inputs: BatchEncoding):
36 + return {"model_inputs": hf_inputs}
37
38
39 def trunc_hf_output(hf_output: Tuple[List[int], str,
tests/models/decoder_only/vision_language/test_paligemma.py
+12 -3
@@ -2,11 +2,12 @@ import os
2 from typing import List, Optional, Tuple, Type
3
4 import pytest
5 -from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5 +from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
6 + BatchEncoding)
7
8 from vllm.multimodal.utils import rescale_image_size
9 from vllm.sequence import SampleLogprobs
9 -from vllm.utils import is_hip
10 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, is_hip
11
12 from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13 from ...utils import check_logprobs_close
@@ -74,6 +75,7 @@ def run_test(
75 Note, the text input is also adjusted to abide by vllm contract.
76 The text output is sanitized to be able to compare with hf.
77 """
78 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
79 images = [asset.pil_image for asset in image_assets]
80
81 inputs_per_image = [(
@@ -100,7 +102,14 @@ def run_test(
102 for prompts, images in inputs_per_image
103 ]
104
103 - with hf_runner(model, dtype=dtype,
105 + def process(hf_inputs: BatchEncoding):
106 + hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
107 + .to(torch_dtype) # type: ignore
108 + return hf_inputs
109 +
110 + with hf_runner(model,
111 + dtype=dtype,
112 + postprocess_inputs=process,
113 auto_cls=AutoModelForVision2Seq) as hf_model:
114 hf_outputs_per_image = [
115 hf_model.generate_greedy_logprobs_limit(prompts,