A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[VLM] Update compatibility with transformers 4.49
Cyrus Leung committed
Feb 6, 2025 at 11:09 UTC
75404d041be0d6e656b59cbbea23520d47d37b66
9 files changed
+50
-59
docs/source/models/supported_models.md
+1
-2
@@ -883,8 +883,7 @@ For more details, please see: <gh-pr:4087#issuecomment-2250397630>
883
:::
884
885
:::{note}
886
-The chat template for Pixtral-HF is incorrect (see [discussion](https://huggingface.co/mistral-community/pixtral-12b/discussions/22)).
887
-A corrected version is available at <gh-file:examples/template_pixtral_hf.jinja>.
886
+`mistral-community/pixtral-12b` does not support V1 yet.
887
:::
888
889
:::{note}
examples/template_pixtral_hf.jinja
deleted
-38
@@ -1,38 +0,0 @@
1
-{%- if messages[0]["role"] == "system" %}
2
- {%- set system_message = messages[0]["content"] %}
3
- {%- set loop_messages = messages[1:] %}
4
-{%- else %}
5
- {%- set loop_messages = messages %}
6
-{%- endif %}
7
-
8
-{{- bos_token }}
9
-{%- for message in loop_messages %}
10
- {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
11
- {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}
12
- {%- endif %}
13
- {%- if message["role"] == "user" %}
14
- {%- if loop.last and system_message is defined %}
15
- {{- "[INST]" + system_message + "\n" }}
16
- {%- else %}
17
- {{- "[INST]" }}
18
- {%- endif %}
19
- {%- if message["content"] is not string %}
20
- {%- for chunk in message["content"] %}
21
- {%- if chunk["type"] == "text" %}
22
- {{- chunk["text"] }}
23
- {%- elif chunk["type"] == "image" %}
24
- {{- "[IMG]" }}
25
- {%- else %}
26
- {{- raise_exception("Unrecognized content type!") }}
27
- {%- endif %}
28
- {%- endfor %}
29
- {%- else %}
30
- {{- message["content"] }}
31
- {%- endif %}
32
- {{- "[/INST]" }}
33
- {%- elif message["role"] == "assistant" %}
34
- {{- message["content"] + eos_token}}
35
- {%- else %}
36
- {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
37
- {%- endif %}
38
-{%- endfor %}
tests/entrypoints/test_chat_utils.py
-1
@@ -761,7 +761,6 @@ def test_resolve_content_format_hf_defined(model, expected_format):
761
("template_falcon.jinja", "string"),
762
("template_inkbot.jinja", "string"),
763
("template_llava.jinja", "string"),
764
- ("template_pixtral_hf.jinja", "openai"),
764
("template_vlm2vec.jinja", "openai"),
765
("tool_chat_template_granite_20b_fc.jinja", "string"),
766
("tool_chat_template_hermes.jinja", "string"),
tests/models/decoder_only/vision_language/test_models.py
+2
-2
@@ -224,7 +224,7 @@ VLM_TEST_SETTINGS = {
224
marks=[
225
pytest.mark.skipif(
226
Version(TRANSFORMERS_VERSION) >= Version("4.48"),
227
- reason="HF model is not compatible with transformers>=4.48.0",
227
+ reason="HF model is not compatible with transformers>=4.48",
228
)
229
],
230
),
@@ -359,7 +359,7 @@ VLM_TEST_SETTINGS = {
359
marks=[
360
pytest.mark.skipif(
361
Version(TRANSFORMERS_VERSION) >= Version("4.48"),
362
- reason="HF model is not compatible with transformers>=4.48.0",
362
+ reason="HF model is not compatible with transformers>=4.48",
363
)
364
],
365
),
tests/models/embedding/vision_language/test_llava_next.py
+4
-3
@@ -4,7 +4,6 @@ from typing import List, Type
4
5
import pytest
6
import torch.nn.functional as F
7
-import transformers
7
from transformers import AutoModelForVision2Seq
8
9
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
@@ -57,6 +56,10 @@ def _run_test(
56
57
with hf_runner(model, dtype=dtype,
58
auto_cls=AutoModelForVision2Seq) as hf_model:
59
+ # Patch the issue where generation_config.json is missing
60
+ hf_model.processor.patch_size = \
61
+ hf_model.model.config.vision_config.patch_size
62
+
63
# Patch the issue where image_token_id
64
# exceeds the maximum allowed vocab size
65
hf_model.model.resize_token_embeddings(
@@ -88,8 +91,6 @@ def _run_test(
91
)
92
93
91
-@pytest.mark.skipif(transformers.__version__ >= "4.46",
92
- reason="Model broken with changes in transformers 4.46")
94
@pytest.mark.core_model
95
@pytest.mark.parametrize("model", MODELS)
96
@pytest.mark.parametrize("dtype", ["half"])
vllm/model_executor/models/llava.py
+23
-10
@@ -293,16 +293,29 @@ class PixtralHFMultiModalProcessor(
293
294
pixel_values = processed_outputs.get("pixel_values")
295
if pixel_values is not None:
296
- images = mm_data["images"]
297
- assert isinstance(images, list)
298
-
299
- # Original output: (1, num_images, C, H, W)
300
- # New output: (num_images, C, H, W)
301
- assert (isinstance(pixel_values, list) and len(pixel_values) == 1)
302
- assert (isinstance(pixel_values[0], list)
303
- and len(pixel_values[0]) == len(images))
304
-
305
- processed_outputs["pixel_values"] = pixel_values[0]
296
+ # Before/after https://github.com/huggingface/transformers/pull/35122
297
+ if Version(TRANSFORMERS_VERSION) <= Version("4.48.2"):
298
+ images = mm_data["images"]
299
+ assert isinstance(images, list)
300
+
301
+ # Original output: (1, num_images, C, H, W)
302
+ # New output: (num_images, C, H, W)
303
+ assert (isinstance(pixel_values, list)
304
+ and len(pixel_values) == 1)
305
+ assert (isinstance(pixel_values[0], list)
306
+ and len(pixel_values[0]) == len(images))
307
+
308
+ processed_outputs["pixel_values"] = pixel_values[0]
309
+ else:
310
+ # Avoid padding since we need the output for each image to be
311
+ # independent of other images for the cache to work correctly
312
+ image_sizes = processed_outputs["image_sizes"]
313
+ assert len(pixel_values) == len(image_sizes)
314
+
315
+ processed_outputs["pixel_values"] = [
316
+ p[:, :h, :w]
317
+ for p, (h, w) in zip(pixel_values, image_sizes)
318
+ ]
319
320
return processed_outputs
321
vllm/model_executor/models/llava_next.py
+9
-1
@@ -73,7 +73,15 @@ class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
73
return self.ctx.get_hf_config(LlavaNextConfig)
74
75
def get_hf_processor(self):
76
- return self.ctx.get_hf_processor(LlavaNextProcessor)
76
+ hf_processor = self.ctx.get_hf_processor(LlavaNextProcessor)
77
+
78
+ # In case patch_size is omitted from `processor_config.json`
79
+ # e.g. for E5-V: https://huggingface.co/royokong/e5-v
80
+ if hf_processor.patch_size is None:
81
+ patch_size = self.get_vision_encoder_info().get_patch_size()
82
+ hf_processor.patch_size = patch_size
83
+
84
+ return hf_processor
85
86
# Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L113
87
def get_num_image_tokens(
vllm/model_executor/models/minicpmv.py
+9
@@ -342,6 +342,15 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
342
**kwargs: object,
343
):
344
hf_processor = self.ctx.get_hf_processor()
345
+
346
+ # NumPy arrays are considered as Iterable but not Sequence in
347
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
348
+ image_processor = hf_processor.image_processor # type: ignore
349
+ for attr in ("mean", "std"):
350
+ val = getattr(image_processor, attr)
351
+ if isinstance(val, np.ndarray):
352
+ setattr(image_processor, attr, val.tolist())
353
+
354
return hf_processor
355
356
def get_image_processor(self):
vllm/multimodal/inputs.py
+2
-2
@@ -141,9 +141,9 @@ Uses a list instead of a tensor if the dimensions of each element do not match.
141
def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
142
"""Equality check between :data:`NestedTensors` objects."""
143
if isinstance(a, torch.Tensor):
144
- return isinstance(b, torch.Tensor) and bool((a == b).all().item())
144
+ return isinstance(b, torch.Tensor) and torch.equal(a, b)
145
elif isinstance(b, torch.Tensor):
146
- return isinstance(a, torch.Tensor) and bool((b == a).all().item())
146
+ return isinstance(a, torch.Tensor) and torch.equal(b, a)
147
148
if isinstance(a, list):
149
return (isinstance(b, list)