@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 75404d041b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[VLM] Update compatibility with transformers 4.49

Cyrus Leung committed Feb 6, 2025 at 11:09 UTC 75404d041be0d6e656b59cbbea23520d47d37b66
9 files changed +50 -59
docs/source/models/supported_models.md
+1 -2
@@ -883,8 +883,7 @@ For more details, please see: <gh-pr:4087#issuecomment-2250397630>
883 :::
884
885 :::{note}
886 -The chat template for Pixtral-HF is incorrect (see [discussion](https://huggingface.co/mistral-community/pixtral-12b/discussions/22)).
887 -A corrected version is available at <gh-file:examples/template_pixtral_hf.jinja>.
886 +`mistral-community/pixtral-12b` does not support V1 yet.
887 :::
888
889 :::{note}
examples/template_pixtral_hf.jinja deleted
-38
@@ -1,38 +0,0 @@
1 -{%- if messages[0]["role"] == "system" %}
2 - {%- set system_message = messages[0]["content"] %}
3 - {%- set loop_messages = messages[1:] %}
4 -{%- else %}
5 - {%- set loop_messages = messages %}
6 -{%- endif %}
7 -
8 -{{- bos_token }}
9 -{%- for message in loop_messages %}
10 - {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
11 - {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}
12 - {%- endif %}
13 - {%- if message["role"] == "user" %}
14 - {%- if loop.last and system_message is defined %}
15 - {{- "[INST]" + system_message + "\n" }}
16 - {%- else %}
17 - {{- "[INST]" }}
18 - {%- endif %}
19 - {%- if message["content"] is not string %}
20 - {%- for chunk in message["content"] %}
21 - {%- if chunk["type"] == "text" %}
22 - {{- chunk["text"] }}
23 - {%- elif chunk["type"] == "image" %}
24 - {{- "[IMG]" }}
25 - {%- else %}
26 - {{- raise_exception("Unrecognized content type!") }}
27 - {%- endif %}
28 - {%- endfor %}
29 - {%- else %}
30 - {{- message["content"] }}
31 - {%- endif %}
32 - {{- "[/INST]" }}
33 - {%- elif message["role"] == "assistant" %}
34 - {{- message["content"] + eos_token}}
35 - {%- else %}
36 - {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
37 - {%- endif %}
38 -{%- endfor %}
tests/entrypoints/test_chat_utils.py
-1
@@ -761,7 +761,6 @@ def test_resolve_content_format_hf_defined(model, expected_format):
761 ("template_falcon.jinja", "string"),
762 ("template_inkbot.jinja", "string"),
763 ("template_llava.jinja", "string"),
764 - ("template_pixtral_hf.jinja", "openai"),
764 ("template_vlm2vec.jinja", "openai"),
765 ("tool_chat_template_granite_20b_fc.jinja", "string"),
766 ("tool_chat_template_hermes.jinja", "string"),
tests/models/decoder_only/vision_language/test_models.py
+2 -2
@@ -224,7 +224,7 @@ VLM_TEST_SETTINGS = {
224 marks=[
225 pytest.mark.skipif(
226 Version(TRANSFORMERS_VERSION) >= Version("4.48"),
227 - reason="HF model is not compatible with transformers>=4.48.0",
227 + reason="HF model is not compatible with transformers>=4.48",
228 )
229 ],
230 ),
@@ -359,7 +359,7 @@ VLM_TEST_SETTINGS = {
359 marks=[
360 pytest.mark.skipif(
361 Version(TRANSFORMERS_VERSION) >= Version("4.48"),
362 - reason="HF model is not compatible with transformers>=4.48.0",
362 + reason="HF model is not compatible with transformers>=4.48",
363 )
364 ],
365 ),
tests/models/embedding/vision_language/test_llava_next.py
+4 -3
@@ -4,7 +4,6 @@ from typing import List, Type
4
5 import pytest
6 import torch.nn.functional as F
7 -import transformers
7 from transformers import AutoModelForVision2Seq
8
9 from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
@@ -57,6 +56,10 @@ def _run_test(
56
57 with hf_runner(model, dtype=dtype,
58 auto_cls=AutoModelForVision2Seq) as hf_model:
59 + # Patch the issue where generation_config.json is missing
60 + hf_model.processor.patch_size = \
61 + hf_model.model.config.vision_config.patch_size
62 +
63 # Patch the issue where image_token_id
64 # exceeds the maximum allowed vocab size
65 hf_model.model.resize_token_embeddings(
@@ -88,8 +91,6 @@ def _run_test(
91 )
92
93
91 -@pytest.mark.skipif(transformers.__version__ >= "4.46",
92 - reason="Model broken with changes in transformers 4.46")
94 @pytest.mark.core_model
95 @pytest.mark.parametrize("model", MODELS)
96 @pytest.mark.parametrize("dtype", ["half"])
vllm/model_executor/models/llava.py
+23 -10
@@ -293,16 +293,29 @@ class PixtralHFMultiModalProcessor(
293
294 pixel_values = processed_outputs.get("pixel_values")
295 if pixel_values is not None:
296 - images = mm_data["images"]
297 - assert isinstance(images, list)
298 -
299 - # Original output: (1, num_images, C, H, W)
300 - # New output: (num_images, C, H, W)
301 - assert (isinstance(pixel_values, list) and len(pixel_values) == 1)
302 - assert (isinstance(pixel_values[0], list)
303 - and len(pixel_values[0]) == len(images))
304 -
305 - processed_outputs["pixel_values"] = pixel_values[0]
296 + # Before/after https://github.com/huggingface/transformers/pull/35122
297 + if Version(TRANSFORMERS_VERSION) <= Version("4.48.2"):
298 + images = mm_data["images"]
299 + assert isinstance(images, list)
300 +
301 + # Original output: (1, num_images, C, H, W)
302 + # New output: (num_images, C, H, W)
303 + assert (isinstance(pixel_values, list)
304 + and len(pixel_values) == 1)
305 + assert (isinstance(pixel_values[0], list)
306 + and len(pixel_values[0]) == len(images))
307 +
308 + processed_outputs["pixel_values"] = pixel_values[0]
309 + else:
310 + # Avoid padding since we need the output for each image to be
311 + # independent of other images for the cache to work correctly
312 + image_sizes = processed_outputs["image_sizes"]
313 + assert len(pixel_values) == len(image_sizes)
314 +
315 + processed_outputs["pixel_values"] = [
316 + p[:, :h, :w]
317 + for p, (h, w) in zip(pixel_values, image_sizes)
318 + ]
319
320 return processed_outputs
321
vllm/model_executor/models/llava_next.py
+9 -1
@@ -73,7 +73,15 @@ class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
73 return self.ctx.get_hf_config(LlavaNextConfig)
74
75 def get_hf_processor(self):
76 - return self.ctx.get_hf_processor(LlavaNextProcessor)
76 + hf_processor = self.ctx.get_hf_processor(LlavaNextProcessor)
77 +
78 + # In case patch_size is omitted from `processor_config.json`
79 + # e.g. for E5-V: https://huggingface.co/royokong/e5-v
80 + if hf_processor.patch_size is None:
81 + patch_size = self.get_vision_encoder_info().get_patch_size()
82 + hf_processor.patch_size = patch_size
83 +
84 + return hf_processor
85
86 # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L113
87 def get_num_image_tokens(
vllm/model_executor/models/minicpmv.py
+9
@@ -342,6 +342,15 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
342 **kwargs: object,
343 ):
344 hf_processor = self.ctx.get_hf_processor()
345 +
346 + # NumPy arrays are considered as Iterable but not Sequence in
347 + # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
348 + image_processor = hf_processor.image_processor # type: ignore
349 + for attr in ("mean", "std"):
350 + val = getattr(image_processor, attr)
351 + if isinstance(val, np.ndarray):
352 + setattr(image_processor, attr, val.tolist())
353 +
354 return hf_processor
355
356 def get_image_processor(self):
vllm/multimodal/inputs.py
+2 -2
@@ -141,9 +141,9 @@ Uses a list instead of a tensor if the dimensions of each element do not match.
141 def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
142 """Equality check between :data:`NestedTensors` objects."""
143 if isinstance(a, torch.Tensor):
144 - return isinstance(b, torch.Tensor) and bool((a == b).all().item())
144 + return isinstance(b, torch.Tensor) and torch.equal(a, b)
145 elif isinstance(b, torch.Tensor):
146 - return isinstance(a, torch.Tensor) and bool((b == a).all().item())
146 + return isinstance(a, torch.Tensor) and torch.equal(b, a)
147
148 if isinstance(a, list):
149 return (isinstance(b, list)