@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ca47e176af
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Misc] Move some model utils into vision file (#11848)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 9, 2025 at 01:04 UTC ca47e176af9e0a4fa9f02325cdad5f11b40aedab
8 files changed +94 -92
vllm/model_executor/models/clip.py
+2 -3
@@ -20,11 +20,10 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
20 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 from vllm.multimodal.utils import (cached_get_tokenizer,
22 consecutive_placeholder_ranges,
23 - repeat_and_pad_placeholder_tokens,
24 - resolve_visual_encoder_outputs)
23 + repeat_and_pad_placeholder_tokens)
24 from vllm.sequence import SequenceData
25
27 -from .vision import VisionEncoderInfo
26 +from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
27
28
29 def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
vllm/model_executor/models/pixtral.py
+2 -3
@@ -31,14 +31,13 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
31 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
32 from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
33 from vllm.multimodal.utils import (cached_get_tokenizer,
34 - consecutive_placeholder_ranges,
35 - resolve_visual_encoder_outputs)
34 + consecutive_placeholder_ranges)
35 from vllm.sequence import IntermediateTensors, SequenceData
36
37 from .interfaces import SupportsMultiModal, SupportsPP
38 from .utils import (init_vllm_registered_model, maybe_prefix,
39 merge_multimodal_embeddings)
41 -from .vision import VisionEncoderInfo
40 +from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
41
42 try:
43 from xformers import ops as xops
vllm/model_executor/models/qwen2_vl.py
+2 -1
@@ -66,8 +66,9 @@ from vllm.sequence import IntermediateTensors
66 from vllm.transformers_utils.config import uses_mrope
67
68 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
69 -from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
69 +from .utils import (AutoWeightsLoader, WeightsMapper,
70 init_vllm_registered_model, maybe_prefix)
71 +from .vision import get_vit_attn_backend
72
73 logger = init_logger(__name__)
74
vllm/model_executor/models/siglip.py
+2 -3
@@ -24,11 +24,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
24 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
25 from vllm.multimodal.utils import (cached_get_tokenizer,
26 consecutive_placeholder_ranges,
27 - repeat_and_pad_placeholder_tokens,
28 - resolve_visual_encoder_outputs)
27 + repeat_and_pad_placeholder_tokens)
28 from vllm.sequence import SequenceData
29
31 -from .vision import VisionEncoderInfo
30 +from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
31
32
33 def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
vllm/model_executor/models/utils.py
+1 -36
@@ -8,16 +8,12 @@ import torch.nn as nn
8 from torch.func import functional_call
9 from transformers import PretrainedConfig
10
11 -import vllm.envs as envs
12 -from vllm.attention.selector import (backend_name_to_enum,
13 - get_global_forced_attn_backend)
11 from vllm.config import VllmConfig
12 from vllm.logger import init_logger
13 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
14 from vllm.multimodal import MultiModalPlaceholderMap, NestedTensors
18 -from vllm.platforms import _Backend, current_platform
15 from vllm.sequence import IntermediateTensors
20 -from vllm.utils import is_pin_memory_available, print_warning_once
16 +from vllm.utils import is_pin_memory_available
17
18 logger = init_logger(__name__)
19
@@ -612,37 +608,6 @@ def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
608 return make_empty_intermediate_tensors
609
610
615 -def get_vit_attn_backend(support_fa: bool = False) -> _Backend:
616 - """
617 - Get the available attention backend for Vision Transformer.
618 - """
619 - # TODO(Isotr0py): Remove `support_fa` after support FA for all ViTs attn.
620 - selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
621 - if selected_backend is None:
622 - backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
623 - if backend_by_env_var is not None:
624 - selected_backend = backend_name_to_enum(backend_by_env_var)
625 - if selected_backend is None:
626 - # For Volta and Turing GPUs, use xformers instead.
627 - device_available = current_platform.has_device_capability(80)
628 - if device_available and support_fa:
629 - from transformers.utils import is_flash_attn_2_available
630 - if is_flash_attn_2_available():
631 - selected_backend = _Backend.FLASH_ATTN
632 - else:
633 - print_warning_once(
634 - "Current `vllm-flash-attn` has a bug inside vision module, "
635 - "so we use xformers backend instead. You can run "
636 - "`pip install flash-attn` to use flash-attention backend.")
637 - selected_backend = _Backend.XFORMERS
638 - elif current_platform.is_cpu() or current_platform.is_rocm():
639 - # ROCM doesn't support xformers
640 - selected_backend = _Backend.TORCH_SDPA
641 - else:
642 - selected_backend = _Backend.XFORMERS
643 - return selected_backend
644 -
645 -
611 def maybe_prefix(prefix: str, name: str) -> str:
612 """Add a prefix to a name if the prefix is non-empty.
613
vllm/model_executor/models/vision.py
+82 -1
@@ -1,8 +1,15 @@
1 from abc import ABC, abstractmethod
2 -from typing import Final, Generic, Protocol, TypeVar
2 +from typing import Final, Generic, Optional, Protocol, TypeVar, Union
3
4 +import torch
5 from transformers import PretrainedConfig
6
7 +import vllm.envs as envs
8 +from vllm.attention.selector import (backend_name_to_enum,
9 + get_global_forced_attn_backend)
10 +from vllm.platforms import _Backend, current_platform
11 +from vllm.utils import print_warning_once
12 +
13 _C = TypeVar("_C", bound=PretrainedConfig)
14
15
@@ -60,3 +67,77 @@ def get_vision_encoder_info(
67
68 msg = f"Unsupported vision config: {type(vision_config)}"
69 raise NotImplementedError(msg)
70 +
71 +
72 +def get_vit_attn_backend(support_fa: bool = False) -> _Backend:
73 + """
74 + Get the available attention backend for Vision Transformer.
75 + """
76 + # TODO(Isotr0py): Remove `support_fa` after support FA for all ViTs attn.
77 + selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
78 + if selected_backend is None:
79 + backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
80 + if backend_by_env_var is not None:
81 + selected_backend = backend_name_to_enum(backend_by_env_var)
82 + if selected_backend is None:
83 + # For Volta and Turing GPUs, use xformers instead.
84 + device_available = current_platform.has_device_capability(80)
85 + if device_available and support_fa:
86 + from transformers.utils import is_flash_attn_2_available
87 + if is_flash_attn_2_available():
88 + selected_backend = _Backend.FLASH_ATTN
89 + else:
90 + print_warning_once(
91 + "Current `vllm-flash-attn` has a bug inside vision module, "
92 + "so we use xformers backend instead. You can run "
93 + "`pip install flash-attn` to use flash-attention backend.")
94 + selected_backend = _Backend.XFORMERS
95 + elif current_platform.is_cpu() or current_platform.is_rocm():
96 + # ROCM doesn't support xformers
97 + selected_backend = _Backend.TORCH_SDPA
98 + else:
99 + selected_backend = _Backend.XFORMERS
100 + return selected_backend
101 +
102 +
103 +def resolve_visual_encoder_outputs(
104 + encoder_outputs: Union[torch.Tensor, list[torch.Tensor]],
105 + feature_sample_layers: Optional[list[int]],
106 + post_layer_norm: Optional[torch.nn.LayerNorm],
107 + max_possible_layers: int,
108 +) -> torch.Tensor:
109 + """Given the outputs a visual encoder module that may correspond to the
110 + output of the last layer, or a list of hidden states to be stacked,
111 + handle post normalization and resolve it into a single output tensor.
112 +
113 + Args:
114 + encoder_outputs: Output of encoder's last layer or all hidden states.
115 + feature_sample_layers: Optional layer indices to grab from the encoder
116 + outputs; if provided, encoder outputs must be a list.
117 + post_layer_norm: Post norm to apply to the output of the encoder.
118 + max_possible_layers: Total layers in the fully loaded visual encoder.
119 +
120 + """
121 + if feature_sample_layers is None:
122 + if post_layer_norm is not None:
123 + return post_layer_norm(encoder_outputs)
124 + return encoder_outputs
125 +
126 + # Get the hidden states corresponding to the layer indices.
127 + # Negative values are relative to the full visual encoder,
128 + # so offset them depending on how many layers were loaded.
129 + # NOTE: this assumes that encoder_outputs contains a list
130 + # of hidden states in the same order as the encoder layers
131 + # that produced them.
132 + offset = max_possible_layers - len(encoder_outputs)
133 + hs_pool = [
134 + encoder_outputs[layer_idx]
135 + if layer_idx >= 0 else encoder_outputs[layer_idx + offset]
136 + for layer_idx in feature_sample_layers
137 + ]
138 +
139 + # Apply post-norm on the final hidden state if we are using it
140 + uses_last_layer = feature_sample_layers[-1] in (len(hs_pool) - 1, -1)
141 + if post_layer_norm is not None and uses_last_layer:
142 + hs_pool[-1] = post_layer_norm(encoder_outputs)
143 + return torch.cat(hs_pool, dim=-1)
vllm/multimodal/inputs.py
+3 -1
@@ -99,6 +99,8 @@ class MultiModalDataBuiltins(TypedDict, total=False):
99 MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]
100 """
101 A dictionary containing an entry for each modality type to input.
102 +
103 +The built-in modalities are defined by :class:`MultiModalDataBuiltins`.
104 """
105
106
@@ -485,7 +487,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
487
488 MultiModalPlaceholderDict = Mapping[str, Sequence[PlaceholderRange]]
489 """
488 -A dictionary containing placeholder ranges.
490 +A dictionary containing placeholder ranges for each modality.
491 """
492
493
vllm/multimodal/utils.py
-44
@@ -5,7 +5,6 @@ from urllib.parse import ParseResult, urlparse
5
6 import numpy as np
7 import numpy.typing as npt
8 -import torch
8 from PIL import Image
9
10 import vllm.envs as envs
@@ -285,49 +284,6 @@ def encode_video_base64(frames: npt.NDArray) -> str:
284 return video_io.encode_base64(frames)
285
286
288 -def resolve_visual_encoder_outputs(
289 - encoder_outputs: Union[torch.Tensor, list[torch.Tensor]],
290 - feature_sample_layers: Optional[list[int]],
291 - post_layer_norm: Optional[torch.nn.LayerNorm],
292 - max_possible_layers: int,
293 -) -> torch.Tensor:
294 - """Given the outputs a visual encoder module that may correspond to the
295 - output of the last layer, or a list of hidden states to be stacked,
296 - handle post normalization and resolve it into a single output tensor.
297 -
298 - Args:
299 - encoder_outputs: Output of encoder's last layer or all hidden states.
300 - feature_sample_layers: Optional layer indices to grab from the encoder
301 - outputs; if provided, encoder outputs must be a list.
302 - post_layer_norm: Post norm to apply to the output of the encoder.
303 - max_possible_layers: Total layers in the fully loaded visual encoder.
304 -
305 - """
306 - if feature_sample_layers is None:
307 - if post_layer_norm is not None:
308 - return post_layer_norm(encoder_outputs)
309 - return encoder_outputs
310 -
311 - # Get the hidden states corresponding to the layer indices.
312 - # Negative values are relative to the full visual encoder,
313 - # so offset them depending on how many layers were loaded.
314 - # NOTE: this assumes that encoder_outputs contains a list
315 - # of hidden states in the same order as the encoder layers
316 - # that produced them.
317 - offset = max_possible_layers - len(encoder_outputs)
318 - hs_pool = [
319 - encoder_outputs[layer_idx]
320 - if layer_idx >= 0 else encoder_outputs[layer_idx + offset]
321 - for layer_idx in feature_sample_layers
322 - ]
323 -
324 - # Apply post-norm on the final hidden state if we are using it
325 - uses_last_layer = feature_sample_layers[-1] in (len(hs_pool) - 1, -1)
326 - if post_layer_norm is not None and uses_last_layer:
327 - hs_pool[-1] = post_layer_norm(encoder_outputs)
328 - return torch.cat(hs_pool, dim=-1)
329 -
330 -
287 # Utilities for input processors
288 _T = TypeVar("_T", str, int)
289