A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Bugfix] Fix qwen2.5-vl overflow issue (#13968)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Feb 28, 2025 at 01:30 UTC
78648758794e2268a3af04b1eeab2347d961d88a
4 files changed
+22
-15
vllm/model_executor/models/minicpmo.py
+3
-8
@@ -47,7 +47,7 @@ from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder,
47
MiniCPMVMultiModalDataParser,
48
MiniCPMVMultiModalProcessor, MiniCPMVProcessingInfo,
49
_minicpmv_field_config)
50
-from .utils import AutoWeightsLoader, maybe_prefix
50
+from .utils import AutoWeightsLoader, cast_overflow_tensors, maybe_prefix
51
52
CPU_DEVICE = torch.device("cpu")
53
@@ -469,13 +469,8 @@ class MiniCPMWhisperEncoderLayer(nn.Module):
469
training=self.training)
470
hidden_states = residual + hidden_states
471
472
- if hidden_states.dtype == torch.float16 and (
473
- torch.isinf(hidden_states).any()
474
- or torch.isnan(hidden_states).any()):
475
- clamp_value = torch.finfo(hidden_states.dtype).max - 1000
476
- hidden_states = torch.clamp(hidden_states,
477
- min=-clamp_value,
478
- max=clamp_value)
472
+ if hidden_states.dtype == torch.float16:
473
+ hidden_states = cast_overflow_tensors(hidden_states)
474
475
outputs = (hidden_states, )
476
vllm/model_executor/models/qwen2_5_vl.py
+6
-1
@@ -63,7 +63,7 @@ from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
63
from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder
64
from .qwen2_vl import (Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo,
65
apply_rotary_pos_emb_vision)
66
-from .utils import (AutoWeightsLoader, WeightsMapper,
66
+from .utils import (AutoWeightsLoader, WeightsMapper, cast_overflow_tensors,
67
init_vllm_registered_model, maybe_prefix,
68
merge_multimodal_embeddings)
69
from .vision import get_vit_attn_backend
@@ -641,6 +641,11 @@ class Qwen2_5_VisionTransformer(nn.Module):
641
cu_seqlens=cu_seqlens_now,
642
rotary_pos_emb=rotary_pos_emb)
643
644
+ # For Qwen2.5-VL-3B, float16 will overflow at last block
645
+ # for long visual tokens sequences.
646
+ if hidden_states.dtype == torch.float16:
647
+ hidden_states = cast_overflow_tensors(hidden_states)
648
+
649
# adapter
650
hidden_states = self.merger(hidden_states)
651
reverse_indices = torch.argsort(window_index)
vllm/model_executor/models/utils.py
+10
@@ -641,3 +641,13 @@ def extract_layer_index(layer_name: str) -> int:
641
assert len(int_vals) == 1, (f"layer name {layer_name} should"
642
" only contain one integer")
643
return int_vals[0]
644
+
645
+
646
+def cast_overflow_tensors(
647
+ tensors: torch.Tensor,
648
+ offset: float = 1000,
649
+) -> torch.Tensor:
650
+ if tensors.isinf().any() or tensors.isnan().any():
651
+ clamp_value = torch.finfo(tensors.dtype).max - offset
652
+ tensors = torch.clamp(tensors, min=-clamp_value, max=clamp_value)
653
+ return tensors
vllm/model_executor/models/whisper.py
+3
-6
@@ -35,7 +35,8 @@ from vllm.multimodal.processing import (BaseProcessingInfo,
35
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
36
37
from .interfaces import SupportsMultiModal, SupportsTranscription
38
-from .utils import AutoWeightsLoader, WeightsMapper, make_layers
38
+from .utils import (AutoWeightsLoader, WeightsMapper, cast_overflow_tensors,
39
+ make_layers)
40
41
logger = init_logger(__name__)
42
@@ -285,11 +286,7 @@ class WhisperEncoderLayer(nn.Module):
286
hidden_states = self.mlp(hidden_states)
287
hidden_states = residual + hidden_states
288
288
- if hidden_states.isinf().any() or hidden_states.isnan().any():
289
- clamp_value = torch.finfo(hidden_states.dtype).max - 1000
290
- hidden_states = torch.clamp(hidden_states,
291
- min=-clamp_value,
292
- max=clamp_value)
289
+ hidden_states = cast_overflow_tensors(hidden_states)
290
291
return hidden_states
292