A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Misc] Qwen2.5-VL Optimization (#13155)
燃 committed
Feb 13, 2025 at 22:17 UTC
02ed8a1fbe41e3ad1bc04fd29b754facd28e329f
2 files changed
+47
-51
vllm/model_executor/models/qwen2_5_vl.py
+25
-36
@@ -45,6 +45,7 @@ from vllm.distributed import utils as dist_utils
45
from vllm.logger import init_logger
46
from vllm.model_executor import SamplingMetadata
47
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
48
+from vllm.model_executor.layers.layernorm import RMSNorm
49
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
50
RowParallelLinear)
51
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -271,8 +272,13 @@ class Qwen2_5_VisionAttention(nn.Module):
272
q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
273
for x in (q, k, v))
274
if rotary_pos_emb is not None:
274
- q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
275
- k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
275
+ use_flash_attn = self.attn_backend == _Backend.FLASH_ATTN
276
+ q = apply_rotary_pos_emb_vision(q,
277
+ rotary_pos_emb,
278
+ use_flash_attn=use_flash_attn)
279
+ k = apply_rotary_pos_emb_vision(k,
280
+ rotary_pos_emb,
281
+ use_flash_attn=use_flash_attn)
282
283
if self.attn_backend == _Backend.FLASH_ATTN:
284
# from vllm_flash_attn.flash_attn_interface import (
@@ -296,20 +302,23 @@ class Qwen2_5_VisionAttention(nn.Module):
302
"(b s) ... -> b s ...",
303
b=batch_size)
304
elif self.attn_backend == _Backend.TORCH_SDPA:
299
- seq_length = q.size(1)
300
- q, k, v = (rearrange(x, "b s h d -> b h s d") for x in [q, k, v])
301
- attention_mask = torch.zeros([1, seq_length, seq_length],
302
- device=q.device,
303
- dtype=torch.bool)
305
+ # Execute attention entry by entry for speed & less VRAM.
306
+ outputs = []
307
for i in range(1, len(cu_seqlens)):
305
- attention_mask[..., cu_seqlens[i - 1]:cu_seqlens[i],
306
- cu_seqlens[i - 1]:cu_seqlens[i]] = True
307
- output = F.scaled_dot_product_attention(q,
308
- k,
309
- v,
310
- attention_mask,
311
- dropout_p=0.0)
312
- context_layer = rearrange(output, "b h s d -> b s h d ")
308
+ start_idx = cu_seqlens[i - 1]
309
+ end_idx = cu_seqlens[i]
310
+ q_i = q[:, start_idx:end_idx]
311
+ k_i = k[:, start_idx:end_idx]
312
+ v_i = v[:, start_idx:end_idx]
313
+ q_i, k_i, v_i = (rearrange(x, "b s h d -> b h s d")
314
+ for x in [q_i, k_i, v_i])
315
+ output_i = F.scaled_dot_product_attention(q_i,
316
+ k_i,
317
+ v_i,
318
+ dropout_p=0.0)
319
+ output_i = rearrange(output_i, "b h s d -> b s h d ")
320
+ outputs.append(output_i)
321
+ context_layer = torch.cat(outputs, dim=1)
322
elif self.attn_backend == _Backend.XFORMERS:
323
from xformers import ops as xops
324
from xformers.ops.fmha.attn_bias import BlockDiagonalMask
@@ -327,25 +336,6 @@ class Qwen2_5_VisionAttention(nn.Module):
336
return output
337
338
330
-class Qwen2RMSNorm(nn.Module):
331
-
332
- def __init__(self, hidden_size, eps=1e-6):
333
- super().__init__()
334
- self.weight = nn.Parameter(torch.ones(hidden_size))
335
- self.variance_epsilon = eps
336
-
337
- def forward(self, hidden_states):
338
- input_dtype = hidden_states.dtype
339
- hidden_states = hidden_states.to(torch.float32)
340
- variance = hidden_states.pow(2).mean(-1, keepdim=True)
341
- hidden_states = hidden_states * torch.rsqrt(variance +
342
- self.variance_epsilon)
343
- return self.weight * hidden_states.to(input_dtype)
344
-
345
- def extra_repr(self):
346
- return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
347
-
348
-
339
class Qwen2_5_VisionBlock(nn.Module):
340
341
def __init__(
@@ -516,8 +506,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
506
hidden_size=self.hidden_size,
507
)
508
519
- # NOTE: We use torch native RMSNorm here for precision purposes.
520
- norm_layer = partial(Qwen2RMSNorm, eps=norm_eps)
509
+ norm_layer = partial(RMSNorm, eps=norm_eps)
510
head_dim = self.hidden_size // self.num_heads
511
self.rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2)
512
vllm/model_executor/models/qwen2_vl.py
+22
-15
@@ -226,11 +226,15 @@ def apply_rotary_emb_torch(x: torch.Tensor,
226
227
228
def apply_rotary_pos_emb_vision(t: torch.Tensor,
229
- freqs: torch.Tensor) -> torch.Tensor:
229
+ freqs: torch.Tensor,
230
+ use_flash_attn=False) -> torch.Tensor:
231
t_ = t.float()
232
cos = freqs.cos()
233
sin = freqs.sin()
233
- output = apply_rotary_emb_torch(t_, cos, sin).type_as(t)
234
+ apply_rotary_emb = apply_rotary_emb_torch
235
+ if use_flash_attn:
236
+ from flash_attn.layers.rotary import apply_rotary_emb
237
+ output = apply_rotary_emb(t_, cos, sin).type_as(t)
238
return output
239
240
@@ -336,20 +340,23 @@ class Qwen2VisionAttention(nn.Module):
340
"(b s) ... -> b s ...",
341
b=batch_size)
342
elif self.attn_backend == _Backend.TORCH_SDPA:
339
- seq_length = q.size(1)
340
- q, k, v = (rearrange(x, "b s h d -> b h s d") for x in [q, k, v])
341
- attention_mask = torch.zeros([1, seq_length, seq_length],
342
- device=q.device,
343
- dtype=torch.bool)
343
+ # Execute attention entry by entry for speed & less VRAM.
344
+ outputs = []
345
for i in range(1, len(cu_seqlens)):
345
- attention_mask[..., cu_seqlens[i - 1]:cu_seqlens[i],
346
- cu_seqlens[i - 1]:cu_seqlens[i]] = True
347
- output = F.scaled_dot_product_attention(q,
348
- k,
349
- v,
350
- attention_mask,
351
- dropout_p=0.0)
352
- context_layer = rearrange(output, "b h s d -> b s h d ")
346
+ start_idx = cu_seqlens[i - 1]
347
+ end_idx = cu_seqlens[i]
348
+ q_i = q[:, start_idx:end_idx]
349
+ k_i = k[:, start_idx:end_idx]
350
+ v_i = v[:, start_idx:end_idx]
351
+ q_i, k_i, v_i = (rearrange(x, "b s h d -> b h s d")
352
+ for x in [q_i, k_i, v_i])
353
+ output_i = F.scaled_dot_product_attention(q_i,
354
+ k_i,
355
+ v_i,
356
+ dropout_p=0.0)
357
+ output_i = rearrange(output_i, "b h s d -> b s h d ")
358
+ outputs.append(output_i)
359
+ context_layer = torch.cat(outputs, dim=1)
360
elif self.attn_backend == _Backend.XFORMERS:
361
from xformers import ops as xops
362
from xformers.ops.fmha.attn_bias import BlockDiagonalMask