A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix bug of xformer prefill for encoder-decoder (#9026)
Xiang Xu committed
Oct 12, 2024 at 00:00 UTC
00298e092c38eb9819f6548a6a246fa207c20c36
1 file changed
+18
-11
vllm/attention/backends/xformers.py
+18
-11
@@ -559,25 +559,32 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
559
self.kv_cache_dtype,
560
k_scale, v_scale)
561
562
- if attn_type != AttentionType.ENCODER:
563
- # Decoder self-attention supports chunked prefill.
564
- # Encoder/decoder cross-attention requires no chunked
565
- # prefill (100% prefill or 100% decode tokens, no mix)
566
- num_prefill_tokens = attn_metadata.num_prefill_tokens
567
- num_decode_tokens = attn_metadata.num_decode_tokens
568
- else:
562
+ if attn_type == AttentionType.ENCODER:
563
# Encoder attention - chunked prefill is not applicable;
564
# derive token-count from query shape & and treat them
565
# as 100% prefill tokens
566
assert attn_metadata.num_encoder_tokens is not None
567
num_prefill_tokens = attn_metadata.num_encoder_tokens
568
+ num_encoder_tokens = attn_metadata.num_encoder_tokens
569
num_decode_tokens = 0
575
-
576
- if attn_type == AttentionType.DECODER:
570
+ elif attn_type == AttentionType.DECODER:
571
+ # Decoder self-attention supports chunked prefill.
572
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
573
+ num_encoder_tokens = attn_metadata.num_prefill_tokens
574
+ num_decode_tokens = attn_metadata.num_decode_tokens
575
# Only enforce this shape-constraint for decoder
576
# self-attention
577
assert key.shape[0] == num_prefill_tokens + num_decode_tokens
578
assert value.shape[0] == num_prefill_tokens + num_decode_tokens
579
+ else: # attn_type == AttentionType.ENCODER_DECODER
580
+ # Encoder/decoder cross-attention requires no chunked
581
+ # prefill (100% prefill or 100% decode tokens, no mix)
582
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
583
+ if attn_metadata.num_encoder_tokens is not None:
584
+ num_encoder_tokens = attn_metadata.num_encoder_tokens
585
+ else:
586
+ num_encoder_tokens = attn_metadata.num_prefill_tokens
587
+ num_decode_tokens = attn_metadata.num_decode_tokens
588
589
output = torch.empty_like(query)
590
# Query for decode. KV is not needed because it is already cached.
@@ -585,8 +592,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
592
# QKV for prefill.
593
query = query[:num_prefill_tokens]
594
if key is not None and value is not None:
588
- key = key[:num_prefill_tokens]
589
- value = value[:num_prefill_tokens]
595
+ key = key[:num_encoder_tokens]
596
+ value = value[:num_encoder_tokens]
597
598
assert query.shape[0] == num_prefill_tokens
599
assert decode_query.shape[0] == num_decode_tokens