@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 00298e092c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fix bug of xformer prefill for encoder-decoder (#9026)

Xiang Xu committed Oct 12, 2024 at 00:00 UTC 00298e092c38eb9819f6548a6a246fa207c20c36
1 file changed +18 -11
vllm/attention/backends/xformers.py
+18 -11
@@ -559,25 +559,32 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
559 self.kv_cache_dtype,
560 k_scale, v_scale)
561
562 - if attn_type != AttentionType.ENCODER:
563 - # Decoder self-attention supports chunked prefill.
564 - # Encoder/decoder cross-attention requires no chunked
565 - # prefill (100% prefill or 100% decode tokens, no mix)
566 - num_prefill_tokens = attn_metadata.num_prefill_tokens
567 - num_decode_tokens = attn_metadata.num_decode_tokens
568 - else:
562 + if attn_type == AttentionType.ENCODER:
563 # Encoder attention - chunked prefill is not applicable;
564 # derive token-count from query shape & and treat them
565 # as 100% prefill tokens
566 assert attn_metadata.num_encoder_tokens is not None
567 num_prefill_tokens = attn_metadata.num_encoder_tokens
568 + num_encoder_tokens = attn_metadata.num_encoder_tokens
569 num_decode_tokens = 0
575 -
576 - if attn_type == AttentionType.DECODER:
570 + elif attn_type == AttentionType.DECODER:
571 + # Decoder self-attention supports chunked prefill.
572 + num_prefill_tokens = attn_metadata.num_prefill_tokens
573 + num_encoder_tokens = attn_metadata.num_prefill_tokens
574 + num_decode_tokens = attn_metadata.num_decode_tokens
575 # Only enforce this shape-constraint for decoder
576 # self-attention
577 assert key.shape[0] == num_prefill_tokens + num_decode_tokens
578 assert value.shape[0] == num_prefill_tokens + num_decode_tokens
579 + else: # attn_type == AttentionType.ENCODER_DECODER
580 + # Encoder/decoder cross-attention requires no chunked
581 + # prefill (100% prefill or 100% decode tokens, no mix)
582 + num_prefill_tokens = attn_metadata.num_prefill_tokens
583 + if attn_metadata.num_encoder_tokens is not None:
584 + num_encoder_tokens = attn_metadata.num_encoder_tokens
585 + else:
586 + num_encoder_tokens = attn_metadata.num_prefill_tokens
587 + num_decode_tokens = attn_metadata.num_decode_tokens
588
589 output = torch.empty_like(query)
590 # Query for decode. KV is not needed because it is already cached.
@@ -585,8 +592,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
592 # QKV for prefill.
593 query = query[:num_prefill_tokens]
594 if key is not None and value is not None:
588 - key = key[:num_prefill_tokens]
589 - value = value[:num_prefill_tokens]
595 + key = key[:num_encoder_tokens]
596 + value = value[:num_encoder_tokens]
597
598 assert query.shape[0] == num_prefill_tokens
599 assert decode_query.shape[0] == num_decode_tokens