A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[torch.compile] hide slicing under custom op for inductor (#8384)
youkaichao committed
Sep 12, 2024 at 00:11 UTC
7de49aa86c7f169eb0962b6db29ad53fff519ffb
2 files changed
+74
-35
tests/compile/test_full_graph.py
+3
-1
@@ -16,5 +16,7 @@ def test_full_graph(model):
16
"The future of AI is",
17
]
18
sampling_params = SamplingParams(temperature=0)
19
- llm = LLM(model="meta-llama/Meta-Llama-3-8B")
19
+ llm = LLM(model="meta-llama/Meta-Llama-3-8B",
20
+ enforce_eager=True,
21
+ load_format="dummy")
22
llm.generate(prompts, sampling_params)
vllm/attention/backends/flash_attn.py
+71
-34
@@ -122,6 +122,40 @@ def _(
122
return torch.empty_like(decode_query)
123
124
125
+@torch.library.custom_op("vllm::reshape_and_cache_flash",
126
+ mutates_args=["kv_cache"])
127
+def reshape_and_cache_flash(
128
+ key: torch.Tensor,
129
+ value: torch.Tensor,
130
+ kv_cache: torch.Tensor,
131
+ slot_mapping: torch.Tensor,
132
+ kv_cache_dtype: str,
133
+ k_scale: float,
134
+ v_scale: float,
135
+) -> None:
136
+ """Inductor cannot deal with inplace operations on views.
137
+ See https://github.com/pytorch/pytorch/issues/131192
138
+ and https://github.com/pytorch/pytorch/issues/130174
139
+ This is a workaround to hide the view operation from the inductor.
140
+ """
141
+ return torch.ops._C_cache_ops.reshape_and_cache_flash(
142
+ key, value, kv_cache[0], kv_cache[1], slot_mapping, kv_cache_dtype,
143
+ k_scale, v_scale)
144
+
145
+
146
+@reshape_and_cache_flash.register_fake # type: ignore
147
+def _(
148
+ key: torch.Tensor,
149
+ value: torch.Tensor,
150
+ kv_cache: torch.Tensor,
151
+ slot_mapping: torch.Tensor,
152
+ kv_cache_dtype: str,
153
+ k_scale: float,
154
+ v_scale: float,
155
+) -> None:
156
+ pass
157
+
158
+
159
class FlashAttentionBackend(AttentionBackend):
160
161
@staticmethod
@@ -653,11 +687,10 @@ class FlashAttentionImpl(AttentionImpl):
687
# Reshape the input keys and values and store them in the cache.
688
# If kv_cache is not provided, the new key and value tensors are
689
# not cached. This happens during the initial memory profiling run.
656
- ops.reshape_and_cache_flash(
690
+ torch.ops.vllm.reshape_and_cache_flash(
691
key,
692
value,
659
- key_cache,
660
- value_cache,
693
+ kv_cache,
694
attn_metadata.slot_mapping.flatten(),
695
self.kv_cache_dtype,
696
k_scale,
@@ -669,7 +702,6 @@ class FlashAttentionImpl(AttentionImpl):
702
assert key.shape[0] == num_prefill_tokens + num_decode_tokens
703
assert value.shape[0] == num_prefill_tokens + num_decode_tokens
704
672
- output = torch.empty_like(query)
705
# Query for decode. KV is not needed because it is already cached.
706
decode_query = query[num_prefill_tokens:]
707
# QKV for prefill.
@@ -680,6 +712,9 @@ class FlashAttentionImpl(AttentionImpl):
712
assert query.shape[0] == num_prefill_tokens
713
assert decode_query.shape[0] == num_decode_tokens
714
715
+ prefill_output: Optional[torch.Tensor] = None
716
+ decode_output: Optional[torch.Tensor] = None
717
+
718
if prefill_meta := attn_metadata.prefill_metadata:
719
# Prompt run.
720
if (kv_cache is None or prefill_meta.block_tables is None
@@ -687,7 +722,7 @@ class FlashAttentionImpl(AttentionImpl):
722
# normal attention
723
# When block_tables are not filled, it means q and k are the
724
# prompt, and they have the same length.
690
- out = torch.ops.vllm.flash_attn_varlen_func(
725
+ prefill_output = torch.ops.vllm.flash_attn_varlen_func(
726
q=query,
727
k=key,
728
v=value,
@@ -701,42 +736,44 @@ class FlashAttentionImpl(AttentionImpl):
736
alibi_slopes=self.alibi_slopes,
737
softcap=self.logits_soft_cap,
738
)
704
- assert output[:num_prefill_tokens].shape == out.shape
705
- output[:num_prefill_tokens] = out
739
else:
740
# prefix-enabled attention
741
assert prefill_meta.seq_lens is not None
742
max_seq_len = max(prefill_meta.seq_lens)
710
- output[:
711
- num_prefill_tokens] = torch.ops.vllm.flash_attn_varlen_func( # noqa
712
- q=query,
713
- k=key_cache,
714
- v=value_cache,
715
- cu_seqlens_q=prefill_meta.query_start_loc,
716
- max_seqlen_q=prefill_meta.max_query_len,
717
- cu_seqlens_k=prefill_meta.seq_start_loc,
718
- max_seqlen_k=max_seq_len,
719
- softmax_scale=self.scale,
720
- causal=True,
721
- alibi_slopes=self.alibi_slopes,
722
- block_table=prefill_meta.block_tables,
723
- softcap=self.logits_soft_cap,
724
- )
725
-
726
- if decode_meta := attn_metadata.decode_metadata:
727
- # Decoding run.
728
- output[
729
- num_prefill_tokens:] = torch.ops.vllm.flash_attn_with_kvcache(
730
- decode_query.unsqueeze(1),
731
- key_cache,
732
- value_cache,
733
- block_table=decode_meta.block_tables,
734
- cache_seqlens=decode_meta.seq_lens_tensor,
743
+ prefill_output = torch.ops.vllm.flash_attn_varlen_func( # noqa
744
+ q=query,
745
+ k=key_cache,
746
+ v=value_cache,
747
+ cu_seqlens_q=prefill_meta.query_start_loc,
748
+ max_seqlen_q=prefill_meta.max_query_len,
749
+ cu_seqlens_k=prefill_meta.seq_start_loc,
750
+ max_seqlen_k=max_seq_len,
751
softmax_scale=self.scale,
752
causal=True,
753
alibi_slopes=self.alibi_slopes,
754
+ block_table=prefill_meta.block_tables,
755
softcap=self.logits_soft_cap,
739
- ).squeeze(1)
756
+ )
757
741
- # Reshape the output tensor.
758
+ if decode_meta := attn_metadata.decode_metadata:
759
+ # Decoding run.
760
+ decode_output = torch.ops.vllm.flash_attn_with_kvcache(
761
+ decode_query.unsqueeze(1),
762
+ key_cache,
763
+ value_cache,
764
+ block_table=decode_meta.block_tables,
765
+ cache_seqlens=decode_meta.seq_lens_tensor,
766
+ softmax_scale=self.scale,
767
+ causal=True,
768
+ alibi_slopes=self.alibi_slopes,
769
+ softcap=self.logits_soft_cap,
770
+ ).squeeze(1)
771
+
772
+ if prefill_output is None:
773
+ assert decode_output is not None
774
+ return decode_output.view(num_decode_tokens, hidden_size)
775
+ if decode_output is None:
776
+ assert prefill_output is not None
777
+ return prefill_output.view(num_prefill_tokens, hidden_size)
778
+ output = torch.cat([prefill_output, decode_output], dim=0)
779
return output.view(num_tokens, hidden_size)