A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[misc] add forward context for attention (#9029)
youkaichao committed
Oct 3, 2024 at 12:09 UTC
9aaf14c62e16a7c74b5192a44d01a78125dab2fc
8 files changed
+250
-334
tests/kernels/test_flash_attn.py
+7
-49
@@ -3,9 +3,9 @@ from typing import List, Optional, Tuple
3
import pytest
4
import torch
5
6
-import vllm.attention.backends.flash_attn # noqa: F401
7
-from tests.kernels.utils import opcheck
6
from vllm.utils import seed_everything
7
+from vllm.vllm_flash_attn import (flash_attn_varlen_func,
8
+ flash_attn_with_kvcache)
9
10
NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
11
HEAD_SIZES = [128, 256]
@@ -112,10 +112,10 @@ def test_flash_attn_with_paged_kv(
112
(num_seqs, max_num_blocks_per_seq),
113
dtype=torch.int32)
114
115
- output = torch.ops.vllm.flash_attn_with_kvcache(
116
- decode_query=query.unsqueeze(1),
117
- key_cache=key_cache,
118
- value_cache=value_cache,
115
+ output = flash_attn_with_kvcache(
116
+ q=query.unsqueeze(1),
117
+ k_cache=key_cache,
118
+ v_cache=value_cache,
119
softmax_scale=scale,
120
causal=True,
121
block_table=block_tables,
@@ -123,25 +123,6 @@ def test_flash_attn_with_paged_kv(
123
softcap=soft_cap if soft_cap is not None else 0,
124
).squeeze(1)
125
126
- if num_blocks <= 2048:
127
- test_utils = ["test_faketensor", "test_schema"]
128
- else:
129
- test_utils = ["test_faketensor"]
130
-
131
- opcheck(torch.ops.vllm.flash_attn_with_kvcache,
132
- args=tuple(),
133
- kwargs=dict(
134
- decode_query=query.unsqueeze(1),
135
- key_cache=key_cache,
136
- value_cache=value_cache,
137
- softmax_scale=scale,
138
- causal=True,
139
- block_table=block_tables,
140
- cache_seqlens=kv_lens_tensor,
141
- softcap=soft_cap if soft_cap is not None else 0,
142
- ),
143
- test_utils=test_utils)
144
-
126
ref_output = ref_paged_attn(
127
query=query,
128
key_cache=key_cache,
@@ -213,7 +194,7 @@ def test_varlen_with_paged_kv(
194
(num_seqs, max_num_blocks_per_seq),
195
dtype=torch.int32)
196
216
- output = torch.ops.vllm.flash_attn_varlen_func(
197
+ output = flash_attn_varlen_func(
198
q=query,
199
k=key_cache,
200
v=value_cache,
@@ -228,29 +209,6 @@ def test_varlen_with_paged_kv(
209
softcap=soft_cap if soft_cap is not None else 0,
210
)
211
231
- if num_blocks <= 2048:
232
- test_utils = ["test_faketensor", "test_schema"]
233
- else:
234
- test_utils = ["test_faketensor"]
235
-
236
- opcheck(torch.ops.vllm.flash_attn_varlen_func,
237
- args=tuple(),
238
- kwargs=dict(
239
- q=query,
240
- k=key_cache,
241
- v=value_cache,
242
- cu_seqlens_q=cu_query_lens,
243
- cu_seqlens_k=cu_kv_lens,
244
- max_seqlen_q=max_query_len,
245
- max_seqlen_k=max_kv_len,
246
- softmax_scale=scale,
247
- causal=True,
248
- window_size=window_size,
249
- block_table=block_tables,
250
- softcap=soft_cap if soft_cap is not None else 0,
251
- ),
252
- test_utils=test_utils)
253
-
212
ref_output = ref_paged_attn(
213
query=query,
214
key_cache=key_cache,
vllm/attention/backends/flash_attn.py
+178
-251
@@ -13,152 +13,15 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
13
compute_slot_mapping,
14
compute_slot_mapping_start_idx,
15
is_block_tables_empty)
16
+from vllm.forward_context import get_forward_context
17
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
18
19
if TYPE_CHECKING:
20
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
21
ModelInputForGPUWithSamplingMetadata)
22
22
-# yapf: disable
23
-from vllm.vllm_flash_attn import (
24
- flash_attn_varlen_func as _flash_attn_varlen_func)
25
-from vllm.vllm_flash_attn import (
26
- flash_attn_with_kvcache as _flash_attn_with_kvcache)
27
-
28
-# yapf: enable
29
-
30
-
31
-@torch.library.custom_op("vllm::flash_attn_varlen_func", mutates_args=[])
32
-def flash_attn_varlen_func(
33
- q: torch.Tensor,
34
- k: torch.Tensor,
35
- v: torch.Tensor,
36
- cu_seqlens_q: torch.Tensor,
37
- cu_seqlens_k: torch.Tensor,
38
- max_seqlen_q: int,
39
- max_seqlen_k: int,
40
- softmax_scale: Optional[float] = None,
41
- causal: bool = False,
42
- window_size: Optional[List[int]] = None,
43
- softcap: float = 0.0,
44
- alibi_slopes: Optional[torch.Tensor] = None,
45
- block_table: Optional[torch.Tensor] = None,
46
-) -> torch.Tensor:
47
- # custom op does not support tuple input
48
- real_window_size: Tuple[int, int]
49
- if window_size is None:
50
- real_window_size = (-1, -1)
51
- else:
52
- assert len(window_size) == 2
53
- real_window_size = (window_size[0], window_size[1])
54
- return _flash_attn_varlen_func(
55
- q=q,
56
- k=k,
57
- v=v,
58
- cu_seqlens_q=cu_seqlens_q,
59
- cu_seqlens_k=cu_seqlens_k,
60
- max_seqlen_q=max_seqlen_q,
61
- max_seqlen_k=max_seqlen_k,
62
- softmax_scale=softmax_scale,
63
- causal=causal,
64
- window_size=real_window_size,
65
- softcap=softcap,
66
- alibi_slopes=alibi_slopes,
67
- block_table=block_table,
68
- )
69
-
70
-
71
-@flash_attn_varlen_func.register_fake # type: ignore
72
-def _(
73
- q: torch.Tensor,
74
- k: torch.Tensor,
75
- v: torch.Tensor,
76
- cu_seqlens_q: torch.Tensor,
77
- cu_seqlens_k: torch.Tensor,
78
- max_seqlen_q: int,
79
- max_seqlen_k: int,
80
- softmax_scale: Optional[float] = None,
81
- causal: bool = False,
82
- window_size: Optional[List[int]] = None,
83
- softcap: float = 0.0,
84
- alibi_slopes: Optional[torch.Tensor] = None,
85
- block_table: Optional[torch.Tensor] = None,
86
-) -> torch.Tensor:
87
- return torch.empty_like(q)
88
-
89
-
90
-@torch.library.custom_op("vllm::flash_attn_with_kvcache", mutates_args=[])
91
-def flash_attn_with_kvcache(
92
- decode_query: torch.Tensor,
93
- key_cache: torch.Tensor,
94
- value_cache: torch.Tensor,
95
- cache_seqlens: Optional[torch.Tensor] = None,
96
- block_table: Optional[torch.Tensor] = None,
97
- softmax_scale: Optional[float] = None,
98
- causal: bool = False,
99
- alibi_slopes: Optional[torch.Tensor] = None,
100
- softcap: float = 0.0,
101
-) -> torch.Tensor:
102
- return _flash_attn_with_kvcache(
103
- decode_query,
104
- key_cache,
105
- value_cache,
106
- cache_seqlens=cache_seqlens,
107
- block_table=block_table,
108
- softmax_scale=softmax_scale,
109
- causal=causal,
110
- alibi_slopes=alibi_slopes,
111
- softcap=softcap,
112
- )
113
-
114
-
115
-@flash_attn_with_kvcache.register_fake # type: ignore
116
-def _(
117
- decode_query: torch.Tensor,
118
- key_cache: torch.Tensor,
119
- value_cache: torch.Tensor,
120
- cache_seqlens: Optional[torch.Tensor] = None,
121
- block_table: Optional[torch.Tensor] = None,
122
- softmax_scale: Optional[float] = None,
123
- causal: bool = False,
124
- alibi_slopes: Optional[torch.Tensor] = None,
125
- softcap: float = 0.0,
126
-) -> torch.Tensor:
127
- return torch.empty_like(decode_query)
128
-
129
-
130
-@torch.library.custom_op("vllm::reshape_and_cache_flash",
131
- mutates_args=["kv_cache"])
132
-def reshape_and_cache_flash(
133
- key: torch.Tensor,
134
- value: torch.Tensor,
135
- kv_cache: torch.Tensor,
136
- slot_mapping: torch.Tensor,
137
- kv_cache_dtype: str,
138
- k_scale: float,
139
- v_scale: float,
140
-) -> None:
141
- """Inductor cannot deal with inplace operations on views.
142
- See https://github.com/pytorch/pytorch/issues/131192
143
- and https://github.com/pytorch/pytorch/issues/130174
144
- This is a workaround to hide the view operation from the inductor.
145
- """
146
- return torch.ops._C_cache_ops.reshape_and_cache_flash(
147
- key, value, kv_cache[0], kv_cache[1], slot_mapping, kv_cache_dtype,
148
- k_scale, v_scale)
149
-
150
-
151
-@reshape_and_cache_flash.register_fake # type: ignore
152
-def _(
153
- key: torch.Tensor,
154
- value: torch.Tensor,
155
- kv_cache: torch.Tensor,
156
- slot_mapping: torch.Tensor,
157
- kv_cache_dtype: str,
158
- k_scale: float,
159
- v_scale: float,
160
-) -> None:
161
- pass
23
+from vllm.vllm_flash_attn import (flash_attn_varlen_func,
24
+ flash_attn_with_kvcache)
25
26
27
class FlashAttentionBackend(AttentionBackend):
@@ -721,118 +584,182 @@ class FlashAttentionImpl(AttentionImpl):
584
assert k_scale == 1.0 and v_scale == 1.0, (
585
"key/v_scale is not supported in FlashAttention.")
586
724
- num_tokens, hidden_size = query.shape
725
- # Reshape the query, key, and value tensors.
726
- query = query.view(-1, self.num_heads, self.head_size)
727
- key = key.view(-1, self.num_kv_heads, self.head_size)
728
- value = value.view(-1, self.num_kv_heads, self.head_size)
729
-
730
- if kv_cache.numel() > 0:
731
- key_cache = kv_cache[0]
732
- value_cache = kv_cache[1]
733
-
734
- # Reshape the input keys and values and store them in the cache.
735
- # If kv_cache is not provided, the new key and value tensors are
736
- # not cached. This happens during the initial memory profiling run.
737
- torch.ops.vllm.reshape_and_cache_flash(
738
- key,
739
- value,
740
- kv_cache,
741
- attn_metadata.slot_mapping.flatten(),
742
- self.kv_cache_dtype,
743
- k_scale,
744
- v_scale,
745
- )
587
+ output = torch.ops.vllm.unified_flash_attention(
588
+ query,
589
+ key,
590
+ value,
591
+ self.num_heads,
592
+ self.head_size,
593
+ self.num_kv_heads,
594
+ kv_cache,
595
+ self.kv_cache_dtype,
596
+ k_scale,
597
+ v_scale,
598
+ self.scale,
599
+ self.sliding_window,
600
+ self.alibi_slopes,
601
+ self.logits_soft_cap,
602
+ )
603
747
- num_prefill_tokens = attn_metadata.num_prefill_tokens
748
- num_decode_tokens = attn_metadata.num_decode_tokens
749
- assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
750
- f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
751
- assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
752
- f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
753
-
754
- # Query for decode. KV is not needed because it is already cached.
755
- decode_query = query[num_prefill_tokens:]
756
- # QKV for prefill.
757
- query = query[:num_prefill_tokens]
758
- key = key[:num_prefill_tokens]
759
- value = value[:num_prefill_tokens]
760
-
761
- assert query.shape[0] == num_prefill_tokens
762
- assert decode_query.shape[0] == num_decode_tokens
763
-
764
- prefill_output: Optional[torch.Tensor] = None
765
- decode_output: Optional[torch.Tensor] = None
766
-
767
- if prefill_meta := attn_metadata.prefill_metadata:
768
- # Prompt run.
769
- if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
770
- or prefill_meta.block_tables.numel() == 0):
771
- # normal attention
772
- # When block_tables are not filled, it means q and k are the
773
- # prompt, and they have the same length.
774
- prefill_output = torch.ops.vllm.flash_attn_varlen_func(
775
- q=query,
776
- k=key,
777
- v=value,
778
- cu_seqlens_q=prefill_meta.seq_start_loc,
779
- cu_seqlens_k=prefill_meta.seq_start_loc,
780
- max_seqlen_q=prefill_meta.max_prefill_seq_len,
781
- max_seqlen_k=prefill_meta.max_prefill_seq_len,
782
- softmax_scale=self.scale,
783
- causal=True,
784
- window_size=self.sliding_window,
785
- alibi_slopes=self.alibi_slopes,
786
- softcap=self.logits_soft_cap,
787
- )
788
- else:
789
- # prefix-enabled attention
790
- assert prefill_meta.seq_lens is not None
791
- max_seq_len = max(prefill_meta.seq_lens)
792
- prefill_output = torch.ops.vllm.flash_attn_varlen_func( # noqa
793
- q=query,
794
- k=key_cache,
795
- v=value_cache,
796
- cu_seqlens_q=prefill_meta.query_start_loc,
797
- max_seqlen_q=prefill_meta.max_query_len,
798
- cu_seqlens_k=prefill_meta.seq_start_loc,
799
- max_seqlen_k=max_seq_len,
800
- softmax_scale=self.scale,
801
- causal=True,
802
- alibi_slopes=self.alibi_slopes,
803
- block_table=prefill_meta.block_tables,
804
- softcap=self.logits_soft_cap,
805
- )
806
-
807
- if decode_meta := attn_metadata.decode_metadata:
808
- # Decoding run.
809
- _, num_head, head_dim = decode_query.shape
810
- decode_query = decode_query.reshape(-1,
811
- decode_meta.decode_query_len,
812
- num_head, head_dim)
813
- decode_output = torch.ops.vllm.flash_attn_with_kvcache(
814
- decode_query,
815
- key_cache,
816
- value_cache,
817
- block_table=decode_meta.block_tables,
818
- cache_seqlens=decode_meta.seq_lens_tensor,
819
- softmax_scale=self.scale,
604
+ return output
605
+
606
+
607
+@torch.library.custom_op("vllm::unified_flash_attention",
608
+ mutates_args=["kv_cache"])
609
+def unified_flash_attention(
610
+ query: torch.Tensor,
611
+ key: torch.Tensor,
612
+ value: torch.Tensor,
613
+ num_heads: int,
614
+ head_size: int,
615
+ num_kv_heads: int,
616
+ kv_cache: torch.Tensor,
617
+ kv_cache_dtype: str,
618
+ k_scale: float,
619
+ v_scale: float,
620
+ softmax_scale: float,
621
+ window_size: Optional[List[int]] = None,
622
+ alibi_slopes: Optional[torch.Tensor] = None,
623
+ logits_soft_cap: Optional[float] = None,
624
+) -> torch.Tensor:
625
+
626
+ current_metadata = get_forward_context()
627
+ assert current_metadata is not None
628
+ assert isinstance(current_metadata, FlashAttentionMetadata)
629
+ attn_metadata: FlashAttentionMetadata = current_metadata
630
+
631
+ num_tokens, hidden_size = query.shape
632
+ # Reshape the query, key, and value tensors.
633
+ query = query.view(-1, num_heads, head_size)
634
+ key = key.view(-1, num_kv_heads, head_size)
635
+ value = value.view(-1, num_kv_heads, head_size)
636
+
637
+ if kv_cache.numel() > 0:
638
+ key_cache = kv_cache[0]
639
+ value_cache = kv_cache[1]
640
+
641
+ # Reshape the input keys and values and store them in the cache.
642
+ # If kv_cache is not provided, the new key and value tensors are
643
+ # not cached. This happens during the initial memory profiling run.
644
+ torch.ops._C_cache_ops.reshape_and_cache_flash(
645
+ key,
646
+ value,
647
+ kv_cache[0],
648
+ kv_cache[1],
649
+ attn_metadata.slot_mapping.flatten(),
650
+ kv_cache_dtype,
651
+ k_scale,
652
+ v_scale,
653
+ )
654
+
655
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
656
+ num_decode_tokens = attn_metadata.num_decode_tokens
657
+ assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
658
+ f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
659
+ assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
660
+ f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
661
+
662
+ # Query for decode. KV is not needed because it is already cached.
663
+ decode_query = query[num_prefill_tokens:]
664
+ # QKV for prefill.
665
+ query = query[:num_prefill_tokens]
666
+ key = key[:num_prefill_tokens]
667
+ value = value[:num_prefill_tokens]
668
+
669
+ assert query.shape[0] == num_prefill_tokens
670
+ assert decode_query.shape[0] == num_decode_tokens
671
+
672
+ prefill_output: Optional[torch.Tensor] = None
673
+ decode_output: Optional[torch.Tensor] = None
674
+
675
+ if prefill_meta := attn_metadata.prefill_metadata:
676
+ # Prompt run.
677
+ if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
678
+ or prefill_meta.block_tables.numel() == 0):
679
+ # normal attention
680
+ # When block_tables are not filled, it means q and k are the
681
+ # prompt, and they have the same length.
682
+ prefill_output = flash_attn_varlen_func(
683
+ q=query,
684
+ k=key,
685
+ v=value,
686
+ cu_seqlens_q=prefill_meta.seq_start_loc,
687
+ cu_seqlens_k=prefill_meta.seq_start_loc,
688
+ max_seqlen_q=prefill_meta.max_prefill_seq_len,
689
+ max_seqlen_k=prefill_meta.max_prefill_seq_len,
690
+ softmax_scale=softmax_scale,
691
+ causal=True,
692
+ window_size=window_size,
693
+ alibi_slopes=alibi_slopes,
694
+ softcap=logits_soft_cap,
695
+ )
696
+ else:
697
+ # prefix-enabled attention
698
+ assert prefill_meta.seq_lens is not None
699
+ max_seq_len = max(prefill_meta.seq_lens)
700
+ prefill_output = flash_attn_varlen_func( # noqa
701
+ q=query,
702
+ k=key_cache,
703
+ v=value_cache,
704
+ cu_seqlens_q=prefill_meta.query_start_loc,
705
+ max_seqlen_q=prefill_meta.max_query_len,
706
+ cu_seqlens_k=prefill_meta.seq_start_loc,
707
+ max_seqlen_k=max_seq_len,
708
+ softmax_scale=softmax_scale,
709
causal=True,
821
- alibi_slopes=self.alibi_slopes,
822
- softcap=self.logits_soft_cap,
710
+ alibi_slopes=alibi_slopes,
711
+ block_table=prefill_meta.block_tables,
712
+ softcap=logits_soft_cap,
713
)
714
825
- if prefill_output is None:
826
- assert decode_output is not None
827
- return decode_output.view(num_decode_tokens, hidden_size)
828
- if decode_output is None:
829
- assert prefill_output is not None
830
- return prefill_output.view(num_prefill_tokens, hidden_size)
831
-
832
- # Chunked prefill does not work with speculative decoding.
833
- # Therefore, the query length for decode should be 1 in chunked prefill.
834
- assert decode_meta is not None
835
- assert decode_meta.decode_query_len == 1
836
- decode_output = decode_output.squeeze(1)
837
- output = torch.cat([prefill_output, decode_output], dim=0)
838
- return output.view(num_tokens, hidden_size)
715
+ if decode_meta := attn_metadata.decode_metadata:
716
+ # Decoding run.
717
+ _, num_head, head_dim = decode_query.shape
718
+ decode_query = decode_query.reshape(-1, decode_meta.decode_query_len,
719
+ num_head, head_dim)
720
+ decode_output = flash_attn_with_kvcache(
721
+ q=decode_query,
722
+ k_cache=key_cache,
723
+ v_cache=value_cache,
724
+ block_table=decode_meta.block_tables,
725
+ cache_seqlens=decode_meta.seq_lens_tensor,
726
+ softmax_scale=softmax_scale,
727
+ causal=True,
728
+ alibi_slopes=alibi_slopes,
729
+ softcap=logits_soft_cap,
730
+ ).squeeze(1)
731
+
732
+ if prefill_output is None:
733
+ assert decode_output is not None
734
+ return decode_output.view(num_decode_tokens, hidden_size)
735
+ if decode_output is None:
736
+ assert prefill_output is not None
737
+ return prefill_output.view(num_prefill_tokens, hidden_size)
738
+
739
+ # Chunked prefill does not work with speculative decoding.
740
+ # Therefore, the query length for decode should be 1 in chunked prefill.
741
+ assert decode_meta is not None
742
+ assert decode_meta.decode_query_len == 1
743
+ decode_output = decode_output.squeeze(1)
744
+ output = torch.cat([prefill_output, decode_output], dim=0)
745
+ return output.view(num_tokens, hidden_size)
746
+
747
+
748
+@unified_flash_attention.register_fake
749
+def _(
750
+ query: torch.Tensor,
751
+ key: torch.Tensor,
752
+ value: torch.Tensor,
753
+ num_heads: int,
754
+ head_size: int,
755
+ num_kv_heads: int,
756
+ kv_cache: torch.Tensor,
757
+ kv_cache_dtype: str,
758
+ k_scale: float,
759
+ v_scale: float,
760
+ softmax_scale: float,
761
+ window_size: Optional[List[int]] = None,
762
+ alibi_slopes: Optional[torch.Tensor] = None,
763
+ logits_soft_cap: Optional[float] = None,
764
+) -> torch.Tensor:
765
+ return torch.empty_like(query)
vllm/attention/backends/flashinfer.py
+2
-2
@@ -7,7 +7,7 @@ try:
7
from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
8
from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
9
10
- import vllm.attention.backends.flash_attn # noqa
10
+ from vllm.vllm_flash_attn import flash_attn_varlen_func
11
FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
12
except ImportError:
13
BatchDecodeWithPagedKVCacheWrapper = None
@@ -799,7 +799,7 @@ class FlashInferImpl(AttentionImpl):
799
# This happens when vllm runs the profiling to
800
# determine the number of blocks.
801
if kv_cache.numel() == 0:
802
- output = torch.ops.vllm.flash_attn_varlen_func(
802
+ output = flash_attn_varlen_func(
803
q=query,
804
k=key,
805
v=value,
vllm/forward_context.py
new
+22
@@ -0,0 +1,22 @@
1
+from contextlib import contextmanager
2
+from typing import Any
3
+
4
+_forward_context: Any = None
5
+
6
+
7
+def get_forward_context() -> Any:
8
+ """Get the current forward context."""
9
+ return _forward_context
10
+
11
+
12
+@contextmanager
13
+def set_forward_context(context: Any):
14
+ """A context manager that stores the current forward context,
15
+ can be attention metadata, etc."""
16
+ global _forward_context
17
+ prev_context = _forward_context
18
+ _forward_context = context
19
+ try:
20
+ yield
21
+ finally:
22
+ _forward_context = prev_context
vllm/spec_decode/draft_model_runner.py
+12
-10
@@ -2,6 +2,7 @@ from typing import List, Optional
2
3
import torch
4
5
+from vllm.forward_context import set_forward_context
6
from vllm.model_executor.layers.sampler import SamplerOutput
7
8
try:
@@ -291,16 +292,17 @@ class TP1DraftModelRunner(ModelRunner):
292
if previous_hidden_states is not None else {}
293
294
# Run model
294
- hidden_states = model_executable(
295
- input_ids=model_input.input_tokens,
296
- positions=model_input.input_positions,
297
- kv_caches=kv_caches,
298
- attn_metadata=model_input.attn_metadata,
299
- intermediate_tensors=intermediate_tensors,
300
- **MultiModalInputs.as_kwargs(multi_modal_kwargs,
301
- device=self.device),
302
- **kwargs,
303
- )
295
+ with set_forward_context(model_input.attn_metadata):
296
+ hidden_states = model_executable(
297
+ input_ids=model_input.input_tokens,
298
+ positions=model_input.input_positions,
299
+ kv_caches=kv_caches,
300
+ attn_metadata=model_input.attn_metadata,
301
+ intermediate_tensors=intermediate_tensors,
302
+ **MultiModalInputs.as_kwargs(multi_modal_kwargs,
303
+ device=self.device),
304
+ **kwargs,
305
+ )
306
307
# Compute the logits.
308
logits = self.model.compute_logits(hidden_states,
vllm/worker/embedding_model_runner.py
+3
-1
@@ -6,6 +6,7 @@ import torch
6
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
ModelConfig, ObservabilityConfig, ParallelConfig,
8
PromptAdapterConfig, SchedulerConfig)
9
+from vllm.forward_context import set_forward_context
10
from vllm.logger import init_logger
11
from vllm.model_executor.pooling_metadata import PoolingMetadata
12
from vllm.multimodal import MultiModalInputs
@@ -119,7 +120,8 @@ class EmbeddingModelRunner(
120
device=self.device),
121
}
122
122
- hidden_states = model_executable(**execute_model_kwargs)
123
+ with set_forward_context(model_input.attn_metadata):
124
+ hidden_states = model_executable(**execute_model_kwargs)
125
126
# Only perform pooling in the driver worker.
127
if not self.is_driver_worker:
vllm/worker/enc_dec_model_runner.py
+13
-11
@@ -14,6 +14,7 @@ from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
14
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15
ModelConfig, ObservabilityConfig, ParallelConfig,
16
PromptAdapterConfig, SchedulerConfig)
17
+from vllm.forward_context import set_forward_context
18
from vllm.inputs import INPUT_REGISTRY, InputRegistry
19
from vllm.logger import init_logger
20
from vllm.model_executor import SamplingMetadata
@@ -198,17 +199,18 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
199
} if self.has_seqlen_agnostic else {}
200
201
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
201
- hidden_or_intermediate_states = model_executable(
202
- input_ids=model_input.input_tokens,
203
- positions=model_input.input_positions,
204
- encoder_input_ids=model_input.encoder_input_tokens,
205
- encoder_positions=model_input.encoder_input_positions,
206
- kv_caches=kv_caches,
207
- attn_metadata=model_input.attn_metadata,
208
- intermediate_tensors=intermediate_tensors,
209
- **MultiModalInputs.as_kwargs(multi_modal_kwargs,
210
- device=self.device),
211
- **seqlen_agnostic_kwargs)
202
+ with set_forward_context(model_input.attn_metadata):
203
+ hidden_or_intermediate_states = model_executable(
204
+ input_ids=model_input.input_tokens,
205
+ positions=model_input.input_positions,
206
+ encoder_input_ids=model_input.encoder_input_tokens,
207
+ encoder_positions=model_input.encoder_input_positions,
208
+ kv_caches=kv_caches,
209
+ attn_metadata=model_input.attn_metadata,
210
+ intermediate_tensors=intermediate_tensors,
211
+ **MultiModalInputs.as_kwargs(multi_modal_kwargs,
212
+ device=self.device),
213
+ **seqlen_agnostic_kwargs)
214
215
logits = self.model.compute_logits(hidden_or_intermediate_states,
216
model_input.sampling_metadata)
vllm/worker/model_runner.py
+13
-10
@@ -24,6 +24,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
24
from vllm.core.scheduler import SchedulerOutputs
25
from vllm.distributed import get_pp_group
26
from vllm.distributed.parallel_state import graph_capture
27
+from vllm.forward_context import set_forward_context
28
from vllm.inputs import INPUT_REGISTRY, InputRegistry
29
from vllm.logger import init_logger
30
from vllm.lora.layers import LoRAMapping
@@ -1499,7 +1500,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1500
self._update_inputs_to_capture_for_enc_dec_model(
1501
capture_inputs)
1502
1502
- graph_runner.capture(**capture_inputs)
1503
+ with set_forward_context(attn_metadata):
1504
+ graph_runner.capture(**capture_inputs)
1505
self.graph_memory_pool = graph_runner.graph.pool()
1506
self.graph_runners[virtual_engine][batch_size] = (
1507
graph_runner)
@@ -1641,15 +1643,16 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1643
model_forward_end = torch.cuda.Event(enable_timing=True)
1644
model_forward_start.record()
1645
1644
- hidden_or_intermediate_states = model_executable(
1645
- input_ids=model_input.input_tokens,
1646
- positions=model_input.input_positions,
1647
- kv_caches=kv_caches,
1648
- attn_metadata=model_input.attn_metadata,
1649
- intermediate_tensors=intermediate_tensors,
1650
- **MultiModalInputs.as_kwargs(multi_modal_kwargs,
1651
- device=self.device),
1652
- **seqlen_agnostic_kwargs)
1646
+ with set_forward_context(model_input.attn_metadata):
1647
+ hidden_or_intermediate_states = model_executable(
1648
+ input_ids=model_input.input_tokens,
1649
+ positions=model_input.input_positions,
1650
+ kv_caches=kv_caches,
1651
+ attn_metadata=model_input.attn_metadata,
1652
+ intermediate_tensors=intermediate_tensors,
1653
+ **MultiModalInputs.as_kwargs(multi_modal_kwargs,
1654
+ device=self.device),
1655
+ **seqlen_agnostic_kwargs)
1656
1657
if (self.observability_config is not None
1658
and self.observability_config.collect_model_forward_time):