@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9aaf14c62e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[misc] add forward context for attention (#9029)

youkaichao committed Oct 3, 2024 at 12:09 UTC 9aaf14c62e16a7c74b5192a44d01a78125dab2fc
8 files changed +250 -334
tests/kernels/test_flash_attn.py
+7 -49
@@ -3,9 +3,9 @@ from typing import List, Optional, Tuple
3 import pytest
4 import torch
5
6 -import vllm.attention.backends.flash_attn # noqa: F401
7 -from tests.kernels.utils import opcheck
6 from vllm.utils import seed_everything
7 +from vllm.vllm_flash_attn import (flash_attn_varlen_func,
8 + flash_attn_with_kvcache)
9
10 NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
11 HEAD_SIZES = [128, 256]
@@ -112,10 +112,10 @@ def test_flash_attn_with_paged_kv(
112 (num_seqs, max_num_blocks_per_seq),
113 dtype=torch.int32)
114
115 - output = torch.ops.vllm.flash_attn_with_kvcache(
116 - decode_query=query.unsqueeze(1),
117 - key_cache=key_cache,
118 - value_cache=value_cache,
115 + output = flash_attn_with_kvcache(
116 + q=query.unsqueeze(1),
117 + k_cache=key_cache,
118 + v_cache=value_cache,
119 softmax_scale=scale,
120 causal=True,
121 block_table=block_tables,
@@ -123,25 +123,6 @@ def test_flash_attn_with_paged_kv(
123 softcap=soft_cap if soft_cap is not None else 0,
124 ).squeeze(1)
125
126 - if num_blocks <= 2048:
127 - test_utils = ["test_faketensor", "test_schema"]
128 - else:
129 - test_utils = ["test_faketensor"]
130 -
131 - opcheck(torch.ops.vllm.flash_attn_with_kvcache,
132 - args=tuple(),
133 - kwargs=dict(
134 - decode_query=query.unsqueeze(1),
135 - key_cache=key_cache,
136 - value_cache=value_cache,
137 - softmax_scale=scale,
138 - causal=True,
139 - block_table=block_tables,
140 - cache_seqlens=kv_lens_tensor,
141 - softcap=soft_cap if soft_cap is not None else 0,
142 - ),
143 - test_utils=test_utils)
144 -
126 ref_output = ref_paged_attn(
127 query=query,
128 key_cache=key_cache,
@@ -213,7 +194,7 @@ def test_varlen_with_paged_kv(
194 (num_seqs, max_num_blocks_per_seq),
195 dtype=torch.int32)
196
216 - output = torch.ops.vllm.flash_attn_varlen_func(
197 + output = flash_attn_varlen_func(
198 q=query,
199 k=key_cache,
200 v=value_cache,
@@ -228,29 +209,6 @@ def test_varlen_with_paged_kv(
209 softcap=soft_cap if soft_cap is not None else 0,
210 )
211
231 - if num_blocks <= 2048:
232 - test_utils = ["test_faketensor", "test_schema"]
233 - else:
234 - test_utils = ["test_faketensor"]
235 -
236 - opcheck(torch.ops.vllm.flash_attn_varlen_func,
237 - args=tuple(),
238 - kwargs=dict(
239 - q=query,
240 - k=key_cache,
241 - v=value_cache,
242 - cu_seqlens_q=cu_query_lens,
243 - cu_seqlens_k=cu_kv_lens,
244 - max_seqlen_q=max_query_len,
245 - max_seqlen_k=max_kv_len,
246 - softmax_scale=scale,
247 - causal=True,
248 - window_size=window_size,
249 - block_table=block_tables,
250 - softcap=soft_cap if soft_cap is not None else 0,
251 - ),
252 - test_utils=test_utils)
253 -
212 ref_output = ref_paged_attn(
213 query=query,
214 key_cache=key_cache,
vllm/attention/backends/flash_attn.py
+178 -251
@@ -13,152 +13,15 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
13 compute_slot_mapping,
14 compute_slot_mapping_start_idx,
15 is_block_tables_empty)
16 +from vllm.forward_context import get_forward_context
17 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
18
19 if TYPE_CHECKING:
20 from vllm.worker.model_runner import (ModelInputForGPUBuilder,
21 ModelInputForGPUWithSamplingMetadata)
22
22 -# yapf: disable
23 -from vllm.vllm_flash_attn import (
24 - flash_attn_varlen_func as _flash_attn_varlen_func)
25 -from vllm.vllm_flash_attn import (
26 - flash_attn_with_kvcache as _flash_attn_with_kvcache)
27 -
28 -# yapf: enable
29 -
30 -
31 -@torch.library.custom_op("vllm::flash_attn_varlen_func", mutates_args=[])
32 -def flash_attn_varlen_func(
33 - q: torch.Tensor,
34 - k: torch.Tensor,
35 - v: torch.Tensor,
36 - cu_seqlens_q: torch.Tensor,
37 - cu_seqlens_k: torch.Tensor,
38 - max_seqlen_q: int,
39 - max_seqlen_k: int,
40 - softmax_scale: Optional[float] = None,
41 - causal: bool = False,
42 - window_size: Optional[List[int]] = None,
43 - softcap: float = 0.0,
44 - alibi_slopes: Optional[torch.Tensor] = None,
45 - block_table: Optional[torch.Tensor] = None,
46 -) -> torch.Tensor:
47 - # custom op does not support tuple input
48 - real_window_size: Tuple[int, int]
49 - if window_size is None:
50 - real_window_size = (-1, -1)
51 - else:
52 - assert len(window_size) == 2
53 - real_window_size = (window_size[0], window_size[1])
54 - return _flash_attn_varlen_func(
55 - q=q,
56 - k=k,
57 - v=v,
58 - cu_seqlens_q=cu_seqlens_q,
59 - cu_seqlens_k=cu_seqlens_k,
60 - max_seqlen_q=max_seqlen_q,
61 - max_seqlen_k=max_seqlen_k,
62 - softmax_scale=softmax_scale,
63 - causal=causal,
64 - window_size=real_window_size,
65 - softcap=softcap,
66 - alibi_slopes=alibi_slopes,
67 - block_table=block_table,
68 - )
69 -
70 -
71 -@flash_attn_varlen_func.register_fake # type: ignore
72 -def _(
73 - q: torch.Tensor,
74 - k: torch.Tensor,
75 - v: torch.Tensor,
76 - cu_seqlens_q: torch.Tensor,
77 - cu_seqlens_k: torch.Tensor,
78 - max_seqlen_q: int,
79 - max_seqlen_k: int,
80 - softmax_scale: Optional[float] = None,
81 - causal: bool = False,
82 - window_size: Optional[List[int]] = None,
83 - softcap: float = 0.0,
84 - alibi_slopes: Optional[torch.Tensor] = None,
85 - block_table: Optional[torch.Tensor] = None,
86 -) -> torch.Tensor:
87 - return torch.empty_like(q)
88 -
89 -
90 -@torch.library.custom_op("vllm::flash_attn_with_kvcache", mutates_args=[])
91 -def flash_attn_with_kvcache(
92 - decode_query: torch.Tensor,
93 - key_cache: torch.Tensor,
94 - value_cache: torch.Tensor,
95 - cache_seqlens: Optional[torch.Tensor] = None,
96 - block_table: Optional[torch.Tensor] = None,
97 - softmax_scale: Optional[float] = None,
98 - causal: bool = False,
99 - alibi_slopes: Optional[torch.Tensor] = None,
100 - softcap: float = 0.0,
101 -) -> torch.Tensor:
102 - return _flash_attn_with_kvcache(
103 - decode_query,
104 - key_cache,
105 - value_cache,
106 - cache_seqlens=cache_seqlens,
107 - block_table=block_table,
108 - softmax_scale=softmax_scale,
109 - causal=causal,
110 - alibi_slopes=alibi_slopes,
111 - softcap=softcap,
112 - )
113 -
114 -
115 -@flash_attn_with_kvcache.register_fake # type: ignore
116 -def _(
117 - decode_query: torch.Tensor,
118 - key_cache: torch.Tensor,
119 - value_cache: torch.Tensor,
120 - cache_seqlens: Optional[torch.Tensor] = None,
121 - block_table: Optional[torch.Tensor] = None,
122 - softmax_scale: Optional[float] = None,
123 - causal: bool = False,
124 - alibi_slopes: Optional[torch.Tensor] = None,
125 - softcap: float = 0.0,
126 -) -> torch.Tensor:
127 - return torch.empty_like(decode_query)
128 -
129 -
130 -@torch.library.custom_op("vllm::reshape_and_cache_flash",
131 - mutates_args=["kv_cache"])
132 -def reshape_and_cache_flash(
133 - key: torch.Tensor,
134 - value: torch.Tensor,
135 - kv_cache: torch.Tensor,
136 - slot_mapping: torch.Tensor,
137 - kv_cache_dtype: str,
138 - k_scale: float,
139 - v_scale: float,
140 -) -> None:
141 - """Inductor cannot deal with inplace operations on views.
142 - See https://github.com/pytorch/pytorch/issues/131192
143 - and https://github.com/pytorch/pytorch/issues/130174
144 - This is a workaround to hide the view operation from the inductor.
145 - """
146 - return torch.ops._C_cache_ops.reshape_and_cache_flash(
147 - key, value, kv_cache[0], kv_cache[1], slot_mapping, kv_cache_dtype,
148 - k_scale, v_scale)
149 -
150 -
151 -@reshape_and_cache_flash.register_fake # type: ignore
152 -def _(
153 - key: torch.Tensor,
154 - value: torch.Tensor,
155 - kv_cache: torch.Tensor,
156 - slot_mapping: torch.Tensor,
157 - kv_cache_dtype: str,
158 - k_scale: float,
159 - v_scale: float,
160 -) -> None:
161 - pass
23 +from vllm.vllm_flash_attn import (flash_attn_varlen_func,
24 + flash_attn_with_kvcache)
25
26
27 class FlashAttentionBackend(AttentionBackend):
@@ -721,118 +584,182 @@ class FlashAttentionImpl(AttentionImpl):
584 assert k_scale == 1.0 and v_scale == 1.0, (
585 "key/v_scale is not supported in FlashAttention.")
586
724 - num_tokens, hidden_size = query.shape
725 - # Reshape the query, key, and value tensors.
726 - query = query.view(-1, self.num_heads, self.head_size)
727 - key = key.view(-1, self.num_kv_heads, self.head_size)
728 - value = value.view(-1, self.num_kv_heads, self.head_size)
729 -
730 - if kv_cache.numel() > 0:
731 - key_cache = kv_cache[0]
732 - value_cache = kv_cache[1]
733 -
734 - # Reshape the input keys and values and store them in the cache.
735 - # If kv_cache is not provided, the new key and value tensors are
736 - # not cached. This happens during the initial memory profiling run.
737 - torch.ops.vllm.reshape_and_cache_flash(
738 - key,
739 - value,
740 - kv_cache,
741 - attn_metadata.slot_mapping.flatten(),
742 - self.kv_cache_dtype,
743 - k_scale,
744 - v_scale,
745 - )
587 + output = torch.ops.vllm.unified_flash_attention(
588 + query,
589 + key,
590 + value,
591 + self.num_heads,
592 + self.head_size,
593 + self.num_kv_heads,
594 + kv_cache,
595 + self.kv_cache_dtype,
596 + k_scale,
597 + v_scale,
598 + self.scale,
599 + self.sliding_window,
600 + self.alibi_slopes,
601 + self.logits_soft_cap,
602 + )
603
747 - num_prefill_tokens = attn_metadata.num_prefill_tokens
748 - num_decode_tokens = attn_metadata.num_decode_tokens
749 - assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
750 - f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
751 - assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
752 - f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
753 -
754 - # Query for decode. KV is not needed because it is already cached.
755 - decode_query = query[num_prefill_tokens:]
756 - # QKV for prefill.
757 - query = query[:num_prefill_tokens]
758 - key = key[:num_prefill_tokens]
759 - value = value[:num_prefill_tokens]
760 -
761 - assert query.shape[0] == num_prefill_tokens
762 - assert decode_query.shape[0] == num_decode_tokens
763 -
764 - prefill_output: Optional[torch.Tensor] = None
765 - decode_output: Optional[torch.Tensor] = None
766 -
767 - if prefill_meta := attn_metadata.prefill_metadata:
768 - # Prompt run.
769 - if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
770 - or prefill_meta.block_tables.numel() == 0):
771 - # normal attention
772 - # When block_tables are not filled, it means q and k are the
773 - # prompt, and they have the same length.
774 - prefill_output = torch.ops.vllm.flash_attn_varlen_func(
775 - q=query,
776 - k=key,
777 - v=value,
778 - cu_seqlens_q=prefill_meta.seq_start_loc,
779 - cu_seqlens_k=prefill_meta.seq_start_loc,
780 - max_seqlen_q=prefill_meta.max_prefill_seq_len,
781 - max_seqlen_k=prefill_meta.max_prefill_seq_len,
782 - softmax_scale=self.scale,
783 - causal=True,
784 - window_size=self.sliding_window,
785 - alibi_slopes=self.alibi_slopes,
786 - softcap=self.logits_soft_cap,
787 - )
788 - else:
789 - # prefix-enabled attention
790 - assert prefill_meta.seq_lens is not None
791 - max_seq_len = max(prefill_meta.seq_lens)
792 - prefill_output = torch.ops.vllm.flash_attn_varlen_func( # noqa
793 - q=query,
794 - k=key_cache,
795 - v=value_cache,
796 - cu_seqlens_q=prefill_meta.query_start_loc,
797 - max_seqlen_q=prefill_meta.max_query_len,
798 - cu_seqlens_k=prefill_meta.seq_start_loc,
799 - max_seqlen_k=max_seq_len,
800 - softmax_scale=self.scale,
801 - causal=True,
802 - alibi_slopes=self.alibi_slopes,
803 - block_table=prefill_meta.block_tables,
804 - softcap=self.logits_soft_cap,
805 - )
806 -
807 - if decode_meta := attn_metadata.decode_metadata:
808 - # Decoding run.
809 - _, num_head, head_dim = decode_query.shape
810 - decode_query = decode_query.reshape(-1,
811 - decode_meta.decode_query_len,
812 - num_head, head_dim)
813 - decode_output = torch.ops.vllm.flash_attn_with_kvcache(
814 - decode_query,
815 - key_cache,
816 - value_cache,
817 - block_table=decode_meta.block_tables,
818 - cache_seqlens=decode_meta.seq_lens_tensor,
819 - softmax_scale=self.scale,
604 + return output
605 +
606 +
607 +@torch.library.custom_op("vllm::unified_flash_attention",
608 + mutates_args=["kv_cache"])
609 +def unified_flash_attention(
610 + query: torch.Tensor,
611 + key: torch.Tensor,
612 + value: torch.Tensor,
613 + num_heads: int,
614 + head_size: int,
615 + num_kv_heads: int,
616 + kv_cache: torch.Tensor,
617 + kv_cache_dtype: str,
618 + k_scale: float,
619 + v_scale: float,
620 + softmax_scale: float,
621 + window_size: Optional[List[int]] = None,
622 + alibi_slopes: Optional[torch.Tensor] = None,
623 + logits_soft_cap: Optional[float] = None,
624 +) -> torch.Tensor:
625 +
626 + current_metadata = get_forward_context()
627 + assert current_metadata is not None
628 + assert isinstance(current_metadata, FlashAttentionMetadata)
629 + attn_metadata: FlashAttentionMetadata = current_metadata
630 +
631 + num_tokens, hidden_size = query.shape
632 + # Reshape the query, key, and value tensors.
633 + query = query.view(-1, num_heads, head_size)
634 + key = key.view(-1, num_kv_heads, head_size)
635 + value = value.view(-1, num_kv_heads, head_size)
636 +
637 + if kv_cache.numel() > 0:
638 + key_cache = kv_cache[0]
639 + value_cache = kv_cache[1]
640 +
641 + # Reshape the input keys and values and store them in the cache.
642 + # If kv_cache is not provided, the new key and value tensors are
643 + # not cached. This happens during the initial memory profiling run.
644 + torch.ops._C_cache_ops.reshape_and_cache_flash(
645 + key,
646 + value,
647 + kv_cache[0],
648 + kv_cache[1],
649 + attn_metadata.slot_mapping.flatten(),
650 + kv_cache_dtype,
651 + k_scale,
652 + v_scale,
653 + )
654 +
655 + num_prefill_tokens = attn_metadata.num_prefill_tokens
656 + num_decode_tokens = attn_metadata.num_decode_tokens
657 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
658 + f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
659 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
660 + f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
661 +
662 + # Query for decode. KV is not needed because it is already cached.
663 + decode_query = query[num_prefill_tokens:]
664 + # QKV for prefill.
665 + query = query[:num_prefill_tokens]
666 + key = key[:num_prefill_tokens]
667 + value = value[:num_prefill_tokens]
668 +
669 + assert query.shape[0] == num_prefill_tokens
670 + assert decode_query.shape[0] == num_decode_tokens
671 +
672 + prefill_output: Optional[torch.Tensor] = None
673 + decode_output: Optional[torch.Tensor] = None
674 +
675 + if prefill_meta := attn_metadata.prefill_metadata:
676 + # Prompt run.
677 + if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
678 + or prefill_meta.block_tables.numel() == 0):
679 + # normal attention
680 + # When block_tables are not filled, it means q and k are the
681 + # prompt, and they have the same length.
682 + prefill_output = flash_attn_varlen_func(
683 + q=query,
684 + k=key,
685 + v=value,
686 + cu_seqlens_q=prefill_meta.seq_start_loc,
687 + cu_seqlens_k=prefill_meta.seq_start_loc,
688 + max_seqlen_q=prefill_meta.max_prefill_seq_len,
689 + max_seqlen_k=prefill_meta.max_prefill_seq_len,
690 + softmax_scale=softmax_scale,
691 + causal=True,
692 + window_size=window_size,
693 + alibi_slopes=alibi_slopes,
694 + softcap=logits_soft_cap,
695 + )
696 + else:
697 + # prefix-enabled attention
698 + assert prefill_meta.seq_lens is not None
699 + max_seq_len = max(prefill_meta.seq_lens)
700 + prefill_output = flash_attn_varlen_func( # noqa
701 + q=query,
702 + k=key_cache,
703 + v=value_cache,
704 + cu_seqlens_q=prefill_meta.query_start_loc,
705 + max_seqlen_q=prefill_meta.max_query_len,
706 + cu_seqlens_k=prefill_meta.seq_start_loc,
707 + max_seqlen_k=max_seq_len,
708 + softmax_scale=softmax_scale,
709 causal=True,
821 - alibi_slopes=self.alibi_slopes,
822 - softcap=self.logits_soft_cap,
710 + alibi_slopes=alibi_slopes,
711 + block_table=prefill_meta.block_tables,
712 + softcap=logits_soft_cap,
713 )
714
825 - if prefill_output is None:
826 - assert decode_output is not None
827 - return decode_output.view(num_decode_tokens, hidden_size)
828 - if decode_output is None:
829 - assert prefill_output is not None
830 - return prefill_output.view(num_prefill_tokens, hidden_size)
831 -
832 - # Chunked prefill does not work with speculative decoding.
833 - # Therefore, the query length for decode should be 1 in chunked prefill.
834 - assert decode_meta is not None
835 - assert decode_meta.decode_query_len == 1
836 - decode_output = decode_output.squeeze(1)
837 - output = torch.cat([prefill_output, decode_output], dim=0)
838 - return output.view(num_tokens, hidden_size)
715 + if decode_meta := attn_metadata.decode_metadata:
716 + # Decoding run.
717 + _, num_head, head_dim = decode_query.shape
718 + decode_query = decode_query.reshape(-1, decode_meta.decode_query_len,
719 + num_head, head_dim)
720 + decode_output = flash_attn_with_kvcache(
721 + q=decode_query,
722 + k_cache=key_cache,
723 + v_cache=value_cache,
724 + block_table=decode_meta.block_tables,
725 + cache_seqlens=decode_meta.seq_lens_tensor,
726 + softmax_scale=softmax_scale,
727 + causal=True,
728 + alibi_slopes=alibi_slopes,
729 + softcap=logits_soft_cap,
730 + ).squeeze(1)
731 +
732 + if prefill_output is None:
733 + assert decode_output is not None
734 + return decode_output.view(num_decode_tokens, hidden_size)
735 + if decode_output is None:
736 + assert prefill_output is not None
737 + return prefill_output.view(num_prefill_tokens, hidden_size)
738 +
739 + # Chunked prefill does not work with speculative decoding.
740 + # Therefore, the query length for decode should be 1 in chunked prefill.
741 + assert decode_meta is not None
742 + assert decode_meta.decode_query_len == 1
743 + decode_output = decode_output.squeeze(1)
744 + output = torch.cat([prefill_output, decode_output], dim=0)
745 + return output.view(num_tokens, hidden_size)
746 +
747 +
748 +@unified_flash_attention.register_fake
749 +def _(
750 + query: torch.Tensor,
751 + key: torch.Tensor,
752 + value: torch.Tensor,
753 + num_heads: int,
754 + head_size: int,
755 + num_kv_heads: int,
756 + kv_cache: torch.Tensor,
757 + kv_cache_dtype: str,
758 + k_scale: float,
759 + v_scale: float,
760 + softmax_scale: float,
761 + window_size: Optional[List[int]] = None,
762 + alibi_slopes: Optional[torch.Tensor] = None,
763 + logits_soft_cap: Optional[float] = None,
764 +) -> torch.Tensor:
765 + return torch.empty_like(query)
vllm/attention/backends/flashinfer.py
+2 -2
@@ -7,7 +7,7 @@ try:
7 from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
8 from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
9
10 - import vllm.attention.backends.flash_attn # noqa
10 + from vllm.vllm_flash_attn import flash_attn_varlen_func
11 FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
12 except ImportError:
13 BatchDecodeWithPagedKVCacheWrapper = None
@@ -799,7 +799,7 @@ class FlashInferImpl(AttentionImpl):
799 # This happens when vllm runs the profiling to
800 # determine the number of blocks.
801 if kv_cache.numel() == 0:
802 - output = torch.ops.vllm.flash_attn_varlen_func(
802 + output = flash_attn_varlen_func(
803 q=query,
804 k=key,
805 v=value,
vllm/forward_context.py new
+22
@@ -0,0 +1,22 @@
1 +from contextlib import contextmanager
2 +from typing import Any
3 +
4 +_forward_context: Any = None
5 +
6 +
7 +def get_forward_context() -> Any:
8 + """Get the current forward context."""
9 + return _forward_context
10 +
11 +
12 +@contextmanager
13 +def set_forward_context(context: Any):
14 + """A context manager that stores the current forward context,
15 + can be attention metadata, etc."""
16 + global _forward_context
17 + prev_context = _forward_context
18 + _forward_context = context
19 + try:
20 + yield
21 + finally:
22 + _forward_context = prev_context
vllm/spec_decode/draft_model_runner.py
+12 -10
@@ -2,6 +2,7 @@ from typing import List, Optional
2
3 import torch
4
5 +from vllm.forward_context import set_forward_context
6 from vllm.model_executor.layers.sampler import SamplerOutput
7
8 try:
@@ -291,16 +292,17 @@ class TP1DraftModelRunner(ModelRunner):
292 if previous_hidden_states is not None else {}
293
294 # Run model
294 - hidden_states = model_executable(
295 - input_ids=model_input.input_tokens,
296 - positions=model_input.input_positions,
297 - kv_caches=kv_caches,
298 - attn_metadata=model_input.attn_metadata,
299 - intermediate_tensors=intermediate_tensors,
300 - **MultiModalInputs.as_kwargs(multi_modal_kwargs,
301 - device=self.device),
302 - **kwargs,
303 - )
295 + with set_forward_context(model_input.attn_metadata):
296 + hidden_states = model_executable(
297 + input_ids=model_input.input_tokens,
298 + positions=model_input.input_positions,
299 + kv_caches=kv_caches,
300 + attn_metadata=model_input.attn_metadata,
301 + intermediate_tensors=intermediate_tensors,
302 + **MultiModalInputs.as_kwargs(multi_modal_kwargs,
303 + device=self.device),
304 + **kwargs,
305 + )
306
307 # Compute the logits.
308 logits = self.model.compute_logits(hidden_states,
vllm/worker/embedding_model_runner.py
+3 -1
@@ -6,6 +6,7 @@ import torch
6 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 ModelConfig, ObservabilityConfig, ParallelConfig,
8 PromptAdapterConfig, SchedulerConfig)
9 +from vllm.forward_context import set_forward_context
10 from vllm.logger import init_logger
11 from vllm.model_executor.pooling_metadata import PoolingMetadata
12 from vllm.multimodal import MultiModalInputs
@@ -119,7 +120,8 @@ class EmbeddingModelRunner(
120 device=self.device),
121 }
122
122 - hidden_states = model_executable(**execute_model_kwargs)
123 + with set_forward_context(model_input.attn_metadata):
124 + hidden_states = model_executable(**execute_model_kwargs)
125
126 # Only perform pooling in the driver worker.
127 if not self.is_driver_worker:
vllm/worker/enc_dec_model_runner.py
+13 -11
@@ -14,6 +14,7 @@ from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
14 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15 ModelConfig, ObservabilityConfig, ParallelConfig,
16 PromptAdapterConfig, SchedulerConfig)
17 +from vllm.forward_context import set_forward_context
18 from vllm.inputs import INPUT_REGISTRY, InputRegistry
19 from vllm.logger import init_logger
20 from vllm.model_executor import SamplingMetadata
@@ -198,17 +199,18 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
199 } if self.has_seqlen_agnostic else {}
200
201 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
201 - hidden_or_intermediate_states = model_executable(
202 - input_ids=model_input.input_tokens,
203 - positions=model_input.input_positions,
204 - encoder_input_ids=model_input.encoder_input_tokens,
205 - encoder_positions=model_input.encoder_input_positions,
206 - kv_caches=kv_caches,
207 - attn_metadata=model_input.attn_metadata,
208 - intermediate_tensors=intermediate_tensors,
209 - **MultiModalInputs.as_kwargs(multi_modal_kwargs,
210 - device=self.device),
211 - **seqlen_agnostic_kwargs)
202 + with set_forward_context(model_input.attn_metadata):
203 + hidden_or_intermediate_states = model_executable(
204 + input_ids=model_input.input_tokens,
205 + positions=model_input.input_positions,
206 + encoder_input_ids=model_input.encoder_input_tokens,
207 + encoder_positions=model_input.encoder_input_positions,
208 + kv_caches=kv_caches,
209 + attn_metadata=model_input.attn_metadata,
210 + intermediate_tensors=intermediate_tensors,
211 + **MultiModalInputs.as_kwargs(multi_modal_kwargs,
212 + device=self.device),
213 + **seqlen_agnostic_kwargs)
214
215 logits = self.model.compute_logits(hidden_or_intermediate_states,
216 model_input.sampling_metadata)
vllm/worker/model_runner.py
+13 -10
@@ -24,6 +24,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
24 from vllm.core.scheduler import SchedulerOutputs
25 from vllm.distributed import get_pp_group
26 from vllm.distributed.parallel_state import graph_capture
27 +from vllm.forward_context import set_forward_context
28 from vllm.inputs import INPUT_REGISTRY, InputRegistry
29 from vllm.logger import init_logger
30 from vllm.lora.layers import LoRAMapping
@@ -1499,7 +1500,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1500 self._update_inputs_to_capture_for_enc_dec_model(
1501 capture_inputs)
1502
1502 - graph_runner.capture(**capture_inputs)
1503 + with set_forward_context(attn_metadata):
1504 + graph_runner.capture(**capture_inputs)
1505 self.graph_memory_pool = graph_runner.graph.pool()
1506 self.graph_runners[virtual_engine][batch_size] = (
1507 graph_runner)
@@ -1641,15 +1643,16 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1643 model_forward_end = torch.cuda.Event(enable_timing=True)
1644 model_forward_start.record()
1645
1644 - hidden_or_intermediate_states = model_executable(
1645 - input_ids=model_input.input_tokens,
1646 - positions=model_input.input_positions,
1647 - kv_caches=kv_caches,
1648 - attn_metadata=model_input.attn_metadata,
1649 - intermediate_tensors=intermediate_tensors,
1650 - **MultiModalInputs.as_kwargs(multi_modal_kwargs,
1651 - device=self.device),
1652 - **seqlen_agnostic_kwargs)
1646 + with set_forward_context(model_input.attn_metadata):
1647 + hidden_or_intermediate_states = model_executable(
1648 + input_ids=model_input.input_tokens,
1649 + positions=model_input.input_positions,
1650 + kv_caches=kv_caches,
1651 + attn_metadata=model_input.attn_metadata,
1652 + intermediate_tensors=intermediate_tensors,
1653 + **MultiModalInputs.as_kwargs(multi_modal_kwargs,
1654 + device=self.device),
1655 + **seqlen_agnostic_kwargs)
1656
1657 if (self.observability_config is not None
1658 and self.observability_config.collect_model_forward_time):