@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4c3aac51e1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Merging PR #12536

Merged via CLI script

Chen Zhang committed Feb 6, 2025 at 05:24 UTC 4c3aac51e14214880a3205b45c26aa16535992b5
1 file changed +22 -6
vllm/attention/layer.py
+22 -6
@@ -156,9 +156,13 @@ class Attention(nn.Module):
156 kv_cache: torch.Tensor,
157 attn_metadata: AttentionMetadata,
158 ) -> torch.Tensor:
159 - if self.calculate_kv_scales and \
160 - attn_metadata.enable_kv_scales_calculation:
161 - self.calc_kv_scales(key, value)
159 + # NOTE: please avoid accessing `kv_cache` and `attn_metadata` arguments
160 + # directly, use `self.kv_cache` and
161 + # `get_forward_context().attn_metadata` instead.
162 + if self.calculate_kv_scales:
163 + ctx_attn_metadata = get_forward_context().attn_metadata
164 + if ctx_attn_metadata.enable_kv_scales_calculation:
165 + self.calc_kv_scales(key, value)
166 if self.use_output:
167 output = torch.empty_like(query)
168 hidden_size = query.size(-1)
@@ -172,15 +176,27 @@ class Attention(nn.Module):
176 if value is not None:
177 value = value.view(-1, self.num_kv_heads, self.head_size)
178 if self.use_direct_call:
175 - unified_attention_with_output(query, key, value, output,
176 - self.layer_name)
179 + forward_context: ForwardContext = get_forward_context()
180 + ctx_attn_metadata = forward_context.attn_metadata
181 + self_kv_cache = self.kv_cache[forward_context.virtual_engine]
182 + self.impl.forward(self,
183 + query,
184 + key,
185 + value,
186 + self_kv_cache,
187 + ctx_attn_metadata,
188 + output=output)
189 else:
190 torch.ops.vllm.unified_attention_with_output(
191 query, key, value, output, self.layer_name)
192 return output.view(-1, hidden_size)
193 else:
194 if self.use_direct_call:
183 - return unified_attention(query, key, value, self.layer_name)
195 + forward_context = get_forward_context()
196 + ctx_attn_metadata = forward_context.attn_metadata
197 + self_kv_cache = self.kv_cache[forward_context.virtual_engine]
198 + return self.impl.forward(self, query, key, value,
199 + self_kv_cache, ctx_attn_metadata)
200 else:
201 return torch.ops.vllm.unified_attention(
202 query, key, value, self.layer_name)