@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 80fcc3ed1c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Kernel] Pipe attn_logits_soft_cap through paged attention TPU kernels (#12482)

Signed-off-by: Fenghui Zhang <fhzhang@google.com>

fenghuizhang committed Jan 28, 2025 at 14:36 UTC 80fcc3ed1c940ea43e1b495bbdf8b9765f837128
2 files changed +16 -26
.buildkite/run-tpu-test.sh
vllm/attention/backends/pallas.py
+16 -26
@@ -110,6 +110,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
110
111 assert self.num_heads % self.num_kv_heads == 0
112 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
113 + self.logits_soft_cap = logits_soft_cap
114 if head_size % 128 != 0:
115 raise NotImplementedError("Head size must be a multiple of 128.")
116 if alibi_slopes is not None:
@@ -120,9 +121,6 @@ class PallasAttentionBackendImpl(AttentionImpl):
121 raise NotImplementedError("FP8 KV cache dtype is not supported.")
122 if blocksparse_params is not None:
123 raise NotImplementedError("Blocksparse is not supported.")
123 - if logits_soft_cap is not None:
124 - raise NotImplementedError(
125 - "Attention logits soft-capping is not supported.")
124
125 if torch_xla.tpu.version() < 4:
126 raise NotImplementedError("TPU version must be 4 or higher.")
@@ -230,6 +228,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
228 num_kv_pages_per_compute_block,
229 num_queries_per_compute_block,
230 use_kernel=True,
231 + attn_logits_soft_cap=self.logits_soft_cap,
232 )
233 else:
234 # Decoding run.
@@ -257,6 +256,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
256 attn_metadata.block_tables,
257 pages_per_compute_block,
258 self.megacore_mode,
259 + attn_logits_soft_cap=self.logits_soft_cap,
260 )
261 else:
262 chunk_size = max_num_seq
@@ -280,6 +280,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
280 attn_metadata.block_tables[chunk_start:chunk_end],
281 pages_per_compute_block,
282 self.megacore_mode,
283 + attn_logits_soft_cap=self.logits_soft_cap,
284 )
285 output[chunk_start:chunk_end] = chunk_output
286
@@ -313,6 +314,8 @@ def paged_attention(
314 block_tables: torch.Tensor,
315 pages_per_compute_block: int,
316 megacore_mode: Optional[str],
317 + *,
318 + attn_logits_soft_cap: Optional[float],
319 ) -> torch.Tensor:
320 batch_size = query.shape[0]
321 if megacore_mode == "batch" and batch_size % 2 != 0:
@@ -320,26 +323,13 @@ def paged_attention(
323 else:
324 megacore_mode = megacore_mode
325
323 - # NOTE(woosuk): A temporary workaround to avoid the error:
324 - # "xla::paged_attention() Expected a value of type 'str' for
325 - # argument 'megacore_mode' but instead found type 'NoneType'."
326 - if megacore_mode is not None:
327 - output = torch.ops.xla.paged_attention(
328 - query,
329 - key_cache,
330 - value_cache,
331 - context_lens,
332 - block_tables,
333 - pages_per_compute_block,
334 - megacore_mode=megacore_mode,
335 - )
336 - else:
337 - output = torch.ops.xla.paged_attention(
338 - query,
339 - key_cache,
340 - value_cache,
341 - context_lens,
342 - block_tables,
343 - pages_per_compute_block,
344 - )
345 - return output
326 + return torch.ops.xla.paged_attention(
327 + query,
328 + key_cache,
329 + value_cache,
330 + context_lens,
331 + block_tables,
332 + pages_per_compute_block,
333 + megacore_mode=megacore_mode,
334 + attn_logits_soft_cap=attn_logits_soft_cap,
335 + )