A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Kernel] Pipe attn_logits_soft_cap through paged attention TPU kernels (#12482)
Signed-off-by: Fenghui Zhang <fhzhang@google.com>
fenghuizhang committed
Jan 28, 2025 at 14:36 UTC
80fcc3ed1c940ea43e1b495bbdf8b9765f837128
2 files changed
+16
-26
.buildkite/run-tpu-test.sh
vllm/attention/backends/pallas.py
+16
-26
@@ -110,6 +110,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
110
111
assert self.num_heads % self.num_kv_heads == 0
112
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
113
+ self.logits_soft_cap = logits_soft_cap
114
if head_size % 128 != 0:
115
raise NotImplementedError("Head size must be a multiple of 128.")
116
if alibi_slopes is not None:
@@ -120,9 +121,6 @@ class PallasAttentionBackendImpl(AttentionImpl):
121
raise NotImplementedError("FP8 KV cache dtype is not supported.")
122
if blocksparse_params is not None:
123
raise NotImplementedError("Blocksparse is not supported.")
123
- if logits_soft_cap is not None:
124
- raise NotImplementedError(
125
- "Attention logits soft-capping is not supported.")
124
125
if torch_xla.tpu.version() < 4:
126
raise NotImplementedError("TPU version must be 4 or higher.")
@@ -230,6 +228,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
228
num_kv_pages_per_compute_block,
229
num_queries_per_compute_block,
230
use_kernel=True,
231
+ attn_logits_soft_cap=self.logits_soft_cap,
232
)
233
else:
234
# Decoding run.
@@ -257,6 +256,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
256
attn_metadata.block_tables,
257
pages_per_compute_block,
258
self.megacore_mode,
259
+ attn_logits_soft_cap=self.logits_soft_cap,
260
)
261
else:
262
chunk_size = max_num_seq
@@ -280,6 +280,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
280
attn_metadata.block_tables[chunk_start:chunk_end],
281
pages_per_compute_block,
282
self.megacore_mode,
283
+ attn_logits_soft_cap=self.logits_soft_cap,
284
)
285
output[chunk_start:chunk_end] = chunk_output
286
@@ -313,6 +314,8 @@ def paged_attention(
314
block_tables: torch.Tensor,
315
pages_per_compute_block: int,
316
megacore_mode: Optional[str],
317
+ *,
318
+ attn_logits_soft_cap: Optional[float],
319
) -> torch.Tensor:
320
batch_size = query.shape[0]
321
if megacore_mode == "batch" and batch_size % 2 != 0:
@@ -320,26 +323,13 @@ def paged_attention(
323
else:
324
megacore_mode = megacore_mode
325
323
- # NOTE(woosuk): A temporary workaround to avoid the error:
324
- # "xla::paged_attention() Expected a value of type 'str' for
325
- # argument 'megacore_mode' but instead found type 'NoneType'."
326
- if megacore_mode is not None:
327
- output = torch.ops.xla.paged_attention(
328
- query,
329
- key_cache,
330
- value_cache,
331
- context_lens,
332
- block_tables,
333
- pages_per_compute_block,
334
- megacore_mode=megacore_mode,
335
- )
336
- else:
337
- output = torch.ops.xla.paged_attention(
338
- query,
339
- key_cache,
340
- value_cache,
341
- context_lens,
342
- block_tables,
343
- pages_per_compute_block,
344
- )
345
- return output
326
+ return torch.ops.xla.paged_attention(
327
+ query,
328
+ key_cache,
329
+ value_cache,
330
+ context_lens,
331
+ block_tables,
332
+ pages_per_compute_block,
333
+ megacore_mode=megacore_mode,
334
+ attn_logits_soft_cap=attn_logits_soft_cap,
335
+ )