A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[TPU] Update PyTorch/XLA (#16288)
Signed-off-by: Chengji Yao <chengjiyao@google.com>
Chengji Yao committed
Apr 8, 2025 at 23:46 UTC
b1eb4ca152773d00c3e8fcadabafb0324c00725e
2 files changed
+10
-14
requirements/tpu.txt
+6
-6
@@ -17,10 +17,10 @@ ray[data]
17
--find-links https://storage.googleapis.com/libtpu-releases/index.html
18
--find-links https://storage.googleapis.com/jax-releases/jax_nightly_releases.html
19
--find-links https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
20
-torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250406-cp39-cp39-linux_x86_64.whl ; python_version == "3.9"
21
-torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250406-cp310-cp310-linux_x86_64.whl ; python_version == "3.10"
22
-torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250406-cp311-cp311-linux_x86_64.whl ; python_version == "3.11"
23
-torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250406-cp39-cp39-linux_x86_64.whl ; python_version == "3.9"
24
-torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250406-cp310-cp310-linux_x86_64.whl ; python_version == "3.10"
25
-torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250406-cp311-cp311-linux_x86_64.whl ; python_version == "3.11"
20
+torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250408-cp39-cp39-linux_x86_64.whl ; python_version == "3.9"
21
+torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250408-cp310-cp310-linux_x86_64.whl ; python_version == "3.10"
22
+torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250408-cp311-cp311-linux_x86_64.whl ; python_version == "3.11"
23
+torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250408-cp39-cp39-linux_x86_64.whl ; python_version == "3.9"
24
+torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250408-cp310-cp310-linux_x86_64.whl ; python_version == "3.10"
25
+torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250408-cp311-cp311-linux_x86_64.whl ; python_version == "3.11"
26
tests/v1/tpu/test_pallas.py
+4
-8
@@ -4,9 +4,7 @@ from unittest.mock import ANY, patch
4
import torch
5
6
from vllm.attention.backends.abstract import AttentionType
7
-from vllm.v1.attention.backends.pallas import (NUM_KV_PAGES_PER_BLOCK,
8
- NUM_QUERIES_PER_BLOCK,
9
- PallasAttentionBackendImpl,
7
+from vllm.v1.attention.backends.pallas import (PallasAttentionBackendImpl,
8
PallasMetadata)
9
10
@@ -32,8 +30,6 @@ def test_ragged_paged_attention():
30
logits_soft_cap=logits_soft_cap,
31
attn_type=AttentionType.DECODER,
32
)
35
- mock_vmem_limit_bytes = 1024
36
- attn_impl.vmem_limit_bytes = mock_vmem_limit_bytes
33
34
class FakeAttentionLayer:
35
_k_scale_float: float
@@ -88,9 +84,9 @@ def test_ragged_paged_attention():
84
ANY, # block_tables
85
ANY, # query_start_loc
86
ANY, # num_seqs
91
- num_kv_pages_per_block=NUM_KV_PAGES_PER_BLOCK,
92
- num_queries_per_block=NUM_QUERIES_PER_BLOCK,
93
- vmem_limit_bytes=mock_vmem_limit_bytes,
87
+ num_kv_pages_per_block=None,
88
+ num_queries_per_block=None,
89
+ vmem_limit_bytes=None,
90
use_kernel=True,
91
sm_scale=scale,
92
sliding_window=sliding_window,