A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[TPU] remove transpose ops in moe kernel (#18923)
Signed-off-by: Chengji Yao <chengjiyao@google.com>
Chengji Yao committed
May 29, 2025 at 16:00 UTC
a1cc9f33a32eef4550daccdc76aefc1baf7bc35d
3 files changed
+8
-13
requirements/tpu.txt
+5
-5
@@ -18,9 +18,9 @@ setuptools==78.1.0
18
--find-links https://storage.googleapis.com/libtpu-releases/index.html
19
--find-links https://storage.googleapis.com/jax-releases/jax_nightly_releases.html
20
--find-links https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
21
-torch==2.8.0.dev20250518
22
-torchvision==0.22.0.dev20250518
23
-torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250518-cp39-cp39-linux_x86_64.whl ; python_version == "3.9"
24
-torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250518-cp310-cp310-linux_x86_64.whl ; python_version == "3.10"
25
-torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250518-cp311-cp311-linux_x86_64.whl ; python_version == "3.11"
21
+torch==2.8.0.dev20250529
22
+torchvision==0.22.0.dev20250529
23
+torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250529-cp39-cp39-linux_x86_64.whl ; python_version == "3.9"
24
+torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250529-cp310-cp310-linux_x86_64.whl ; python_version == "3.10"
25
+torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250529-cp311-cp311-linux_x86_64.whl ; python_version == "3.11"
26
tests/tpu/test_moe_pallas.py
+1
-1
@@ -26,7 +26,7 @@ TOP_KS = [2, 6]
26
# The Pallas GMM kernel requires num_tokens * topk to be a multiple of 16
27
@pytest.mark.parametrize("m", [8, 16, 64, 2048])
28
@pytest.mark.parametrize("n", [128, 1024, 2048])
29
-@pytest.mark.parametrize("k", [128, 511, 1024])
29
+@pytest.mark.parametrize("k", [128, 512, 1024])
30
@pytest.mark.parametrize("e", NUM_EXPERTS)
31
@pytest.mark.parametrize("topk", TOP_KS)
32
@pytest.mark.parametrize("ep_size", EP_SIZE)
vllm/model_executor/layers/fused_moe/moe_pallas.py
+2
-7
@@ -67,15 +67,10 @@ def fused_moe(
67
token_indices = token_indices[topk_argsort_indices]
68
group_sizes = _histogram(topk_indices.to(torch.int32), 0, num_experts - 1)
69
70
- # NOTE(woosuk): The GMM Pallas kernel requires a different weight layout
71
- # from HF Transformers.
72
- w1 = w1.transpose(1, 2)
73
- w2 = w2.transpose(1, 2)
74
-
70
x = hidden_states[token_indices]
76
- x = torch.ops.xla.gmm(x, w1, group_sizes)
71
+ x = torch.ops.xla.gmm(x, w1, group_sizes, transpose_rhs=True)
72
x = F.silu(x[..., :intermediate_size]) * x[..., intermediate_size:]
78
- x = torch.ops.xla.gmm(x, w2, group_sizes)
73
+ x = torch.ops.xla.gmm(x, w2, group_sizes, transpose_rhs=True)
74
x = x[topk_argsort_revert_indices].reshape(-1, topk, hidden_size)
75
76
x = x * topk_weights.unsqueeze(dim=-1)