@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c467dff24f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Hardware][TPU] Support MoE with Pallas GMM kernel (#6457)

Woosuk Kwon committed Jul 16, 2024 at 09:56 UTC c467dff24f5dfa0b8e4045c3d265c6e606e35f99
5 files changed +89 -8
Dockerfile.tpu
+3 -1
@@ -1,4 +1,4 @@
1 -ARG NIGHTLY_DATE="20240601"
1 +ARG NIGHTLY_DATE="20240713"
2 ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
3
4 FROM $BASE_IMAGE
@@ -6,6 +6,8 @@ WORKDIR /workspace
6
7 # Install aiohttp separately to avoid build errors.
8 RUN pip install aiohttp
9 +# Install NumPy 1 instead of NumPy 2.
10 +RUN pip install "numpy<2"
11 # Install the TPU and Pallas dependencies.
12 RUN pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
13 RUN pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
docs/source/getting_started/tpu-installation.rst
+2 -2
@@ -56,7 +56,7 @@ First, install the dependencies:
56 $ pip uninstall torch torch-xla -y
57
58 $ # Install PyTorch and PyTorch XLA.
59 - $ export DATE="+20240601"
59 + $ export DATE="+20240713"
60 $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-nightly${DATE}-cp310-cp310-linux_x86_64.whl
61 $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-nightly${DATE}-cp310-cp310-linux_x86_64.whl
62
@@ -85,7 +85,7 @@ Next, build vLLM from source. This will only take a few seconds:
85 ImportError: libopenblas.so.0: cannot open shared object file: No such file or directory
86
87
88 - You can install OpenBLAS with the following command:
88 + Please install OpenBLAS with the following command:
89
90 .. code-block:: console
91
vllm/model_executor/layers/fused_moe/layer.py
+18
@@ -104,6 +104,24 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
104 raise NotImplementedError(
105 "The CPU backend currently does not support MoE.")
106
107 + def forward_tpu(
108 + self,
109 + x: torch.Tensor,
110 + w1: torch.Tensor,
111 + w2: torch.Tensor,
112 + router_logits: torch.Tensor,
113 + top_k: int,
114 + renormalize: bool,
115 + use_grouped_topk: bool,
116 + num_expert_group: Optional[int],
117 + topk_group: Optional[int],
118 + ) -> torch.Tensor:
119 + from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe
120 + assert not use_grouped_topk
121 + assert num_expert_group is None
122 + assert topk_group is None
123 + return fused_moe(x, w1, w2, router_logits, top_k, renormalize)
124 +
125
126 class FusedMoE(torch.nn.Module):
127 """FusedMoE layer for MoE models.
vllm/model_executor/layers/fused_moe/moe_pallas.py new
+62
@@ -0,0 +1,62 @@
1 +import torch
2 +import torch.nn.functional as F
3 +from torch_xla.experimental.custom_kernel import _histogram
4 +
5 +
6 +def fused_moe(
7 + hidden_states: torch.Tensor,
8 + w1: torch.Tensor,
9 + w2: torch.Tensor,
10 + gating_output: torch.Tensor,
11 + topk: int,
12 + renormalize: bool,
13 +) -> torch.Tensor:
14 + """
15 + Args:
16 + hidden_states: [*, hidden_size]
17 + w1: [num_experts, intermediate_size * 2, hidden_size]
18 + w2: [num_experts, hidden_size, intermediate_size]
19 + gating_output: [*, num_experts]
20 + """
21 + orig_shape = hidden_states.shape
22 + hidden_size = hidden_states.shape[-1]
23 + num_tokens = hidden_states.shape[:-1].numel()
24 + num_experts = w1.shape[0]
25 + intermediate_size = w2.shape[-1]
26 + device = hidden_states.device
27 + dtype = hidden_states.dtype
28 + assert (num_tokens * topk) % 16 == 0, (
29 + "The Pallas GMM kernel requires num_tokens * topk to be a multiple of "
30 + f"16 but got {num_tokens * topk}")
31 +
32 + hidden_states = hidden_states.view(num_tokens, hidden_size)
33 + gating_output = gating_output.view(num_tokens, num_experts)
34 + topk_weights = gating_output.softmax(dim=-1, dtype=torch.float)
35 + topk_weights, topk_indices = topk_weights.topk(topk, dim=-1)
36 + if renormalize:
37 + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
38 + topk_weights = topk_weights.to(dtype)
39 +
40 + topk_indices = topk_indices.flatten()
41 + topk_argsort_indices = topk_indices.argsort()
42 + topk_argsort_revert_indices = topk_argsort_indices.argsort()
43 + token_indices = torch.arange(num_tokens,
44 + device=device).repeat_interleave(topk)
45 + token_indices = token_indices[topk_argsort_indices]
46 + group_sizes = _histogram(topk_indices.to(torch.int32), 0, num_experts - 1)
47 +
48 + # NOTE(woosuk): The GMM Pallas kernel requires a different weight layout
49 + # from HF Transformers.
50 + w1 = w1.transpose(1, 2)
51 + w2 = w2.transpose(1, 2)
52 +
53 + x = hidden_states[token_indices]
54 + x = torch.ops.xla.gmm(x, w1, group_sizes)
55 + x = F.silu(x[..., :intermediate_size]) * x[..., intermediate_size:]
56 + x = torch.ops.xla.gmm(x, w2, group_sizes)
57 + x = x[topk_argsort_revert_indices].reshape(-1, topk, hidden_size)
58 +
59 + x = x * topk_weights.unsqueeze_(dim=-1)
60 + x = x.sum(dim=-2)
61 + x = x.reshape(orig_shape)
62 + return x
vllm/worker/tpu_model_runner.py
+4 -5
@@ -598,11 +598,10 @@ def _get_padded_prefill_len(x: int) -> int:
598
599
600 def _get_padded_batch_size(batch_size: int) -> int:
601 - if batch_size <= 2:
602 - return batch_size
603 - elif batch_size <= 4:
604 - return 4
605 - elif batch_size <= 8:
601 + # The GMM Pallas kernel requires num_tokens * topk to be a multiple of 16.
602 + # To meet this requirement in the simplest way, we set the minimal batch
603 + # size to 8.
604 + if batch_size <= 8:
605 return 8
606 else:
607 return ((batch_size + 15) // 16) * 16