@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 263a870ee1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Hardware][TPU] workaround fix for MoE on TPU (#11764)

Avshalom Manevich committed Jan 12, 2025 at 17:53 UTC 263a870ee18bd6a90e25dbfa342be32c6b92c33e
3 files changed +60 -1
tests/kernels/test_moe.py
+7
@@ -14,6 +14,8 @@ from vllm import _custom_ops as ops
14 from vllm.model_executor.layers.fused_moe import fused_moe
15 from vllm.model_executor.layers.fused_moe.fused_moe import (
16 fused_topk, moe_align_block_size)
17 +from vllm.model_executor.layers.fused_moe.moe_torch_iterative import (
18 + fused_moe as iterative_moe)
19 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
20 marlin_quantize)
21 from vllm.model_executor.models.mixtral import MixtralMoE
@@ -46,6 +48,11 @@ def test_fused_moe(
48 triton_output = fused_moe(a, w1, w2, score, topk, renormalize=False)
49 torch_output = torch_moe(a, w1, w2, score, topk)
50 torch.testing.assert_close(triton_output, torch_output, atol=2e-2, rtol=0)
51 + iterative_output = iterative_moe(a, w1, w2, score, topk, renormalize=False)
52 + torch.testing.assert_close(iterative_output,
53 + torch_output,
54 + atol=2e-2,
55 + rtol=0)
56
57
58 @pytest.mark.parametrize("dtype",
vllm/model_executor/layers/fused_moe/layer.py
+2 -1
@@ -20,7 +20,8 @@ if current_platform.is_cuda_alike():
20 else:
21 fused_experts = None # type: ignore
22 if current_platform.is_tpu():
23 - from .moe_pallas import fused_moe as fused_moe_pallas
23 + # the iterative moe implementation is used until the moe_pallas is fixed
24 + from .moe_torch_iterative import fused_moe as fused_moe_pallas
25 else:
26 fused_moe_pallas = None # type: ignore
27 logger = init_logger(__name__)
vllm/model_executor/layers/fused_moe/moe_torch_iterative.py new
+51
@@ -0,0 +1,51 @@
1 +import torch
2 +import torch.nn.functional as F
3 +
4 +
5 +def fused_moe(
6 + hidden_states: torch.Tensor,
7 + w1: torch.Tensor,
8 + w2: torch.Tensor,
9 + gating_output: torch.Tensor,
10 + topk: int,
11 + renormalize: bool,
12 +) -> torch.Tensor:
13 + """
14 + Args:
15 + hidden_states: [*, hidden_size]
16 + w1: [num_experts, intermediate_size * 2, hidden_size]
17 + w2: [num_experts, hidden_size, intermediate_size]
18 + gating_output: [*, num_experts]
19 + """
20 + orig_shape = hidden_states.shape
21 + hidden_size = hidden_states.shape[-1]
22 + num_tokens = hidden_states.shape[:-1].numel()
23 + num_experts = w1.shape[0]
24 + intermediate_size = w2.shape[-1]
25 + dtype = hidden_states.dtype
26 +
27 + hidden_states = hidden_states.view(num_tokens, hidden_size)
28 + gating_output = gating_output.view(num_tokens, num_experts)
29 + topk_weights = gating_output.softmax(dim=-1, dtype=torch.float)
30 + topk_weights, selected_experts = topk_weights.topk(topk, dim=-1)
31 + if renormalize:
32 + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
33 + topk_weights = topk_weights.to(dtype)
34 +
35 + final_hidden_states = None
36 + for expert_idx in range(num_experts):
37 + expert_w1 = w1[expert_idx]
38 + expert_w2 = w2[expert_idx]
39 + expert_mask = (selected_experts == expert_idx)
40 + expert_weights = (topk_weights * expert_mask).sum(dim=-1, keepdim=True)
41 + x = F.linear(hidden_states, expert_w1)
42 + gate = F.silu(x[:, :intermediate_size])
43 + x = x[:, intermediate_size:] * gate
44 + x = F.linear(x, expert_w2)
45 + current_hidden_states = x * expert_weights
46 + if final_hidden_states is None:
47 + final_hidden_states = current_hidden_states
48 + else:
49 + final_hidden_states = final_hidden_states + current_hidden_states
50 +
51 + return final_hidden_states.view(orig_shape) # type: ignore