@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4bd18ec0c7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Minor] Fix type annotation in fused moe (#3045)

Woosuk Kwon committed Feb 26, 2024 at 19:44 UTC 4bd18ec0c719d2910040e22fa60503fdbfce1332
1 file changed +4 -3
vllm/model_executor/layers/fused_moe/fused_moe.py
+4 -3
@@ -2,7 +2,7 @@
2 import functools
3 import json
4 import os
5 -from typing import Any, Dict, Optional
5 +from typing import Any, Dict, Optional, Tuple
6
7 import torch
8 import triton
@@ -137,7 +137,7 @@ def fused_moe_kernel(
137
138 def moe_align_block_size(
139 topk_ids: torch.Tensor, block_size: int,
140 - num_experts: int) -> (torch.Tensor, torch.Tensor, torch.Tensor):
140 + num_experts: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
141 """
142 Aligns the token distribution across experts to be compatible with block size for matrix multiplication.
143
@@ -185,7 +185,8 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
185 sorted_token_ids: torch.Tensor,
186 expert_ids: torch.Tensor,
187 num_tokens_post_padded: torch.Tensor,
188 - mul_routed_weight: bool, top_k: int, config: dict):
188 + mul_routed_weight: bool, top_k: int,
189 + config: Dict[str, Any]) -> None:
190 assert topk_weights.stride(1) == 1
191 assert sorted_token_ids.stride(0) == 1
192