A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Minor] Fix type annotation in Mixtral (#2036)
Woosuk Kwon committed
Dec 11, 2023 at 11:32 UTC
81ce2a4b26c7a76742039d7b96882e2e741ac5c8
1 file changed
+1
-1
vllm/model_executor/models/mixtral.py
+1
-1
@@ -251,7 +251,7 @@ class BlockSparseMoE(nn.Module):
251
return column_indices_t, offsets_t, block_offsets_t
252
253
def topology(self, x: torch.Tensor,
254
- padded_bins: torch.Tensor) -> stk.Matrix:
254
+ padded_bins: torch.Tensor) -> "stk.Matrix":
255
padded_tokens, _ = x.size()
256
assert padded_tokens % self.blocking == 0
257
assert self.ffn_dim_per_partition % self.blocking == 0