@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 81ce2a4b26
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Minor] Fix type annotation in Mixtral (#2036)

Woosuk Kwon committed Dec 11, 2023 at 11:32 UTC 81ce2a4b26c7a76742039d7b96882e2e741ac5c8
1 file changed +1 -1
vllm/model_executor/models/mixtral.py
+1 -1
@@ -251,7 +251,7 @@ class BlockSparseMoE(nn.Module):
251 return column_indices_t, offsets_t, block_offsets_t
252
253 def topology(self, x: torch.Tensor,
254 - padded_bins: torch.Tensor) -> stk.Matrix:
254 + padded_bins: torch.Tensor) -> "stk.Matrix":
255 padded_tokens, _ = x.size()
256 assert padded_tokens % self.blocking == 0
257 assert self.ffn_dim_per_partition % self.blocking == 0