@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ca0d92227e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix compressed_tensors_moe bad config.strategy (#9677)

Michael Goin committed Oct 25, 2024 at 15:40 UTC ca0d92227e3a5e5880dde67da9d96c6d06454328
1 file changed +1 -1
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+1 -1
@@ -245,7 +245,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
245 config = self.quant_config.target_scheme_map["Linear"].get("weights")
246 self.num_bits = config.num_bits
247 self.packed_factor = 32 // config.num_bits
248 - self.strategy = config.strategy.value
248 + self.strategy = config.strategy
249 self.group_size = config.group_size
250 assert config.symmetric, (
251 "Only symmetric quantization is supported for MoE")