A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix compressed_tensors_moe bad config.strategy (#9677)
Michael Goin committed
Oct 25, 2024 at 15:40 UTC
ca0d92227e3a5e5880dde67da9d96c6d06454328
1 file changed
+1
-1
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+1
-1
@@ -245,7 +245,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
245
config = self.quant_config.target_scheme_map["Linear"].get("weights")
246
self.num_bits = config.num_bits
247
self.packed_factor = 32 // config.num_bits
248
- self.strategy = config.strategy.value
248
+ self.strategy = config.strategy
249
self.group_size = config.group_size
250
assert config.symmetric, (
251
"Only symmetric quantization is supported for MoE")