A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
Fix unknown attribute of topk_indices_dtype in CompressedTensorsW8A8Fp8MoECutlassMethod (#20507)
Co-authored-by: Lucia (Lu) Fang <fanglu@meta.com>
Lucia Fang committed
Jul 5, 2025 at 14:03 UTC
8aeaa910a2c514022ab7521ce74eec0734886a1b
1 file changed
+2
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+2
@@ -368,6 +368,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
368
"weights")
369
self.input_quant = self.quant_config.target_scheme_map["Linear"].get(
370
"input_activations")
371
+ self.topk_indices_dtype = None
372
373
per_tensor = (self.weight_quant.strategy == QuantizationStrategy.TENSOR
374
and self.input_quant.strategy
@@ -738,6 +739,7 @@ class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod):
739
740
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
741
cutlass_moe_fp8)
742
+ self.topk_indices_dtype = None
743
self.fused_experts = cutlass_moe_fp8 # type: ignore
744
self.disable_expert_map = False
745