A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Misc] Fix input_scale typing in w8a8_utils.py (#6579)
Michael Goin committed
Jul 20, 2024 at 19:11 UTC
f952bbc8ffccdd109b5bd8936655ce3fe3220807
1 file changed
+2
-2
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2
-2
@@ -104,7 +104,7 @@ def apply_fp8_linear(
104
input: torch.Tensor,
105
weight: torch.Tensor,
106
weight_scale: torch.Tensor,
107
- input_scale: torch.Tensor,
107
+ input_scale: Optional[torch.Tensor] = None,
108
input_scale_ub: Optional[torch.Tensor] = None,
109
bias: Optional[torch.Tensor] = None,
110
cutlass_fp8_supported: bool = True,
@@ -192,7 +192,7 @@ def apply_int8_linear(
192
input: torch.Tensor,
193
weight: torch.Tensor,
194
weight_scale: torch.Tensor,
195
- input_scale: torch.Tensor,
195
+ input_scale: Optional[torch.Tensor] = None,
196
bias: Optional[torch.Tensor] = None,
197
):
198
# ops.scaled_int8_quant supports both dynamic and static quant.