A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Bugfix] Add int8 torch dtype for KVCache (#15260)
Signed-off-by: shen-shanshan <467638484@qq.com>
Shanshan Shen committed
Mar 21, 2025 at 16:58 UTC
a989ca2bf6398b63607637892240375c63d76a9e
1 file changed
+1
vllm/utils.py
+1
@@ -153,6 +153,7 @@ STR_DTYPE_TO_TORCH_DTYPE = {
153
"fp8": torch.uint8,
154
"fp8_e4m3": torch.uint8,
155
"fp8_e5m2": torch.uint8,
156
+ "int8": torch.int8,
157
}
158
159
TORCH_DTYPE_TO_NUMPY_DTYPE = {