A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Add assertion and helpful message for marlin24 compressed models (#11388)
Dipika Sikka committed
Dec 23, 2024 at 13:23 UTC
b866cdbd05b13e0c0ab349efc6fca834fbe21760
1 file changed
+4
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+4
@@ -61,6 +61,10 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
61
params_dtype: torch.dtype, weight_loader: Callable,
62
**kwargs):
63
64
+ assert params_dtype == torch.float16, (
65
+ "float16 is required for marlin24 compressd models. Set dtype=torch.float16" # noqa: E501
66
+ )
67
+
68
pack_factor = 32 // self.quant_type.size_bits
69
output_size_per_partition = sum(output_partition_sizes)
70