@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b866cdbd05
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Add assertion and helpful message for marlin24 compressed models (#11388)

Dipika Sikka committed Dec 23, 2024 at 13:23 UTC b866cdbd05b13e0c0ab349efc6fca834fbe21760
1 file changed +4
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+4
@@ -61,6 +61,10 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
61 params_dtype: torch.dtype, weight_loader: Callable,
62 **kwargs):
63
64 + assert params_dtype == torch.float16, (
65 + "float16 is required for marlin24 compressd models. Set dtype=torch.float16" # noqa: E501
66 + )
67 +
68 pack_factor = 32 // self.quant_type.size_bits
69 output_size_per_partition = sum(output_partition_sizes)
70