A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 20:15 UTC
[Bugfix] Fix activation quantization for compressed-tensors W4A16 (#31572)
Signed-off-by: Tmn07 <tmn0796@gmail.com>
Tmn07 committed
Jan 2, 2026 at 13:13 UTC
ca179d0f64743537a430631e7fc79405ec2887cb
1 file changed
+1
-1
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+1
-1
@@ -114,7 +114,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
114
logger.info("Using %s for CompressedTensorsWNA16", kernel_type.__name__)
115
self._kernel_backends_being_used.add(kernel_type.__name__)
116
117
- if isinstance(kernel_type, MarlinLinearKernel):
117
+ if kernel_type is MarlinLinearKernel:
118
input_dtype = get_marlin_input_dtype(self.layer_name)
119
if input_dtype is not None:
120
mp_linear_kernel_config.act_type = input_dtype