A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
Allow torchao quantization in SiglipMLP (#15575)
Jerry Zhang committed
Mar 26, 2025 at 22:45 UTC
dcf2a590f52018ed91ff16d3ae439a0740420bca
1 file changed
+4
-2
vllm/model_executor/models/siglip.py
+4
-2
@@ -208,8 +208,10 @@ class SiglipMLP(nn.Module):
208
209
self.config = config
210
self.activation_fn = get_act_fn(config.hidden_act)
211
- # Special handling for BNB quantization
212
- if quant_config and quant_config.get_name() == "bitsandbytes":
211
+ # Special handling for BNB and torchao quantization
212
+ if quant_config and quant_config.get_name() in [
213
+ "bitsandbytes", "torchao"
214
+ ]:
215
quantizable = True
216
else:
217
# For other quantization, we require the hidden size to be a