@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 70dc14fbd0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] support bitsandbytes quantization with minicpm3 model (#10682)

Signed-off-by: Ubuntu <zixuanzhang@bytedance.com>

zixuanzhang226 committed Nov 27, 2024 at 23:58 UTC 70dc14fbd09d054ff75850036b81212ca67e5275
1 file changed +6
vllm/model_executor/models/minicpm3.py
+6
@@ -241,6 +241,12 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
241 # `embedding_modules` and `embedding_padding_modules`
242 # are inherited from MiniCPMForCausalLM
243
244 + bitsandbytes_stacked_params_mapping = {
245 + # shard_name, weight_name, index
246 + "gate_proj": ("gate_up_proj", 0),
247 + "up_proj": ("gate_up_proj", 1),
248 + }
249 +
250 def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
251 self.model = MiniCPM3Model(vllm_config=vllm_config,
252 prefix=maybe_prefix(prefix, "model"))