A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] support bitsandbytes quantization with minicpm3 model (#10682)
Signed-off-by: Ubuntu <zixuanzhang@bytedance.com>
zixuanzhang226 committed
Nov 27, 2024 at 23:58 UTC
70dc14fbd09d054ff75850036b81212ca67e5275
1 file changed
+6
vllm/model_executor/models/minicpm3.py
+6
@@ -241,6 +241,12 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
241
# `embedding_modules` and `embedding_padding_modules`
242
# are inherited from MiniCPMForCausalLM
243
244
+ bitsandbytes_stacked_params_mapping = {
245
+ # shard_name, weight_name, index
246
+ "gate_proj": ("gate_up_proj", 0),
247
+ "up_proj": ("gate_up_proj", 1),
248
+ }
249
+
250
def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
251
self.model = MiniCPM3Model(vllm_config=vllm_config,
252
prefix=maybe_prefix(prefix, "model"))