A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Quant] BaiChuan SupportsQuant (#13710)
Kyle Sayers committed
Feb 22, 2025 at 22:21 UTC
d5ca2110f1c799155eac846724e841a81240d0c3
1 file changed
+3
-2
vllm/model_executor/models/baichuan.py
+3
-2
@@ -46,7 +46,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
from vllm.sequence import IntermediateTensors
48
49
-from .interfaces import SupportsLoRA, SupportsPP
49
+from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
50
from .utils import (is_pp_missing_parameter,
51
make_empty_intermediate_tensors_factory, make_layers)
52
@@ -334,7 +334,8 @@ class BaiChuanModel(nn.Module):
334
return hidden_states
335
336
337
-class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
337
+class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
338
+ SupportsQuant):
339
packed_modules_mapping = {
340
"W_pack": ["W_pack"],
341
"gate_up_proj": [