A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Quant] BartModel SupportsQuant (#14699)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
Kyle Sayers committed
Mar 13, 2025 at 00:55 UTC
36d1ccb2861a8137eb8ab66e100f68a1bc0801fb
1 file changed
+4
-3
vllm/model_executor/models/bart.py
+4
-3
@@ -44,7 +44,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.sequence import IntermediateTensors
46
47
-from .interfaces import SupportsV0Only
47
+from .interfaces import SupportsQuant, SupportsV0Only
48
from .utils import maybe_prefix
49
50
logger = logging.get_logger(__name__)
@@ -697,7 +697,7 @@ class BartDecoder(nn.Module):
697
return hidden_states
698
699
700
-class BartModel(nn.Module):
700
+class BartModel(nn.Module, SupportsQuant):
701
_tied_weights_keys = [
702
"encoder.embed_tokens.weight", "decoder.embed_tokens.weight"
703
]
@@ -763,7 +763,8 @@ class BartModel(nn.Module):
763
return decoder_outputs
764
765
766
-class BartForConditionalGeneration(nn.Module, SupportsV0Only):
766
+class BartForConditionalGeneration(nn.Module, SupportsV0Only, SupportsQuant):
767
+ packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
768
base_model_prefix = "model"
769
770
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):