A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Quant] Molmo SupportsQuant (#13336)
Kyle Sayers committed
Feb 18, 2025 at 00:34 UTC
88787bce1d33cacc77a6747d1c8e5e8ff97c8904
1 file changed
+7
-5
vllm/model_executor/models/molmo.py
+7
-5
@@ -52,7 +52,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
52
from vllm.sequence import IntermediateTensors
53
from vllm.utils import JSONTree, json_map_leaves
54
55
-from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
55
+from .interfaces import (SupportsLoRA, SupportsMultiModal, SupportsPP,
56
+ SupportsQuant)
57
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
58
is_pp_missing_parameter,
59
make_empty_intermediate_tensors_factory, make_layers,
@@ -633,7 +634,8 @@ class MolmoDecoderNormAfterLayer(MolmoDecoderLayer):
634
return hidden_states, residual
635
636
636
-class MolmoVisionBackbone(nn.Module):
637
+class MolmoVisionBackbone(nn.Module, SupportsQuant):
638
+ packed_modules_mapping = {"merged_linear": ["gate_proj", "up_proj"]}
639
640
def __init__(
641
self,
@@ -794,7 +796,7 @@ class MolmoVisionBackbone(nn.Module):
796
797
798
@support_torch_compile
797
-class MolmoModel(nn.Module):
799
+class MolmoModel(nn.Module, SupportsQuant):
800
801
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
802
super().__init__()
@@ -1402,8 +1404,8 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1404
@MULTIMODAL_REGISTRY.register_processor(MolmoMultiModalProcessor,
1405
info=MolmoProcessingInfo,
1406
dummy_inputs=MolmoDummyInputsBuilder)
1405
-class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1406
- SupportsLoRA):
1407
+class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1408
+ SupportsQuant):
1409
hf_to_vllm_mapper = WeightsMapper(
1410
orig_to_new_substr={
1411
# vision backbone mapping