A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Quant] Arctic SupportsQuant (#13366)
Kyle Sayers committed
Feb 18, 2025 at 00:35 UTC
00294e1bc63de233966c5b99c3118ebece442611
1 file changed
+3
-2
vllm/model_executor/models/arctic.py
+3
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.utils import set_weight_attrs
33
from vllm.sequence import IntermediateTensors
34
from vllm.transformers_utils.configs.arctic import ArcticConfig
35
36
-from .interfaces import SupportsPP
36
+from .interfaces import SupportsPP, SupportsQuant
37
from .utils import (extract_layer_index, is_pp_missing_parameter,
38
make_empty_intermediate_tensors_factory, make_layers,
39
maybe_prefix)
@@ -423,7 +423,8 @@ class ArcticModel(nn.Module):
423
return hidden_states
424
425
426
-class ArcticForCausalLM(nn.Module, SupportsPP):
426
+class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):
427
+ packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
428
429
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
430
super().__init__()