A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Quant] Bamba SupportsQuant (#14698)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
Kyle Sayers committed
Mar 13, 2025 at 00:57 UTC
5d043c16853053f5c6daacb8cac35a3bbc63dcd2
1 file changed
+2
-2
vllm/model_executor/models/bamba.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.sequence import IntermediateTensors
33
from vllm.utils import LayerBlockType
34
35
from .interfaces import (HasInnerState, IsHybrid, SupportsLoRA, SupportsPP,
36
- SupportsV0Only)
36
+ SupportsQuant, SupportsV0Only)
37
from .utils import (is_pp_missing_parameter,
38
make_empty_intermediate_tensors_factory, make_layers,
39
maybe_prefix)
@@ -367,7 +367,7 @@ class BambaModel(nn.Module):
367
368
369
class BambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
370
- IsHybrid, SupportsV0Only):
370
+ IsHybrid, SupportsV0Only, SupportsQuant):
371
packed_modules_mapping = {
372
"qkv_proj": [
373
"q_proj",