@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 421c462948

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 16:00 UTC

[SupportsQuant] Bert, Blip, Blip2, Bloom (#15573)

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>

Kyle Sayers committed Apr 3, 2025 at 11:23 UTC 421c462948018a565017d9d3504f9445e1193b66

4 files changed +16 -9

vllm/model_executor/models/bert.py

+6 -4

@@ -26,7 +26,7 @@ from vllm.sequence import IntermediateTensors, PoolerOutput

26 from vllm.transformers_utils.config import (

27 get_cross_encoder_activation_function)

29 -from .interfaces import SupportsCrossEncoding, SupportsV0Only

29 +from .interfaces import SupportsCrossEncoding, SupportsQuant, SupportsV0Only

30 from .utils import WeightsMapper, maybe_prefix

@@ -313,7 +313,8 @@ class BertOutput(nn.Module):

313 return hidden_states

314

315

316 -class BertModel(nn.Module):

316 +class BertModel(nn.Module, SupportsQuant):

317 + packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]}

318

319 def __init__(self,

320 *,

@@ -385,7 +386,7 @@ class BertModel(nn.Module):

386 return loaded_params

387

388

388 -class BertEmbeddingModel(nn.Module, SupportsV0Only):

389 +class BertEmbeddingModel(nn.Module, SupportsV0Only, SupportsQuant):

390 """A model that uses Bert to provide embedding functionalities.

391

392 This class encapsulates the BertModel and provides an interface for

@@ -443,7 +444,8 @@ class BertEmbeddingModel(nn.Module, SupportsV0Only):

444 softmax=False)

445

446

446 -class BertForSequenceClassification(nn.Module, SupportsCrossEncoding):

447 +class BertForSequenceClassification(nn.Module, SupportsCrossEncoding,

448 + SupportsQuant):

449 """A model that uses Bert to provide embedding functionalities.

450

451 This class encapsulates the BertModel and provides an interface for

vllm/model_executor/models/blip.py

+4 -1

@@ -16,6 +16,8 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

16 from vllm.model_executor.layers.quantization import QuantizationConfig

17 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

19 +from .interfaces import SupportsQuant

20 +

22 def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:

23 assert image_size % patch_size == 0

@@ -243,9 +245,10 @@ class BlipEncoder(nn.Module):

245 return hidden_states

246

247

246 -class BlipVisionModel(nn.Module):

248 +class BlipVisionModel(nn.Module, SupportsQuant):

249 config_class = BlipVisionConfig

250 main_input_name = "pixel_values"

251 + packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}

252

253 def __init__(

254 self,

vllm/model_executor/models/blip2.py

+4 -2

@@ -24,7 +24,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

24 from vllm.sequence import IntermediateTensors

26 from .blip import BlipVisionModel

27 -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

27 +from .interfaces import (MultiModalEmbeddings, SupportsMultiModal, SupportsPP,

28 + SupportsQuant)

29 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

30 maybe_prefix, merge_multimodal_embeddings)

@@ -498,7 +499,8 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):

499 @MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor,

500 info=Blip2ProcessingInfo,

501 dummy_inputs=Blip2DummyInputsBuilder)

501 -class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

502 +class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

503 + SupportsQuant):

504

505 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

506

vllm/model_executor/models/bloom.py

+2 -2

@@ -42,7 +42,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader

42 from vllm.model_executor.sampling_metadata import SamplingMetadata

43 from vllm.sequence import IntermediateTensors

45 -from .interfaces import SupportsPP, SupportsV0Only

45 +from .interfaces import SupportsPP, SupportsQuant, SupportsV0Only

46 from .utils import (is_pp_missing_parameter,

47 make_empty_intermediate_tensors_factory, make_layers,

48 maybe_prefix)

@@ -279,7 +279,7 @@ class BloomModel(nn.Module):

279 return hidden_states

280

281

282 -class BloomForCausalLM(nn.Module, SupportsPP, SupportsV0Only):

282 +class BloomForCausalLM(nn.Module, SupportsPP, SupportsV0Only, SupportsQuant):

283

284 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

285 super().__init__()