A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Model] Refactor BLIP/BLIP-2 to support composite model loading (#8407)
Cyrus Leung committed
Sep 22, 2024 at 20:24 UTC
06ed2815e2be50e527839c7ab09ce2639b7910b6
10 files changed
+113
-114
vllm/model_executor/models/blip.py
+58
-3
@@ -1,6 +1,6 @@
1
"""Minimal implementation of BlipVisionModel intended to be only used
2
within a vision language model."""
3
-from typing import Optional, Union
3
+from typing import Iterable, Optional, Tuple, Union
4
5
import torch
6
import torch.nn as nn
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
QKVParallelLinear,
17
RowParallelLinear)
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
from vllm.multimodal.utils import (cached_get_tokenizer,
21
repeat_and_pad_placeholder_tokens)
22
from vllm.sequence import SequenceData
@@ -342,6 +343,10 @@ class BlipVisionModel(nn.Module):
343
num_hidden_layers_override: Optional[int] = None):
344
super().__init__()
345
346
+ tp_size = get_tensor_model_parallel_world_size()
347
+ num_heads = config.num_attention_heads
348
+ self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
349
+
350
self.config = config
351
352
self.embeddings = BlipVisionEmbeddings(config)
@@ -350,11 +355,61 @@ class BlipVisionModel(nn.Module):
355
quant_config=quant_config,
356
num_hidden_layers_override=num_hidden_layers_override,
357
)
353
- self.post_layernorm = nn.LayerNorm(config.hidden_size,
354
- eps=config.layer_norm_eps)
358
+
359
+ if len(self.encoder.layers) > config.num_hidden_layers:
360
+ raise ValueError(
361
+ f"The original encoder only has {config.num_hidden_layers} "
362
+ f"layers, but you requested {len(self.encoder.layers)} layers."
363
+ )
364
+ elif len(self.encoder.layers) == config.num_hidden_layers:
365
+ self.post_layernorm = nn.LayerNorm(config.hidden_size,
366
+ eps=config.layer_norm_eps)
367
+ else:
368
+ # post_layernorm is unused when we extract intermediate features
369
+ # In this case, we can skip it to conserve memory
370
+ self.post_layernorm = None
371
372
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
373
hidden_states = self.embeddings(pixel_values)
374
hidden_states = self.encoder(inputs_embeds=hidden_states)
375
376
+ if self.post_layernorm is None:
377
+ return hidden_states
378
+
379
return self.post_layernorm(hidden_states)
380
+
381
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
382
+ stacked_params_mapping = [
383
+ # (param_name, shard_name, shard_id)
384
+ ("qkv_proj", "q_proj", "q"),
385
+ ("qkv_proj", "k_proj", "k"),
386
+ ("qkv_proj", "v_proj", "v"),
387
+ ] if self.shard_weight else []
388
+ params_dict = dict(self.named_parameters())
389
+ layer_count = len(self.encoder.layers)
390
+
391
+ for name, loaded_weight in weights:
392
+ # post_layernorm is not needed in BlipVisionModel
393
+ if (name.startswith("post_layernorm")
394
+ and self.post_layernorm is None):
395
+ continue
396
+
397
+ # omit layers when num_hidden_layers_override is set
398
+ if name.startswith("encoder.layers"):
399
+ layer_idx = int(name.split(".")[2])
400
+ if layer_idx >= layer_count:
401
+ continue
402
+
403
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
404
+ if weight_name not in name:
405
+ continue
406
+
407
+ param = params_dict[name.replace(weight_name, param_name)]
408
+ weight_loader = param.weight_loader
409
+ weight_loader(param, loaded_weight, shard_id)
410
+ break
411
+ else:
412
+ param = params_dict[name]
413
+ weight_loader = getattr(param, "weight_loader",
414
+ default_weight_loader)
415
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/blip2.py
+47
-74
@@ -10,11 +10,9 @@ from vllm.attention import AttentionMetadata
10
from vllm.config import CacheConfig, MultiModalConfig
11
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12
from vllm.model_executor.layers.activation import get_act_fn
13
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
13
from vllm.model_executor.layers.quantization import QuantizationConfig
15
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
14
+from vllm.model_executor.layers.sampler import SamplerOutput
15
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
-from vllm.model_executor.models.opt import OPTModel
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
from vllm.sequence import IntermediateTensors, SequenceData
@@ -22,12 +20,8 @@ from vllm.sequence import IntermediateTensors, SequenceData
20
from .blip import (BlipVisionModel, dummy_image_for_blip,
21
get_max_blip_image_tokens)
22
from .interfaces import SupportsMultiModal
25
-from .utils import merge_multimodal_embeddings
26
-
27
-_KEYS_TO_MODIFY_MAPPING = {
28
- "language_model.lm_head": "lm_head",
29
- "language_model.model": "language_model",
30
-}
23
+from .utils import (group_weights_with_prefix, init_vllm_registered_model,
24
+ merge_multimodal_embeddings)
25
26
# We use this internally as placeholders since there is no image token
27
# defined on the HuggingFace repo
@@ -491,9 +485,6 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
485
486
super().__init__()
487
494
- # currently all existing BLIP-2 models have `tie_word_embeddings`
495
- # enabled
496
- assert config.tie_word_embeddings
488
self.config = config
489
self.multimodal_config = multimodal_config
490
@@ -514,17 +505,8 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
505
bias=True,
506
)
507
517
- self.quant_config = quant_config
518
-
519
- self.language_model = OPTModel(config.text_config, cache_config,
520
- quant_config)
521
-
522
- self.unpadded_vocab_size = config.text_config.vocab_size
523
- self.logits_processor = LogitsProcessor(self.unpadded_vocab_size)
524
- self.sampler = Sampler()
525
-
526
- def get_lm_head(self):
527
- return self.language_model.decoder.embed_tokens
508
+ self.language_model = init_vllm_registered_model(
509
+ config.text_config, cache_config, quant_config)
510
511
def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
512
h = w = self.config.vision_config.image_size
@@ -653,7 +635,8 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
635
636
if image_input is not None:
637
vision_embeddings = self._process_image_input(image_input)
656
- inputs_embeds = self.language_model.get_input_embeddings(input_ids)
638
+ inputs_embeds = self.language_model.model.get_input_embeddings(
639
+ input_ids)
640
641
inputs_embeds = merge_multimodal_embeddings(
642
input_ids, inputs_embeds, vision_embeddings,
@@ -663,11 +646,11 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
646
else:
647
inputs_embeds = None
648
666
- hidden_states = self.language_model(input_ids,
667
- positions,
668
- kv_caches,
669
- attn_metadata,
670
- inputs_embeds=inputs_embeds)
649
+ hidden_states = self.language_model.model(input_ids,
650
+ positions,
651
+ kv_caches,
652
+ attn_metadata,
653
+ inputs_embeds=inputs_embeds)
654
655
return hidden_states
656
@@ -676,56 +659,46 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
659
hidden_states: torch.Tensor,
660
sampling_metadata: SamplingMetadata,
661
) -> Optional[torch.Tensor]:
679
- logits = self.logits_processor(self.get_lm_head(), hidden_states,
680
- sampling_metadata)
681
- return logits
662
+ return self.language_model.compute_logits(hidden_states,
663
+ sampling_metadata)
664
665
def sample(
666
self,
667
logits: torch.Tensor,
668
sampling_metadata: SamplingMetadata,
669
) -> Optional[SamplerOutput]:
688
- next_tokens = self.sampler(logits, sampling_metadata)
689
- return next_tokens
670
+ return self.language_model.sample(logits, sampling_metadata)
671
672
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
692
- # only doing this for language model part for now.
693
- stacked_params_mapping = [
694
- # (param_name, shard_name, shard_id)
695
- ("qkv_proj", "q_proj", "q"),
696
- ("qkv_proj", "k_proj", "k"),
697
- ("qkv_proj", "v_proj", "v"),
698
- ("gate_up_proj", "gate_proj", 0),
699
- ("gate_up_proj", "up_proj", 1),
700
- ]
701
- params_dict = dict(self.named_parameters())
702
-
703
- for name, loaded_weight in weights:
704
- if "lm_head.weight" in name:
705
- continue
706
- if "rotary_emb.inv_freq" in name:
707
- continue
708
- for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
709
- if key_to_modify in name:
710
- name = name.replace(key_to_modify, new_key)
711
- use_default_weight_loading = False
712
- if "vision" in name:
713
- if self.vision_model is not None:
714
- # BlipVisionModel does not need sharding
715
- use_default_weight_loading = True
716
- else:
717
- for (param_name, weight_name,
718
- shard_id) in stacked_params_mapping:
719
- if weight_name not in name:
720
- continue
721
- param = params_dict[name.replace(weight_name, param_name)]
722
- weight_loader = param.weight_loader
723
- weight_loader(param, loaded_weight, shard_id)
724
- break
725
- else:
726
- use_default_weight_loading = True
727
- if use_default_weight_loading:
728
- param = params_dict[name]
729
- weight_loader = getattr(param, "weight_loader",
730
- default_weight_loader)
731
- weight_loader(param, loaded_weight)
673
+ # prepare weight iterators for components
674
+ weights_group = group_weights_with_prefix(weights)
675
+
676
+ # load vision encoder
677
+ self.vision_model.load_weights(weights_group["vision_model"])
678
+
679
+ # load query tokens
680
+ for name, loaded_weight in weights_group["query_tokens"]:
681
+ assert name == ""
682
+ param = self.query_tokens
683
+ weight_loader = getattr(param, "weight_loader",
684
+ default_weight_loader)
685
+ weight_loader(param, loaded_weight)
686
+
687
+ # load qformer
688
+ qformer_params_dict = dict(self.qformer.named_parameters())
689
+ for name, loaded_weight in weights_group["qformer"]:
690
+ param = qformer_params_dict[name]
691
+ weight_loader = getattr(param, "weight_loader",
692
+ default_weight_loader)
693
+ weight_loader(param, loaded_weight)
694
+
695
+ # load mlp projector
696
+ mlp_params_dict = dict(self.language_projection.named_parameters())
697
+ for name, loaded_weight in weights_group["language_projection"]:
698
+ param = mlp_params_dict[name]
699
+ weight_loader = getattr(param, "weight_loader",
700
+ default_weight_loader)
701
+ weight_loader(param, loaded_weight)
702
+
703
+ # load llm backbone
704
+ self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/chameleon.py
-3
@@ -12,7 +12,6 @@ from vllm.attention import Attention, AttentionMetadata
12
from vllm.config import CacheConfig, MultiModalConfig
13
from vllm.distributed import get_tensor_model_parallel_world_size
14
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15
-from vllm.logger import init_logger
15
from vllm.model_executor.layers.activation import SiluAndMul
16
from vllm.model_executor.layers.layernorm import RMSNorm
17
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -36,8 +35,6 @@ from vllm.utils import print_warning_once
35
36
from .interfaces import SupportsMultiModal
37
39
-logger = init_logger(__name__)
40
-
38
# These configs are not part of the model config but the preprocessor
39
# and processor files, so we hardcode them in the model file for now.
40
CHAMELEON_CROP_SIZE_HEIGHT = CHAMELEON_CROP_SIZE_WIDTH = 512
vllm/model_executor/models/clip.py
+4
-7
@@ -391,6 +391,7 @@ class CLIPVisionModel(nn.Module):
391
quant_config: Optional[QuantizationConfig] = None,
392
num_hidden_layers_override: Optional[int] = None):
393
super().__init__()
394
+
395
tp_size = get_tensor_model_parallel_world_size()
396
num_heads = config.num_attention_heads
397
self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
@@ -400,10 +401,6 @@ class CLIPVisionModel(nn.Module):
401
quant_config=quant_config,
402
num_hidden_layers_override=num_hidden_layers_override)
403
403
- @property
404
- def _require_post_layernorm(self) -> bool:
405
- return self.vision_model.post_layernorm is not None
406
-
404
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
405
return self.vision_model(pixel_values)
406
@@ -425,12 +422,12 @@ class CLIPVisionModel(nn.Module):
422
423
for name, loaded_weight in weights:
424
# post_layernorm is not needed in CLIPVisionModel
428
- if ("vision_model.post_layernorm" in name
429
- and not self._require_post_layernorm):
425
+ if (name.startswith("vision_model.post_layernorm")
426
+ and self.vision_model.post_layernorm is None):
427
continue
428
429
# omit layers when num_hidden_layers_override is set
433
- if "vision_model.encoder.layers." in name:
430
+ if name.startswith("vision_model.encoder.layers"):
431
layer_idx = int(name.split(".")[3])
432
if layer_idx >= layer_count:
433
continue
vllm/model_executor/models/fuyu.py
-3
@@ -28,7 +28,6 @@ from transformers import FuyuConfig, FuyuImageProcessor
28
from vllm.attention import AttentionMetadata
29
from vllm.config import CacheConfig, MultiModalConfig
30
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31
-from vllm.logger import init_logger
31
from vllm.model_executor.layers.linear import ColumnParallelLinear
32
from vllm.model_executor.layers.quantization import QuantizationConfig
33
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -45,8 +44,6 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
44
from .interfaces import SupportsMultiModal
45
from .utils import merge_multimodal_embeddings
46
48
-logger = init_logger(__name__)
49
-
47
# Cannot find the following 2 numbers from hf config.
48
_IMAGE_TOKEN_ID = 71011
49
_NEWLINE_TOKEN_ID = 71019
vllm/model_executor/models/llava_next.py
-8
@@ -12,7 +12,6 @@ from typing_extensions import NotRequired
12
from vllm.attention import AttentionMetadata
13
from vllm.config import CacheConfig, MultiModalConfig
14
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15
-from vllm.logger import init_logger
15
from vllm.model_executor.layers.quantization import QuantizationConfig
16
from vllm.model_executor.layers.sampler import SamplerOutput
17
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -32,13 +31,6 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31
from .utils import (flatten_bn, group_weights_with_prefix,
32
init_vllm_registered_model, merge_multimodal_embeddings)
33
35
-logger = init_logger(__name__)
36
-
37
-_KEYS_TO_MODIFY_MAPPING = {
38
- "language_model.lm_head": "lm_head",
39
- "language_model.model": "language_model",
40
-}
41
-
34
# Result in the max possible feature size (2x2 grid of 336x336px tiles)
35
MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
36
vllm/model_executor/models/llava_next_video.py
-3
@@ -11,7 +11,6 @@ from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
11
from vllm.attention import AttentionMetadata
12
from vllm.config import CacheConfig, MultiModalConfig
13
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14
-from vllm.logger import init_logger
14
from vllm.model_executor.layers.activation import get_act_fn
15
from vllm.model_executor.layers.quantization.base_config import (
16
QuantizationConfig)
@@ -32,8 +31,6 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31
from .utils import (group_weights_with_prefix, init_vllm_registered_model,
32
merge_multimodal_embeddings)
33
35
-logger = init_logger(__name__)
36
-
34
# For profile run
35
_MAX_FRAMES_PER_VIDEO = 32
36
_MAX_NUM_VIDEOS = 1
vllm/model_executor/models/minicpmv.py
-3
@@ -37,7 +37,6 @@ from transformers import PretrainedConfig
37
from vllm.attention import AttentionMetadata
38
from vllm.config import CacheConfig, MultiModalConfig
39
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
40
-from vllm.logger import init_logger
40
from vllm.model_executor.layers.linear import ReplicatedLinear
41
from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -59,8 +58,6 @@ from vllm.sequence import IntermediateTensors, SequenceData
58
59
from .idefics2_vision_model import Idefics2VisionTransformer
60
62
-logger = init_logger(__name__)
63
-
61
_KEYS_TO_MODIFY_MAPPING = {
62
"llm.lm_head": "lm_head",
63
"llm.model": "llm",
vllm/model_executor/models/siglip.py
+4
-7
@@ -501,6 +501,7 @@ class SiglipVisionModel(nn.Module):
501
num_hidden_layers_override: Optional[int] = None,
502
):
503
super().__init__()
504
+
505
num_heads = config.num_attention_heads
506
tp_size = get_tensor_model_parallel_world_size()
507
self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
@@ -511,10 +512,6 @@ class SiglipVisionModel(nn.Module):
512
num_hidden_layers_override=num_hidden_layers_override,
513
)
514
514
- @property
515
- def _require_post_layernorm(self) -> bool:
516
- return self.vision_model.post_layernorm is not None
517
-
515
def get_input_embeddings(self) -> nn.Module:
516
return self.vision_model.embeddings.patch_embedding
517
@@ -540,12 +537,12 @@ class SiglipVisionModel(nn.Module):
537
538
for name, loaded_weight in weights:
539
# post_layernorm is optional in SiglipVisionModel
543
- if ("vision_model.post_layernorm" in name
544
- and not self._require_post_layernorm):
540
+ if (name.startswith("vision_model.post_layernorm")
541
+ and self.vision_model.post_layernorm is None):
542
continue
543
544
# omit layers when num_hidden_layers_override is set
548
- if "vision_model.encoder.layers." in name:
545
+ if name.startswith("vision_model.encoder.layers"):
546
layer_idx = int(name.split(".")[3])
547
if layer_idx >= layer_count:
548
continue
vllm/model_executor/models/ultravox.py
-3
@@ -20,7 +20,6 @@ from vllm.config import CacheConfig, MultiModalConfig
20
from vllm.inputs import INPUT_REGISTRY
21
from vllm.inputs.data import LLMInputs
22
from vllm.inputs.registry import InputContext
23
-from vllm.logger import init_logger
23
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
24
from vllm.model_executor.layers.layernorm import RMSNorm
25
from vllm.model_executor.layers.quantization.base_config import (
@@ -43,8 +42,6 @@ from vllm.transformers_utils.configs.ultravox import UltravoxConfig
42
_AUDIO_PLACEHOLDER_TOKEN = 128002
43
_AUDIO_TOKENS_PER_SECOND = 6.25
44
46
-logger = init_logger(__name__)
47
-
45
46
class UltravoxAudioFeatureInputs(TypedDict):
47
type: Literal["audio_features"]