@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 06ed2815e2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Model] Refactor BLIP/BLIP-2 to support composite model loading (#8407)

Cyrus Leung committed Sep 22, 2024 at 20:24 UTC 06ed2815e2be50e527839c7ab09ce2639b7910b6
10 files changed +113 -114
vllm/model_executor/models/blip.py
+58 -3
@@ -1,6 +1,6 @@
1 """Minimal implementation of BlipVisionModel intended to be only used
2 within a vision language model."""
3 -from typing import Optional, Union
3 +from typing import Iterable, Optional, Tuple, Union
4
5 import torch
6 import torch.nn as nn
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 QKVParallelLinear,
17 RowParallelLinear)
18 from vllm.model_executor.layers.quantization import QuantizationConfig
19 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20 from vllm.multimodal.utils import (cached_get_tokenizer,
21 repeat_and_pad_placeholder_tokens)
22 from vllm.sequence import SequenceData
@@ -342,6 +343,10 @@ class BlipVisionModel(nn.Module):
343 num_hidden_layers_override: Optional[int] = None):
344 super().__init__()
345
346 + tp_size = get_tensor_model_parallel_world_size()
347 + num_heads = config.num_attention_heads
348 + self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
349 +
350 self.config = config
351
352 self.embeddings = BlipVisionEmbeddings(config)
@@ -350,11 +355,61 @@ class BlipVisionModel(nn.Module):
355 quant_config=quant_config,
356 num_hidden_layers_override=num_hidden_layers_override,
357 )
353 - self.post_layernorm = nn.LayerNorm(config.hidden_size,
354 - eps=config.layer_norm_eps)
358 +
359 + if len(self.encoder.layers) > config.num_hidden_layers:
360 + raise ValueError(
361 + f"The original encoder only has {config.num_hidden_layers} "
362 + f"layers, but you requested {len(self.encoder.layers)} layers."
363 + )
364 + elif len(self.encoder.layers) == config.num_hidden_layers:
365 + self.post_layernorm = nn.LayerNorm(config.hidden_size,
366 + eps=config.layer_norm_eps)
367 + else:
368 + # post_layernorm is unused when we extract intermediate features
369 + # In this case, we can skip it to conserve memory
370 + self.post_layernorm = None
371
372 def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
373 hidden_states = self.embeddings(pixel_values)
374 hidden_states = self.encoder(inputs_embeds=hidden_states)
375
376 + if self.post_layernorm is None:
377 + return hidden_states
378 +
379 return self.post_layernorm(hidden_states)
380 +
381 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
382 + stacked_params_mapping = [
383 + # (param_name, shard_name, shard_id)
384 + ("qkv_proj", "q_proj", "q"),
385 + ("qkv_proj", "k_proj", "k"),
386 + ("qkv_proj", "v_proj", "v"),
387 + ] if self.shard_weight else []
388 + params_dict = dict(self.named_parameters())
389 + layer_count = len(self.encoder.layers)
390 +
391 + for name, loaded_weight in weights:
392 + # post_layernorm is not needed in BlipVisionModel
393 + if (name.startswith("post_layernorm")
394 + and self.post_layernorm is None):
395 + continue
396 +
397 + # omit layers when num_hidden_layers_override is set
398 + if name.startswith("encoder.layers"):
399 + layer_idx = int(name.split(".")[2])
400 + if layer_idx >= layer_count:
401 + continue
402 +
403 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
404 + if weight_name not in name:
405 + continue
406 +
407 + param = params_dict[name.replace(weight_name, param_name)]
408 + weight_loader = param.weight_loader
409 + weight_loader(param, loaded_weight, shard_id)
410 + break
411 + else:
412 + param = params_dict[name]
413 + weight_loader = getattr(param, "weight_loader",
414 + default_weight_loader)
415 + weight_loader(param, loaded_weight)
vllm/model_executor/models/blip2.py
+47 -74
@@ -10,11 +10,9 @@ from vllm.attention import AttentionMetadata
10 from vllm.config import CacheConfig, MultiModalConfig
11 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12 from vllm.model_executor.layers.activation import get_act_fn
13 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
13 from vllm.model_executor.layers.quantization import QuantizationConfig
15 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
14 +from vllm.model_executor.layers.sampler import SamplerOutput
15 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 -from vllm.model_executor.models.opt import OPTModel
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.sequence import IntermediateTensors, SequenceData
@@ -22,12 +20,8 @@ from vllm.sequence import IntermediateTensors, SequenceData
20 from .blip import (BlipVisionModel, dummy_image_for_blip,
21 get_max_blip_image_tokens)
22 from .interfaces import SupportsMultiModal
25 -from .utils import merge_multimodal_embeddings
26 -
27 -_KEYS_TO_MODIFY_MAPPING = {
28 - "language_model.lm_head": "lm_head",
29 - "language_model.model": "language_model",
30 -}
23 +from .utils import (group_weights_with_prefix, init_vllm_registered_model,
24 + merge_multimodal_embeddings)
25
26 # We use this internally as placeholders since there is no image token
27 # defined on the HuggingFace repo
@@ -491,9 +485,6 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
485
486 super().__init__()
487
494 - # currently all existing BLIP-2 models have `tie_word_embeddings`
495 - # enabled
496 - assert config.tie_word_embeddings
488 self.config = config
489 self.multimodal_config = multimodal_config
490
@@ -514,17 +505,8 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
505 bias=True,
506 )
507
517 - self.quant_config = quant_config
518 -
519 - self.language_model = OPTModel(config.text_config, cache_config,
520 - quant_config)
521 -
522 - self.unpadded_vocab_size = config.text_config.vocab_size
523 - self.logits_processor = LogitsProcessor(self.unpadded_vocab_size)
524 - self.sampler = Sampler()
525 -
526 - def get_lm_head(self):
527 - return self.language_model.decoder.embed_tokens
508 + self.language_model = init_vllm_registered_model(
509 + config.text_config, cache_config, quant_config)
510
511 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
512 h = w = self.config.vision_config.image_size
@@ -653,7 +635,8 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
635
636 if image_input is not None:
637 vision_embeddings = self._process_image_input(image_input)
656 - inputs_embeds = self.language_model.get_input_embeddings(input_ids)
638 + inputs_embeds = self.language_model.model.get_input_embeddings(
639 + input_ids)
640
641 inputs_embeds = merge_multimodal_embeddings(
642 input_ids, inputs_embeds, vision_embeddings,
@@ -663,11 +646,11 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
646 else:
647 inputs_embeds = None
648
666 - hidden_states = self.language_model(input_ids,
667 - positions,
668 - kv_caches,
669 - attn_metadata,
670 - inputs_embeds=inputs_embeds)
649 + hidden_states = self.language_model.model(input_ids,
650 + positions,
651 + kv_caches,
652 + attn_metadata,
653 + inputs_embeds=inputs_embeds)
654
655 return hidden_states
656
@@ -676,56 +659,46 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
659 hidden_states: torch.Tensor,
660 sampling_metadata: SamplingMetadata,
661 ) -> Optional[torch.Tensor]:
679 - logits = self.logits_processor(self.get_lm_head(), hidden_states,
680 - sampling_metadata)
681 - return logits
662 + return self.language_model.compute_logits(hidden_states,
663 + sampling_metadata)
664
665 def sample(
666 self,
667 logits: torch.Tensor,
668 sampling_metadata: SamplingMetadata,
669 ) -> Optional[SamplerOutput]:
688 - next_tokens = self.sampler(logits, sampling_metadata)
689 - return next_tokens
670 + return self.language_model.sample(logits, sampling_metadata)
671
672 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
692 - # only doing this for language model part for now.
693 - stacked_params_mapping = [
694 - # (param_name, shard_name, shard_id)
695 - ("qkv_proj", "q_proj", "q"),
696 - ("qkv_proj", "k_proj", "k"),
697 - ("qkv_proj", "v_proj", "v"),
698 - ("gate_up_proj", "gate_proj", 0),
699 - ("gate_up_proj", "up_proj", 1),
700 - ]
701 - params_dict = dict(self.named_parameters())
702 -
703 - for name, loaded_weight in weights:
704 - if "lm_head.weight" in name:
705 - continue
706 - if "rotary_emb.inv_freq" in name:
707 - continue
708 - for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
709 - if key_to_modify in name:
710 - name = name.replace(key_to_modify, new_key)
711 - use_default_weight_loading = False
712 - if "vision" in name:
713 - if self.vision_model is not None:
714 - # BlipVisionModel does not need sharding
715 - use_default_weight_loading = True
716 - else:
717 - for (param_name, weight_name,
718 - shard_id) in stacked_params_mapping:
719 - if weight_name not in name:
720 - continue
721 - param = params_dict[name.replace(weight_name, param_name)]
722 - weight_loader = param.weight_loader
723 - weight_loader(param, loaded_weight, shard_id)
724 - break
725 - else:
726 - use_default_weight_loading = True
727 - if use_default_weight_loading:
728 - param = params_dict[name]
729 - weight_loader = getattr(param, "weight_loader",
730 - default_weight_loader)
731 - weight_loader(param, loaded_weight)
673 + # prepare weight iterators for components
674 + weights_group = group_weights_with_prefix(weights)
675 +
676 + # load vision encoder
677 + self.vision_model.load_weights(weights_group["vision_model"])
678 +
679 + # load query tokens
680 + for name, loaded_weight in weights_group["query_tokens"]:
681 + assert name == ""
682 + param = self.query_tokens
683 + weight_loader = getattr(param, "weight_loader",
684 + default_weight_loader)
685 + weight_loader(param, loaded_weight)
686 +
687 + # load qformer
688 + qformer_params_dict = dict(self.qformer.named_parameters())
689 + for name, loaded_weight in weights_group["qformer"]:
690 + param = qformer_params_dict[name]
691 + weight_loader = getattr(param, "weight_loader",
692 + default_weight_loader)
693 + weight_loader(param, loaded_weight)
694 +
695 + # load mlp projector
696 + mlp_params_dict = dict(self.language_projection.named_parameters())
697 + for name, loaded_weight in weights_group["language_projection"]:
698 + param = mlp_params_dict[name]
699 + weight_loader = getattr(param, "weight_loader",
700 + default_weight_loader)
701 + weight_loader(param, loaded_weight)
702 +
703 + # load llm backbone
704 + self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/chameleon.py
-3
@@ -12,7 +12,6 @@ from vllm.attention import Attention, AttentionMetadata
12 from vllm.config import CacheConfig, MultiModalConfig
13 from vllm.distributed import get_tensor_model_parallel_world_size
14 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15 -from vllm.logger import init_logger
15 from vllm.model_executor.layers.activation import SiluAndMul
16 from vllm.model_executor.layers.layernorm import RMSNorm
17 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -36,8 +35,6 @@ from vllm.utils import print_warning_once
35
36 from .interfaces import SupportsMultiModal
37
39 -logger = init_logger(__name__)
40 -
38 # These configs are not part of the model config but the preprocessor
39 # and processor files, so we hardcode them in the model file for now.
40 CHAMELEON_CROP_SIZE_HEIGHT = CHAMELEON_CROP_SIZE_WIDTH = 512
vllm/model_executor/models/clip.py
+4 -7
@@ -391,6 +391,7 @@ class CLIPVisionModel(nn.Module):
391 quant_config: Optional[QuantizationConfig] = None,
392 num_hidden_layers_override: Optional[int] = None):
393 super().__init__()
394 +
395 tp_size = get_tensor_model_parallel_world_size()
396 num_heads = config.num_attention_heads
397 self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
@@ -400,10 +401,6 @@ class CLIPVisionModel(nn.Module):
401 quant_config=quant_config,
402 num_hidden_layers_override=num_hidden_layers_override)
403
403 - @property
404 - def _require_post_layernorm(self) -> bool:
405 - return self.vision_model.post_layernorm is not None
406 -
404 def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
405 return self.vision_model(pixel_values)
406
@@ -425,12 +422,12 @@ class CLIPVisionModel(nn.Module):
422
423 for name, loaded_weight in weights:
424 # post_layernorm is not needed in CLIPVisionModel
428 - if ("vision_model.post_layernorm" in name
429 - and not self._require_post_layernorm):
425 + if (name.startswith("vision_model.post_layernorm")
426 + and self.vision_model.post_layernorm is None):
427 continue
428
429 # omit layers when num_hidden_layers_override is set
433 - if "vision_model.encoder.layers." in name:
430 + if name.startswith("vision_model.encoder.layers"):
431 layer_idx = int(name.split(".")[3])
432 if layer_idx >= layer_count:
433 continue
vllm/model_executor/models/fuyu.py
-3
@@ -28,7 +28,6 @@ from transformers import FuyuConfig, FuyuImageProcessor
28 from vllm.attention import AttentionMetadata
29 from vllm.config import CacheConfig, MultiModalConfig
30 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31 -from vllm.logger import init_logger
31 from vllm.model_executor.layers.linear import ColumnParallelLinear
32 from vllm.model_executor.layers.quantization import QuantizationConfig
33 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -45,8 +44,6 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
44 from .interfaces import SupportsMultiModal
45 from .utils import merge_multimodal_embeddings
46
48 -logger = init_logger(__name__)
49 -
47 # Cannot find the following 2 numbers from hf config.
48 _IMAGE_TOKEN_ID = 71011
49 _NEWLINE_TOKEN_ID = 71019
vllm/model_executor/models/llava_next.py
-8
@@ -12,7 +12,6 @@ from typing_extensions import NotRequired
12 from vllm.attention import AttentionMetadata
13 from vllm.config import CacheConfig, MultiModalConfig
14 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15 -from vllm.logger import init_logger
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.model_executor.layers.sampler import SamplerOutput
17 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -32,13 +31,6 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31 from .utils import (flatten_bn, group_weights_with_prefix,
32 init_vllm_registered_model, merge_multimodal_embeddings)
33
35 -logger = init_logger(__name__)
36 -
37 -_KEYS_TO_MODIFY_MAPPING = {
38 - "language_model.lm_head": "lm_head",
39 - "language_model.model": "language_model",
40 -}
41 -
34 # Result in the max possible feature size (2x2 grid of 336x336px tiles)
35 MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
36
vllm/model_executor/models/llava_next_video.py
-3
@@ -11,7 +11,6 @@ from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
11 from vllm.attention import AttentionMetadata
12 from vllm.config import CacheConfig, MultiModalConfig
13 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14 -from vllm.logger import init_logger
14 from vllm.model_executor.layers.activation import get_act_fn
15 from vllm.model_executor.layers.quantization.base_config import (
16 QuantizationConfig)
@@ -32,8 +31,6 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31 from .utils import (group_weights_with_prefix, init_vllm_registered_model,
32 merge_multimodal_embeddings)
33
35 -logger = init_logger(__name__)
36 -
34 # For profile run
35 _MAX_FRAMES_PER_VIDEO = 32
36 _MAX_NUM_VIDEOS = 1
vllm/model_executor/models/minicpmv.py
-3
@@ -37,7 +37,6 @@ from transformers import PretrainedConfig
37 from vllm.attention import AttentionMetadata
38 from vllm.config import CacheConfig, MultiModalConfig
39 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
40 -from vllm.logger import init_logger
40 from vllm.model_executor.layers.linear import ReplicatedLinear
41 from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -59,8 +58,6 @@ from vllm.sequence import IntermediateTensors, SequenceData
58
59 from .idefics2_vision_model import Idefics2VisionTransformer
60
62 -logger = init_logger(__name__)
63 -
61 _KEYS_TO_MODIFY_MAPPING = {
62 "llm.lm_head": "lm_head",
63 "llm.model": "llm",
vllm/model_executor/models/siglip.py
+4 -7
@@ -501,6 +501,7 @@ class SiglipVisionModel(nn.Module):
501 num_hidden_layers_override: Optional[int] = None,
502 ):
503 super().__init__()
504 +
505 num_heads = config.num_attention_heads
506 tp_size = get_tensor_model_parallel_world_size()
507 self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
@@ -511,10 +512,6 @@ class SiglipVisionModel(nn.Module):
512 num_hidden_layers_override=num_hidden_layers_override,
513 )
514
514 - @property
515 - def _require_post_layernorm(self) -> bool:
516 - return self.vision_model.post_layernorm is not None
517 -
515 def get_input_embeddings(self) -> nn.Module:
516 return self.vision_model.embeddings.patch_embedding
517
@@ -540,12 +537,12 @@ class SiglipVisionModel(nn.Module):
537
538 for name, loaded_weight in weights:
539 # post_layernorm is optional in SiglipVisionModel
543 - if ("vision_model.post_layernorm" in name
544 - and not self._require_post_layernorm):
540 + if (name.startswith("vision_model.post_layernorm")
541 + and self.vision_model.post_layernorm is None):
542 continue
543
544 # omit layers when num_hidden_layers_override is set
548 - if "vision_model.encoder.layers." in name:
545 + if name.startswith("vision_model.encoder.layers"):
546 layer_idx = int(name.split(".")[3])
547 if layer_idx >= layer_count:
548 continue
vllm/model_executor/models/ultravox.py
-3
@@ -20,7 +20,6 @@ from vllm.config import CacheConfig, MultiModalConfig
20 from vllm.inputs import INPUT_REGISTRY
21 from vllm.inputs.data import LLMInputs
22 from vllm.inputs.registry import InputContext
23 -from vllm.logger import init_logger
23 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
24 from vllm.model_executor.layers.layernorm import RMSNorm
25 from vllm.model_executor.layers.quantization.base_config import (
@@ -43,8 +42,6 @@ from vllm.transformers_utils.configs.ultravox import UltravoxConfig
42 _AUDIO_PLACEHOLDER_TOKEN = 128002
43 _AUDIO_TOKENS_PER_SECOND = 6.25
44
46 -logger = init_logger(__name__)
47 -
45
46 class UltravoxAudioFeatureInputs(TypedDict):
47 type: Literal["audio_features"]