A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model] Refactor and decouple weight loading logic for InternVL2 model (#7067)
Isotr0py committed
Aug 3, 2024 at 13:36 UTC
0c25435daa0a399460a676e7c9b604bd23ea2d22
2 files changed
+38
-55
vllm/model_executor/models/intern_vit.py
+10
-1
@@ -4,7 +4,7 @@
4
# Copyright (c) 2023 OpenGVLab
5
# Licensed under The MIT License [see LICENSE for details]
6
# --------------------------------------------------------
7
-from typing import Optional
7
+from typing import Iterable, Optional, Tuple
8
9
import torch
10
import torch.nn as nn
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm
16
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
RowParallelLinear)
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
21
NORM2FN = {
22
'rms_norm': RMSNorm,
@@ -268,3 +269,11 @@ class InternVisionModel(nn.Module):
269
encoder_outputs = self.encoder(inputs_embeds=hidden_states)
270
271
return encoder_outputs
272
+
273
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
274
+ params_dict = dict(self.named_parameters())
275
+ for name, loaded_weight in weights:
276
+ param = params_dict[name]
277
+ weight_loader = getattr(param, "weight_loader",
278
+ default_weight_loader)
279
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/internvl.py
+28
-54
@@ -4,6 +4,7 @@
4
# Copyright (c) 2023 OpenGVLab
5
# Licensed under The MIT License [see LICENSE for details]
6
# --------------------------------------------------------
7
+import itertools
8
from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
9
10
import torch
@@ -414,58 +415,31 @@ class InternVLChatModel(nn.Module, SupportsVision):
415
) -> Optional[SamplerOutput]:
416
return self.language_model.sample(logits, sampling_metadata)
417
417
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
418
- stacked_params_mapping = [
419
- # (param_name, shard_name, shard_id)
420
- (".qkv_proj", ".q_proj", "q"),
421
- (".qkv_proj", ".k_proj", "k"),
422
- (".qkv_proj", ".v_proj", "v"),
423
- (".gate_up_proj", ".gate_proj", 0),
424
- (".gate_up_proj", ".up_proj", 1),
425
- (".gate_up_proj", ".w1", 0),
426
- (".gate_up_proj", ".w3", 1),
427
- ]
428
- params_dict = dict(self.named_parameters())
418
+ def _filter_weights(self, weights: Iterable[Tuple[str, torch.Tensor]],
419
+ prefix: str):
420
for name, loaded_weight in weights:
430
- if "rotary_emb.inv_freq" in name:
431
- continue
432
- if self.config.text_config.tie_word_embeddings \
433
- and "lm_head.weight" in name:
434
- continue
435
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
436
- # We only do sharding for language model
437
- # and not vision model for now.
438
- if "vision_embed_tokens" in name and self.vision_embed_tokens:
439
- continue
440
- if weight_name not in name:
441
- continue
442
- param = params_dict[name.replace(weight_name, param_name)]
443
- weight_loader = param.weight_loader
444
- weight_loader(param, loaded_weight, shard_id)
445
- break
446
- else:
447
- # Skip loading extra bias for GPTQ models.
448
- if name.endswith(".bias") and name not in params_dict:
449
- continue
450
- param = params_dict[name]
451
- if "wqkv" in name:
452
- config = self.config.text_config
453
- kv_groups = (config.num_attention_heads //
454
- config.num_key_value_heads)
455
- head_dim = config.hidden_size // config.num_attention_heads
456
- loaded_weight = loaded_weight.view(-1, 2 + kv_groups,
457
- head_dim,
458
- loaded_weight.shape[-1])
459
- wq, wk, wv = torch.split(loaded_weight, [kv_groups, 1, 1],
460
- dim=1)
461
- wq = wq.reshape(-1, wq.shape[-1])
462
- wk = wk.reshape(-1, wk.shape[-1])
463
- wv = wv.reshape(-1, wv.shape[-1])
464
- weight_loader = param.weight_loader
465
- weight_loader(param, wq, 'q')
466
- weight_loader(param, wk, 'k')
467
- weight_loader(param, wv, 'v')
468
- continue
469
- weight_loader = getattr(param, "weight_loader",
470
- default_weight_loader)
471
- weight_loader(param, loaded_weight)
421
+ name = name.split(".")
422
+ if prefix == name.pop(0):
423
+ name = ".".join(name)
424
+ yield name, loaded_weight
425
+
426
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
427
+ # prepare weight iterators for components
428
+ vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
429
+
430
+ # load vision encoder
431
+ vit_weights = self._filter_weights(vit_weights, "vision_model")
432
+ self.vision_model.load_weights(vit_weights)
433
+
434
+ # load mlp projector
435
+ mlp_weights = self._filter_weights(mlp_weights, "mlp1")
436
+ mlp_params_dict = dict(self.mlp1.named_parameters())
437
+ for name, loaded_weight in mlp_weights:
438
+ param = mlp_params_dict[name]
439
+ weight_loader = getattr(param, "weight_loader",
440
+ default_weight_loader)
441
+ weight_loader(param, loaded_weight)
442
+
443
+ # load llm backbone
444
+ llm_weights = self._filter_weights(llm_weights, "language_model")
445
+ self.language_model.load_weights(llm_weights)