A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model][VLM] Decouple weight loading logic for `Paligemma` (#8269)
Isotr0py committed
Sep 8, 2024 at 01:45 UTC
36bf8150cc3a048d69d9d2196128462014b9599d
2 files changed
+54
-81
vllm/model_executor/models/paligemma.py
+35
-77
@@ -1,3 +1,4 @@
1
+import itertools
2
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3
TypedDict, Union)
4
@@ -13,7 +14,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
14
from vllm.model_executor.layers.quantization import QuantizationConfig
15
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
-from vllm.model_executor.models.gemma import GemmaModel
17
+from vllm.model_executor.models.gemma import GemmaForCausalLM
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
from vllm.multimodal.utils import cached_get_tokenizer
@@ -22,14 +23,10 @@ from vllm.sequence import IntermediateTensors
23
from .interfaces import SupportsMultiModal
24
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
25
dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
25
-from .utils import merge_multimodal_embeddings
26
+from .utils import filter_weights, merge_multimodal_embeddings
27
28
logger = init_logger(__name__)
29
29
-_KEYS_TO_MODIFY_MAPPING = {
30
- "language_model.model": "language_model",
31
-}
32
-
30
31
class PaliGemmaImagePixelInputs(TypedDict):
32
type: Literal["pixel_values"]
@@ -151,8 +148,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
148
projection_dim=config.vision_config.projection_dim)
149
150
self.quant_config = quant_config
154
- self.language_model = GemmaModel(config.text_config, cache_config,
155
- quant_config)
151
+ self.language_model = GemmaForCausalLM(config.text_config,
152
+ cache_config, quant_config)
153
self.unpadded_vocab_size = config.text_config.vocab_size
154
logit_scale = getattr(config, "logit_scale", 1.0)
155
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
@@ -252,7 +249,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
249
vision_embeddings = vision_embeddings * (self.config.hidden_size**
250
-0.5)
251
255
- inputs_embeds = self.language_model.get_input_embeddings(input_ids)
252
+ inputs_embeds = self.language_model.model.get_input_embeddings(
253
+ input_ids)
254
255
inputs_embeds = merge_multimodal_embeddings(
256
input_ids, inputs_embeds, vision_embeddings,
@@ -262,87 +260,47 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
260
else:
261
inputs_embeds = None
262
265
- hidden_states = self.language_model(input_ids,
266
- positions,
267
- kv_caches,
268
- attn_metadata,
269
- None,
270
- inputs_embeds=inputs_embeds)
263
+ hidden_states = self.language_model.model(input_ids,
264
+ positions,
265
+ kv_caches,
266
+ attn_metadata,
267
+ None,
268
+ inputs_embeds=inputs_embeds)
269
270
return hidden_states
271
274
- # Copied from vllm/model_executor/models/gemma.py
272
def compute_logits(
273
self,
274
hidden_states: torch.Tensor,
275
sampling_metadata: SamplingMetadata,
276
) -> Optional[torch.Tensor]:
280
- logits = self.logits_processor(self.language_model.embed_tokens,
281
- hidden_states, sampling_metadata)
282
- return logits
277
+ return self.language_model.compute_logits(hidden_states,
278
+ sampling_metadata)
279
284
- # Copied from vllm/model_executor/models/gemma.py
280
def sample(
281
self,
282
logits: torch.Tensor,
283
sampling_metadata: SamplingMetadata,
284
) -> Optional[SamplerOutput]:
290
- next_tokens = self.sampler(logits, sampling_metadata)
291
- return next_tokens
285
+ return self.language_model.sample(logits, sampling_metadata)
286
293
- # Adapted from vllm/model_executor/models/gemma.py
287
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
295
- stacked_params_mapping = [
296
- # (param_name, shard_name, shard_id)
297
- ("qkv_proj", "q_proj", "q"),
298
- ("qkv_proj", "k_proj", "k"),
299
- ("qkv_proj", "v_proj", "v"),
300
- ("gate_up_proj", "gate_proj", 0),
301
- ("gate_up_proj", "up_proj", 1),
302
- ]
303
- params_dict = dict(self.named_parameters())
304
- loaded_params = set()
305
- for name, loaded_weight in weights:
306
- for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
307
- if key_to_modify in name:
308
- name = name.replace(key_to_modify, new_key)
309
- use_default_weight_loading = False
310
- if "vision" not in name or self.vision_tower.shard_weight:
311
- for (param_name, shard_name,
312
- shard_id) in stacked_params_mapping:
313
- if shard_name not in name:
314
- continue
315
- name = name.replace(shard_name, param_name)
316
- # Skip loading extra bias for GPTQ models.
317
- if name.endswith(".bias") and name not in params_dict:
318
- continue
319
- param = params_dict[name]
320
- weight_loader = param.weight_loader
321
- weight_loader(param, loaded_weight, shard_id)
322
- break
323
- else:
324
- # lm_head is not used in vllm as it is tied with
325
- # embed_token. To prevent errors, skip loading
326
- # lm_head.weight.
327
- if "lm_head.weight" in name:
328
- continue
329
- # Skip loading extra bias for GPTQ models.
330
- if name.endswith(".bias") and name not in params_dict:
331
- continue
332
- use_default_weight_loading = True
333
- else:
334
- use_default_weight_loading = True
335
-
336
- if use_default_weight_loading:
337
- param = params_dict[name]
338
- weight_loader = getattr(param, "weight_loader",
339
- default_weight_loader)
340
- weight_loader(param, loaded_weight)
341
-
342
- loaded_params.add(name)
343
-
344
- unloaded_params = params_dict.keys() - loaded_params
345
- if unloaded_params:
346
- logger.warning(
347
- "Some weights are not initialized from checkpoints: %s",
348
- unloaded_params)
288
+ # prepare weight iterators for components
289
+ vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
290
+
291
+ # load vision tower
292
+ vit_weights = filter_weights(vit_weights, "vision_tower")
293
+ self.vision_tower.load_weights(vit_weights)
294
+
295
+ # load mlp projector
296
+ mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
297
+ mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
298
+ for name, loaded_weight in mlp_weights:
299
+ param = mlp_params_dict[name]
300
+ weight_loader = getattr(param, "weight_loader",
301
+ default_weight_loader)
302
+ weight_loader(param, loaded_weight)
303
+
304
+ # load llm backbone
305
+ llm_weights = filter_weights(llm_weights, "language_model")
306
+ self.language_model.load_weights(llm_weights)
vllm/model_executor/models/siglip.py
+19
-4
@@ -529,6 +529,12 @@ class SiglipVisionModel(nn.Module):
529
)
530
531
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
532
+ stacked_params_mapping = [
533
+ # (param_name, shard_name, shard_id)
534
+ ("qkv_proj", "q_proj", "q"),
535
+ ("qkv_proj", "k_proj", "k"),
536
+ ("qkv_proj", "v_proj", "v"),
537
+ ] if self.shard_weight else []
538
params_dict = dict(self.named_parameters())
539
layer_count = len(self.vision_model.encoder.layers)
540
@@ -544,7 +550,16 @@ class SiglipVisionModel(nn.Module):
550
if layer_idx >= layer_count:
551
continue
552
547
- param = params_dict[name]
548
- weight_loader = getattr(param, "weight_loader",
549
- default_weight_loader)
550
- weight_loader(param, loaded_weight)
553
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
554
+ if weight_name not in name:
555
+ continue
556
+
557
+ param = params_dict[name.replace(weight_name, param_name)]
558
+ weight_loader = param.weight_loader
559
+ weight_loader(param, loaded_weight, shard_id)
560
+ break
561
+ else:
562
+ param = params_dict[name]
563
+ weight_loader = getattr(param, "weight_loader",
564
+ default_weight_loader)
565
+ weight_loader(param, loaded_weight)