@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 36bf8150cc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Model][VLM] Decouple weight loading logic for `Paligemma` (#8269)

Isotr0py committed Sep 8, 2024 at 01:45 UTC 36bf8150cc3a048d69d9d2196128462014b9599d
2 files changed +54 -81
vllm/model_executor/models/paligemma.py
+35 -77
@@ -1,3 +1,4 @@
1 +import itertools
2 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3 TypedDict, Union)
4
@@ -13,7 +14,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
14 from vllm.model_executor.layers.quantization import QuantizationConfig
15 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 -from vllm.model_executor.models.gemma import GemmaModel
17 +from vllm.model_executor.models.gemma import GemmaForCausalLM
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
20 from vllm.multimodal.utils import cached_get_tokenizer
@@ -22,14 +23,10 @@ from vllm.sequence import IntermediateTensors
23 from .interfaces import SupportsMultiModal
24 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
25 dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
25 -from .utils import merge_multimodal_embeddings
26 +from .utils import filter_weights, merge_multimodal_embeddings
27
28 logger = init_logger(__name__)
29
29 -_KEYS_TO_MODIFY_MAPPING = {
30 - "language_model.model": "language_model",
31 -}
32 -
30
31 class PaliGemmaImagePixelInputs(TypedDict):
32 type: Literal["pixel_values"]
@@ -151,8 +148,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
148 projection_dim=config.vision_config.projection_dim)
149
150 self.quant_config = quant_config
154 - self.language_model = GemmaModel(config.text_config, cache_config,
155 - quant_config)
151 + self.language_model = GemmaForCausalLM(config.text_config,
152 + cache_config, quant_config)
153 self.unpadded_vocab_size = config.text_config.vocab_size
154 logit_scale = getattr(config, "logit_scale", 1.0)
155 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
@@ -252,7 +249,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
249 vision_embeddings = vision_embeddings * (self.config.hidden_size**
250 -0.5)
251
255 - inputs_embeds = self.language_model.get_input_embeddings(input_ids)
252 + inputs_embeds = self.language_model.model.get_input_embeddings(
253 + input_ids)
254
255 inputs_embeds = merge_multimodal_embeddings(
256 input_ids, inputs_embeds, vision_embeddings,
@@ -262,87 +260,47 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
260 else:
261 inputs_embeds = None
262
265 - hidden_states = self.language_model(input_ids,
266 - positions,
267 - kv_caches,
268 - attn_metadata,
269 - None,
270 - inputs_embeds=inputs_embeds)
263 + hidden_states = self.language_model.model(input_ids,
264 + positions,
265 + kv_caches,
266 + attn_metadata,
267 + None,
268 + inputs_embeds=inputs_embeds)
269
270 return hidden_states
271
274 - # Copied from vllm/model_executor/models/gemma.py
272 def compute_logits(
273 self,
274 hidden_states: torch.Tensor,
275 sampling_metadata: SamplingMetadata,
276 ) -> Optional[torch.Tensor]:
280 - logits = self.logits_processor(self.language_model.embed_tokens,
281 - hidden_states, sampling_metadata)
282 - return logits
277 + return self.language_model.compute_logits(hidden_states,
278 + sampling_metadata)
279
284 - # Copied from vllm/model_executor/models/gemma.py
280 def sample(
281 self,
282 logits: torch.Tensor,
283 sampling_metadata: SamplingMetadata,
284 ) -> Optional[SamplerOutput]:
290 - next_tokens = self.sampler(logits, sampling_metadata)
291 - return next_tokens
285 + return self.language_model.sample(logits, sampling_metadata)
286
293 - # Adapted from vllm/model_executor/models/gemma.py
287 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
295 - stacked_params_mapping = [
296 - # (param_name, shard_name, shard_id)
297 - ("qkv_proj", "q_proj", "q"),
298 - ("qkv_proj", "k_proj", "k"),
299 - ("qkv_proj", "v_proj", "v"),
300 - ("gate_up_proj", "gate_proj", 0),
301 - ("gate_up_proj", "up_proj", 1),
302 - ]
303 - params_dict = dict(self.named_parameters())
304 - loaded_params = set()
305 - for name, loaded_weight in weights:
306 - for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
307 - if key_to_modify in name:
308 - name = name.replace(key_to_modify, new_key)
309 - use_default_weight_loading = False
310 - if "vision" not in name or self.vision_tower.shard_weight:
311 - for (param_name, shard_name,
312 - shard_id) in stacked_params_mapping:
313 - if shard_name not in name:
314 - continue
315 - name = name.replace(shard_name, param_name)
316 - # Skip loading extra bias for GPTQ models.
317 - if name.endswith(".bias") and name not in params_dict:
318 - continue
319 - param = params_dict[name]
320 - weight_loader = param.weight_loader
321 - weight_loader(param, loaded_weight, shard_id)
322 - break
323 - else:
324 - # lm_head is not used in vllm as it is tied with
325 - # embed_token. To prevent errors, skip loading
326 - # lm_head.weight.
327 - if "lm_head.weight" in name:
328 - continue
329 - # Skip loading extra bias for GPTQ models.
330 - if name.endswith(".bias") and name not in params_dict:
331 - continue
332 - use_default_weight_loading = True
333 - else:
334 - use_default_weight_loading = True
335 -
336 - if use_default_weight_loading:
337 - param = params_dict[name]
338 - weight_loader = getattr(param, "weight_loader",
339 - default_weight_loader)
340 - weight_loader(param, loaded_weight)
341 -
342 - loaded_params.add(name)
343 -
344 - unloaded_params = params_dict.keys() - loaded_params
345 - if unloaded_params:
346 - logger.warning(
347 - "Some weights are not initialized from checkpoints: %s",
348 - unloaded_params)
288 + # prepare weight iterators for components
289 + vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
290 +
291 + # load vision tower
292 + vit_weights = filter_weights(vit_weights, "vision_tower")
293 + self.vision_tower.load_weights(vit_weights)
294 +
295 + # load mlp projector
296 + mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
297 + mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
298 + for name, loaded_weight in mlp_weights:
299 + param = mlp_params_dict[name]
300 + weight_loader = getattr(param, "weight_loader",
301 + default_weight_loader)
302 + weight_loader(param, loaded_weight)
303 +
304 + # load llm backbone
305 + llm_weights = filter_weights(llm_weights, "language_model")
306 + self.language_model.load_weights(llm_weights)
vllm/model_executor/models/siglip.py
+19 -4
@@ -529,6 +529,12 @@ class SiglipVisionModel(nn.Module):
529 )
530
531 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
532 + stacked_params_mapping = [
533 + # (param_name, shard_name, shard_id)
534 + ("qkv_proj", "q_proj", "q"),
535 + ("qkv_proj", "k_proj", "k"),
536 + ("qkv_proj", "v_proj", "v"),
537 + ] if self.shard_weight else []
538 params_dict = dict(self.named_parameters())
539 layer_count = len(self.vision_model.encoder.layers)
540
@@ -544,7 +550,16 @@ class SiglipVisionModel(nn.Module):
550 if layer_idx >= layer_count:
551 continue
552
547 - param = params_dict[name]
548 - weight_loader = getattr(param, "weight_loader",
549 - default_weight_loader)
550 - weight_loader(param, loaded_weight)
553 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
554 + if weight_name not in name:
555 + continue
556 +
557 + param = params_dict[name.replace(weight_name, param_name)]
558 + weight_loader = param.weight_loader
559 + weight_loader(param, loaded_weight, shard_id)
560 + break
561 + else:
562 + param = params_dict[name]
563 + weight_loader = getattr(param, "weight_loader",
564 + default_weight_loader)
565 + weight_loader(param, loaded_weight)