A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Model] Add PaliGemma (#5189)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Roger Wang committed
Jul 6, 2024 at 18:25 UTC
6206dcb29eb99b3eebf5f00c97a5690c9b7df4f1
6 files changed
+557
-2
docs/source/models/supported_models.rst
+4
@@ -186,6 +186,10 @@ Vision Language Models
186
- LLaVA-NeXT
187
- :code:`llava-hf/llava-v1.6-mistral-7b-hf`, :code:`llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
188
-
189
+ * - :code:`PaliGemmaForConditionalGeneration`
190
+ - PaliGemma
191
+ - :code:`google/paligemma-3b-pt-224`, :code:`google/paligemma-3b-mix-224`, etc.
192
+ -
193
* - :code:`Phi3VForCausalLM`
194
- Phi-3-Vision
195
- :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
examples/paligemma_example.py
new
+52
@@ -0,0 +1,52 @@
1
+import os
2
+import subprocess
3
+
4
+from PIL import Image
5
+
6
+from vllm import LLM
7
+
8
+# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9
+# You can use `.buildkite/download-images.sh` to download them
10
+
11
+
12
+def run_paligemma():
13
+ llm = LLM(model="google/paligemma-3b-mix-224")
14
+
15
+ prompt = "caption es"
16
+
17
+ image = Image.open("images/stop_sign.jpg")
18
+
19
+ outputs = llm.generate({
20
+ "prompt": prompt,
21
+ "multi_modal_data": {
22
+ "image": image
23
+ },
24
+ })
25
+
26
+ for o in outputs:
27
+ generated_text = o.outputs[0].text
28
+ print(generated_text)
29
+
30
+
31
+def main():
32
+ run_paligemma()
33
+
34
+
35
+if __name__ == "__main__":
36
+ # Download from s3
37
+ s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
38
+ local_directory = "images"
39
+
40
+ # Make sure the local directory exists or create it
41
+ os.makedirs(local_directory, exist_ok=True)
42
+
43
+ # Use AWS CLI to sync the directory, assume anonymous access
44
+ subprocess.check_call([
45
+ "aws",
46
+ "s3",
47
+ "sync",
48
+ s3_bucket_path,
49
+ local_directory,
50
+ "--no-sign-request",
51
+ ])
52
+ main()
tests/models/test_paligemma.py
new
+147
@@ -0,0 +1,147 @@
1
+from typing import List, Optional, Tuple, Type
2
+
3
+import pytest
4
+from transformers import AutoTokenizer
5
+
6
+from vllm.multimodal.utils import rescale_image_size
7
+from vllm.sequence import SampleLogprobs
8
+
9
+from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
+from .utils import check_logprobs_close
11
+
12
+pytestmark = pytest.mark.vlm
13
+
14
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
15
+ "stop_sign": "caption es",
16
+ "cherry_blossom": "What is in the picture?",
17
+ "boardwalk": "What is in the picture?",
18
+})
19
+
20
+IMAGE_TOKEN_ID = 257152
21
+
22
+models = ["google/paligemma-3b-mix-224"]
23
+
24
+
25
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
26
+ Optional[SampleLogprobs]],
27
+ model: str):
28
+ """Sanitize vllm output to be comparable with hf output."""
29
+ output_ids, output_str, out_logprobs = vllm_output
30
+
31
+ tokenizer = AutoTokenizer.from_pretrained(model)
32
+ eos_token_id = tokenizer.eos_token_id
33
+
34
+ hf_output_ids = [
35
+ token_id for idx, token_id in enumerate(output_ids)
36
+ if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
37
+ ]
38
+
39
+ hf_output_str = output_str
40
+
41
+ if hf_output_ids[-1] == eos_token_id:
42
+ hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
43
+
44
+ return hf_output_ids, hf_output_str, out_logprobs
45
+
46
+
47
+def run_test(
48
+ hf_runner: Type[HfRunner],
49
+ vllm_runner: Type[VllmRunner],
50
+ image_assets: _ImageAssets,
51
+ model: str,
52
+ *,
53
+ size_factors: List[float],
54
+ dtype: str,
55
+ max_tokens: int,
56
+ num_logprobs: int,
57
+ tensor_parallel_size: int,
58
+ distributed_executor_backend: Optional[str] = None,
59
+):
60
+ """Inference result should be the same between hf and vllm.
61
+
62
+ All the image fixtures for the test is under tests/images.
63
+ For huggingface runner, we provide the PIL images as input.
64
+ For vllm runner, we provide MultiModalDataDict objects
65
+ and corresponding vision language config as input.
66
+ Note, the text input is also adjusted to abide by vllm contract.
67
+ The text output is sanitized to be able to compare with hf.
68
+ """
69
+ images = [asset.pil_image for asset in image_assets]
70
+
71
+ inputs_per_image = [(
72
+ [prompt for _ in size_factors],
73
+ [rescale_image_size(image, factor) for factor in size_factors],
74
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
75
+
76
+ # NOTE: take care of the order. run vLLM first, and then run HF.
77
+ # vLLM needs a fresh new process without cuda initialization.
78
+ # if we run HF first, the cuda initialization will be done and it
79
+ # will hurt multiprocessing backend with fork method (the default method).
80
+
81
+ # max_model_len should be greater than image_feature_size
82
+ with vllm_runner(model,
83
+ dtype=dtype,
84
+ tensor_parallel_size=tensor_parallel_size,
85
+ distributed_executor_backend=distributed_executor_backend,
86
+ enforce_eager=True) as vllm_model:
87
+ vllm_outputs_per_image = [
88
+ vllm_model.generate_greedy_logprobs(prompts,
89
+ max_tokens,
90
+ num_logprobs=num_logprobs,
91
+ images=images)
92
+ for prompts, images in inputs_per_image
93
+ ]
94
+
95
+ with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
96
+ hf_outputs_per_image = [
97
+ hf_model.generate_greedy_logprobs_limit(prompts,
98
+ max_tokens,
99
+ num_logprobs=num_logprobs,
100
+ images=images)
101
+ for prompts, images in inputs_per_image
102
+ ]
103
+
104
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
105
+ vllm_outputs_per_image):
106
+
107
+ check_logprobs_close(
108
+ outputs_0_lst=hf_outputs,
109
+ outputs_1_lst=[
110
+ vllm_to_hf_output(vllm_output, model)
111
+ for vllm_output in vllm_outputs
112
+ ],
113
+ name_0="hf",
114
+ name_1="vllm",
115
+ )
116
+
117
+
118
+@pytest.mark.parametrize("model", models)
119
+@pytest.mark.parametrize(
120
+ "size_factors",
121
+ [
122
+ # No image
123
+ [],
124
+ # Single-scale
125
+ [1.0],
126
+ # Single-scale, batched
127
+ [1.0, 1.0, 1.0],
128
+ # Multi-scale
129
+ [0.25, 0.5, 1.0],
130
+ ],
131
+)
132
+@pytest.mark.parametrize("dtype", ["float"])
133
+@pytest.mark.parametrize("max_tokens", [128])
134
+@pytest.mark.parametrize("num_logprobs", [5])
135
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
136
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
137
+ run_test(
138
+ hf_runner,
139
+ vllm_runner,
140
+ image_assets,
141
+ model,
142
+ size_factors=size_factors,
143
+ dtype=dtype,
144
+ max_tokens=max_tokens,
145
+ num_logprobs=num_logprobs,
146
+ tensor_parallel_size=1,
147
+ )
vllm/model_executor/models/__init__.py
+2
@@ -49,6 +49,8 @@ _GENERATION_MODELS = {
49
"OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
50
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
51
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
52
+ "PaliGemmaForConditionalGeneration":
53
+ ("paligemma", "PaliGemmaForConditionalGeneration"),
54
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
55
"Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
56
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
vllm/model_executor/models/gemma.py
+8
-2
@@ -268,16 +268,22 @@ class GemmaModel(nn.Module):
268
normalizer = self.config.hidden_size**0.5
269
self.register_buffer("normalizer", torch.tensor(normalizer))
270
271
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
272
+ return self.embed_tokens(input_ids)
273
+
274
def forward(
275
self,
276
input_ids: torch.Tensor,
277
positions: torch.Tensor,
278
kv_caches: List[torch.Tensor],
279
attn_metadata: AttentionMetadata,
280
+ inputs_embeds: Optional[torch.Tensor] = None,
281
) -> torch.Tensor:
278
- hidden_states = self.embed_tokens(input_ids)
282
+ if inputs_embeds is not None:
283
+ hidden_states = inputs_embeds
284
+ else:
285
+ hidden_states = self.get_input_embeddings(input_ids)
286
hidden_states *= self.normalizer
280
-
287
residual = None
288
for i in range(len(self.layers)):
289
layer = self.layers[i]
vllm/model_executor/models/paligemma.py
new
+344
@@ -0,0 +1,344 @@
1
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
2
+
3
+import torch
4
+from PIL import Image
5
+from torch import nn
6
+from transformers import PaliGemmaConfig, SiglipVisionConfig, SiglipVisionModel
7
+
8
+from vllm.attention import AttentionMetadata
9
+from vllm.config import CacheConfig, MultiModalConfig
10
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11
+from vllm.logger import init_logger
12
+from vllm.model_executor.layers.linear import ColumnParallelLinear
13
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
14
+from vllm.model_executor.layers.quantization.base_config import (
15
+ QuantizationConfig)
16
+from vllm.model_executor.layers.sampler import Sampler
17
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
+from vllm.model_executor.models.gemma import GemmaModel
19
+from vllm.model_executor.sampling_metadata import SamplingMetadata
20
+from vllm.multimodal import MULTIMODAL_REGISTRY
21
+from vllm.multimodal.image import cached_get_tokenizer
22
+from vllm.sequence import SamplerOutput, SequenceData
23
+
24
+from .interfaces import SupportsVision
25
+from .utils import merge_vision_embeddings
26
+
27
+logger = init_logger(__name__)
28
+
29
+_KEYS_TO_MODIFY_MAPPING = {
30
+ "language_model.model": "language_model",
31
+}
32
+
33
+
34
+def get_max_paligemma_image_tokens(ctx: InputContext):
35
+ hf_config = ctx.get_hf_config(PaliGemmaConfig)
36
+ text_config = hf_config.text_config
37
+
38
+ return text_config.num_image_tokens
39
+
40
+
41
+def dummy_seq_data_for_paligemma(
42
+ hf_config: PaliGemmaConfig,
43
+ seq_len: int,
44
+ *,
45
+ image_token_id: int,
46
+ image_feature_size_override: Optional[int] = None,
47
+):
48
+ if image_feature_size_override is None:
49
+ image_feature_size = hf_config.text_config.num_image_tokens
50
+ else:
51
+ image_feature_size = image_feature_size_override
52
+
53
+ token_ids = [image_token_id] * image_feature_size
54
+ token_ids += [0] * (seq_len - image_feature_size)
55
+ return SequenceData(token_ids)
56
+
57
+
58
+def dummy_image_for_paligemma(
59
+ hf_config: SiglipVisionConfig,
60
+ *,
61
+ image_width_override: Optional[int] = None,
62
+ image_height_override: Optional[int] = None,
63
+):
64
+ width = height = hf_config.image_size
65
+ if image_width_override is not None:
66
+ width = image_width_override
67
+ if image_height_override is not None:
68
+ height = image_height_override
69
+
70
+ image = Image.new("RGB", (width, height), color=0)
71
+ return {"image": image}
72
+
73
+
74
+def dummy_data_for_paligemma(ctx: InputContext, seq_len: int):
75
+ hf_config = ctx.get_hf_config(PaliGemmaConfig)
76
+ vision_config = hf_config.vision_config
77
+
78
+ seq_data = dummy_seq_data_for_paligemma(
79
+ hf_config,
80
+ seq_len,
81
+ image_token_id=hf_config.image_token_index,
82
+ )
83
+
84
+ mm_data = dummy_image_for_paligemma(vision_config)
85
+ return seq_data, mm_data
86
+
87
+
88
+def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
89
+
90
+ """
91
+ The correct prompt format needs to be:
92
+ '<image>' * image_feature_size + '<bos>' + prompt + '\n'
93
+
94
+ See https://github.com/huggingface/transformers/blob/25245ec26dc29bcf6102e1b4ddd0dfd02e720cf5/src/transformers/models/paligemma/processing_paligemma.py#L55
95
+ """ # noqa
96
+
97
+ multi_modal_data = llm_inputs.get("multi_modal_data")
98
+ if multi_modal_data is None or "image" not in multi_modal_data:
99
+ return llm_inputs
100
+
101
+ model_config = ctx.model_config
102
+ hf_config = ctx.get_hf_config(PaliGemmaConfig)
103
+
104
+ tokenizer = cached_get_tokenizer(model_config.tokenizer)
105
+ image_feature_size = hf_config.text_config.num_image_tokens
106
+ image_token_str = tokenizer.decode(hf_config.image_token_index)
107
+ bos_token = tokenizer.decode(hf_config.bos_token_id)
108
+ image_token_str_pad = image_token_str * image_feature_size
109
+ image_token_ids_pad = [hf_config.image_token_index] * image_feature_size
110
+
111
+ orig_prompt = llm_inputs.get("prompt")
112
+ orig_prompt_ids = llm_inputs.get("prompt_token_ids")
113
+
114
+ if image_token_str in orig_prompt:
115
+ logger.warning(
116
+ "The image token '%s' was detected in the prompt and "
117
+ "will be removed. Please follow the proper prompt format"
118
+ " documented on HuggingFace.", image_token_str)
119
+ orig_prompt = orig_prompt.replace(image_token_str, "")
120
+ orig_prompt_ids.remove(hf_config.image_token_index)
121
+
122
+ new_prompt = f"{image_token_str_pad}{bos_token}{orig_prompt}\n"
123
+ new_token_ids = image_token_ids_pad + orig_prompt_ids + [108] #newline
124
+
125
+ # NOTE: Create a defensive copy of the original inputs
126
+ return LLMInputs(prompt_token_ids=new_token_ids,
127
+ prompt=new_prompt,
128
+ multi_modal_data=multi_modal_data)
129
+
130
+
131
+class PaliGemmaMultiModalProjector(nn.Module):
132
+
133
+ def __init__(self, vision_hidden_size: int, projection_dim: int):
134
+ super().__init__()
135
+
136
+ self.linear = ColumnParallelLinear(vision_hidden_size,
137
+ projection_dim,
138
+ bias=True)
139
+
140
+ def forward(self, image_features: torch.Tensor) -> torch.Tensor:
141
+ hidden_states, _ = self.linear(image_features)
142
+ return hidden_states
143
+
144
+
145
+class PaliGemmaImagePixelInputs(TypedDict):
146
+ type: Literal["pixel_values"]
147
+ data: torch.Tensor
148
+ """Shape: (batch_size, num_channels, height, width)"""
149
+
150
+
151
+PaliGemmaImageInputs = PaliGemmaImagePixelInputs
152
+
153
+
154
+@MULTIMODAL_REGISTRY.register_image_input_mapper()
155
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_paligemma_image_tokens)
156
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_paligemma)
157
+@INPUT_REGISTRY.register_input_processor(input_processor_for_paligemma)
158
+class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
159
+
160
+ def __init__(self,
161
+ config: PaliGemmaConfig,
162
+ multimodal_config: MultiModalConfig,
163
+ cache_config: Optional[CacheConfig] = None,
164
+ quant_config: Optional[QuantizationConfig] = None) -> None:
165
+ super().__init__()
166
+
167
+ self.config = config
168
+ self.multimodal_config = multimodal_config
169
+
170
+ # TODO(ywang96): Port over SiglipVisionModel & TP
171
+ self.vision_tower = SiglipVisionModel(config.vision_config)
172
+ self.multi_modal_projector = PaliGemmaMultiModalProjector(
173
+ vision_hidden_size=config.vision_config.hidden_size,
174
+ projection_dim=config.vision_config.projection_dim)
175
+
176
+ self.quant_config = quant_config
177
+ self.language_model = GemmaModel(config.text_config, cache_config,
178
+ quant_config)
179
+ self.unpadded_vocab_size = config.text_config.vocab_size
180
+ logit_scale = getattr(config, "logit_scale", 1.0)
181
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
182
+ config.vocab_size, logit_scale)
183
+ self.sampler = Sampler()
184
+
185
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
186
+ h = w = self.config.vision_config.image_size
187
+ expected_dims = (3, h, w)
188
+ actual_dims = tuple(data.shape[1:])
189
+
190
+ if actual_dims != expected_dims:
191
+ expected_expr = ("batch_size", *map(str, expected_dims))
192
+ raise ValueError(
193
+ f"The expected shape of pixel values is {expected_expr}. "
194
+ f"You supplied {tuple(data.shape)}.")
195
+
196
+ return data
197
+
198
+ def _parse_and_validate_image_input(
199
+ self, **kwargs: object) -> Optional[PaliGemmaImageInputs]:
200
+ pixel_values = kwargs.pop("pixel_values", None)
201
+
202
+ if pixel_values is None:
203
+ return None
204
+
205
+ if not isinstance(pixel_values, torch.Tensor):
206
+ raise ValueError("Incorrect type of pixel values. "
207
+ f"Got type: {type(pixel_values)}")
208
+
209
+ return PaliGemmaImagePixelInputs(
210
+ type="pixel_values",
211
+ data=self._validate_pixel_values(pixel_values),
212
+ )
213
+
214
+ def _image_pixels_to_features(self, vision_tower: SiglipVisionModel,
215
+ pixel_values: torch.Tensor) -> torch.Tensor:
216
+
217
+ image_outputs = vision_tower(pixel_values, output_hidden_states=True)
218
+
219
+ selected_image_features = image_outputs.last_hidden_state
220
+
221
+ return selected_image_features
222
+
223
+ def _process_image_pixels(
224
+ self, inputs: PaliGemmaImagePixelInputs) -> torch.Tensor:
225
+ assert self.vision_tower is not None
226
+
227
+ pixel_values = inputs["data"]
228
+
229
+ return self._image_pixels_to_features(self.vision_tower, pixel_values)
230
+
231
+ def _process_image_input(
232
+ self, image_input: PaliGemmaImageInputs) -> torch.Tensor:
233
+
234
+ assert self.vision_tower is not None
235
+ image_features = self._process_image_pixels(image_input)
236
+
237
+ return self.multi_modal_projector(image_features)
238
+
239
+ def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
240
+ kv_caches: List[torch.Tensor],
241
+ attn_metadata: AttentionMetadata,
242
+ **kwargs: object) -> SamplerOutput:
243
+
244
+ parsed_image_input = self._parse_and_validate_image_input(**kwargs)
245
+
246
+ if parsed_image_input is not None:
247
+ vision_embeddings = self._process_image_input(parsed_image_input)
248
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/models/paligemma/modeling_paligemma.py#L294 # noqa
249
+ vision_embeddings = vision_embeddings * (self.config.hidden_size**
250
+ -0.5)
251
+
252
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
253
+
254
+ inputs_embeds = merge_vision_embeddings(
255
+ input_ids, inputs_embeds, vision_embeddings,
256
+ self.config.image_token_index)
257
+
258
+ input_ids = None
259
+ else:
260
+ inputs_embeds = None
261
+
262
+ hidden_states = self.language_model(input_ids,
263
+ positions,
264
+ kv_caches,
265
+ attn_metadata,
266
+ inputs_embeds=inputs_embeds)
267
+
268
+ return hidden_states
269
+
270
+ # Copied from vllm/model_executor/models/gemma.py
271
+ def compute_logits(self, hidden_states: torch.Tensor,
272
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
273
+ logits = self.logits_processor(self.language_model.embed_tokens,
274
+ hidden_states, sampling_metadata)
275
+ return logits
276
+
277
+ # Copied from vllm/model_executor/models/gemma.py
278
+ def sample(
279
+ self,
280
+ logits: torch.Tensor,
281
+ sampling_metadata: SamplingMetadata,
282
+ ) -> Optional[SamplerOutput]:
283
+ next_tokens = self.sampler(logits, sampling_metadata)
284
+ return next_tokens
285
+
286
+ # Adapted from vllm/model_executor/models/gemma.py
287
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
288
+ stacked_params_mapping = [
289
+ # (param_name, shard_name, shard_id)
290
+ ("qkv_proj", "q_proj", "q"),
291
+ ("qkv_proj", "k_proj", "k"),
292
+ ("qkv_proj", "v_proj", "v"),
293
+ ("gate_up_proj", "gate_proj", 0),
294
+ ("gate_up_proj", "up_proj", 1),
295
+ ]
296
+ params_dict = dict(self.named_parameters())
297
+ loaded_params = set()
298
+ for name, loaded_weight in weights:
299
+ for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
300
+ if key_to_modify in name:
301
+ name = name.replace(key_to_modify, new_key)
302
+ use_default_weight_loading = False
303
+ if "vision" in name:
304
+ if self.vision_tower is not None:
305
+ # We only do sharding for language model and
306
+ # not vision model for now.
307
+ use_default_weight_loading = True
308
+ else:
309
+ for (param_name, shard_name,
310
+ shard_id) in stacked_params_mapping:
311
+ if shard_name not in name:
312
+ continue
313
+ name = name.replace(shard_name, param_name)
314
+ # Skip loading extra bias for GPTQ models.
315
+ if name.endswith(".bias") and name not in params_dict:
316
+ continue
317
+ param = params_dict[name]
318
+ weight_loader = param.weight_loader
319
+ weight_loader(param, loaded_weight, shard_id)
320
+ break
321
+ else:
322
+ # lm_head is not used in vllm as it is tied with
323
+ # embed_token. To prevent errors, skip loading
324
+ # lm_head.weight.
325
+ if "lm_head.weight" in name:
326
+ continue
327
+ # Skip loading extra bias for GPTQ models.
328
+ if name.endswith(".bias") and name not in params_dict:
329
+ continue
330
+ use_default_weight_loading = True
331
+
332
+ if use_default_weight_loading:
333
+ param = params_dict[name]
334
+ weight_loader = getattr(param, "weight_loader",
335
+ default_weight_loader)
336
+ weight_loader(param, loaded_weight)
337
+
338
+ loaded_params.add(name)
339
+
340
+ unloaded_params = params_dict.keys() - loaded_params
341
+ if unloaded_params:
342
+ raise RuntimeError(
343
+ "Some weights are not initialized from checkpoints: "
344
+ f"{unloaded_params}")