A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Model] Initialize Fuyu-8B support (#3924)
Co-authored-by: Roger Wang <ywang@roblox.com>
Isotr0py committed
Jul 14, 2024 at 13:27 UTC
540c0368b14ddd8d3efac0b182761bf6600f104f
6 files changed
+844
docs/source/models/supported_models.rst
+8
@@ -137,6 +137,10 @@ Decoder-only Language Models
137
- Phi-3-Small
138
- :code:`microsoft/Phi-3-small-8k-instruct`, :code:`microsoft/Phi-3-small-128k-instruct`, etc.
139
-
140
+ * - :code:`PersimmonForCausalLM`
141
+ - Persimmon
142
+ - :code:`adept/persimmon-8b-base`, :code:`adept/persimmon-8b-chat`, etc.
143
+ -
144
* - :code:`QWenLMHeadModel`
145
- Qwen
146
- :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
@@ -178,6 +182,10 @@ Vision Language Models
182
- Models
183
- Example HuggingFace Models
184
- :ref:`LoRA <lora>`
185
+ * - :code:`FuyuForCausalLM`
186
+ - Fuyu
187
+ - :code:`adept/fuyu-8b` etc.
188
+ -
189
* - :code:`LlavaForConditionalGeneration`
190
- LLaVA-1.5
191
- :code:`llava-hf/llava-1.5-7b-hf`, :code:`llava-hf/llava-1.5-13b-hf`, etc.
examples/fuyu_example.py
new
+31
@@ -0,0 +1,31 @@
1
+import requests
2
+from PIL import Image
3
+
4
+from vllm import LLM, SamplingParams
5
+
6
+
7
+def run_fuyu():
8
+ llm = LLM(model="adept/fuyu-8b", max_model_len=4096)
9
+
10
+ # single-image prompt
11
+ prompt = "What is the highest life expectancy at of male?\n"
12
+ url = "https://huggingface.co/adept/fuyu-8b/resolve/main/chart.png"
13
+ image = Image.open(requests.get(url, stream=True).raw)
14
+ sampling_params = SamplingParams(temperature=0, max_tokens=64)
15
+
16
+ outputs = llm.generate(
17
+ {
18
+ "prompt": prompt,
19
+ "multi_modal_data": {
20
+ "image": image
21
+ },
22
+ },
23
+ sampling_params=sampling_params)
24
+
25
+ for o in outputs:
26
+ generated_text = o.outputs[0].text
27
+ print(generated_text)
28
+
29
+
30
+if __name__ == "__main__":
31
+ run_fuyu()
tests/models/test_fuyu.py
new
+142
@@ -0,0 +1,142 @@
1
+from typing import List, Optional, Tuple, Type
2
+
3
+import pytest
4
+
5
+from vllm.multimodal.utils import rescale_image_size
6
+from vllm.sequence import SampleLogprobs
7
+from vllm.utils import is_cpu
8
+
9
+from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
+from .utils import check_logprobs_close
11
+
12
+pytestmark = pytest.mark.vlm
13
+
14
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
15
+ "stop_sign": "What's the content of the image?\n", # noqa: E501
16
+ "cherry_blossom": "What is the season?\n",
17
+ "boardwalk": "What's in this image?\n",
18
+})
19
+
20
+models = ["adept/fuyu-8b"]
21
+
22
+
23
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
24
+ Optional[SampleLogprobs]]):
25
+ """Sanitize vllm output to be comparable with hf output."""
26
+ output_ids, output_str, out_logprobs = vllm_output
27
+
28
+ hf_output_str = output_str.lstrip() + "|ENDOFTEXT|"
29
+
30
+ return output_ids, hf_output_str, out_logprobs
31
+
32
+
33
+def run_test(
34
+ hf_runner: Type[HfRunner],
35
+ vllm_runner: Type[VllmRunner],
36
+ image_assets: _ImageAssets,
37
+ model: str,
38
+ *,
39
+ size_factors: List[float],
40
+ dtype: str,
41
+ max_tokens: int,
42
+ num_logprobs: int,
43
+ tensor_parallel_size: int,
44
+ distributed_executor_backend: Optional[str] = None,
45
+):
46
+ """Inference result should be the same between hf and vllm.
47
+
48
+ All the image fixtures for the test is under tests/images.
49
+ For huggingface runner, we provide the PIL images as input.
50
+ For vllm runner, we provide MultiModalDataDict objects
51
+ and corresponding vision language config as input.
52
+ Note, the text input is also adjusted to abide by vllm contract.
53
+ The text output is sanitized to be able to compare with hf.
54
+ """
55
+ images = [asset.pil_image for asset in image_assets]
56
+
57
+ inputs_per_image = [(
58
+ [prompt for _ in size_factors],
59
+ [rescale_image_size(image, factor) for factor in size_factors],
60
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
61
+
62
+ # NOTE: take care of the order. run vLLM first, and then run HF.
63
+ # vLLM needs a fresh new process without cuda initialization.
64
+ # if we run HF first, the cuda initialization will be done and it
65
+ # will hurt multiprocessing backend with fork method (the default method).
66
+
67
+ # max_model_len should be greater than image_feature_size
68
+ with vllm_runner(model,
69
+ max_model_len=2560,
70
+ max_num_seqs=1,
71
+ dtype=dtype,
72
+ tensor_parallel_size=tensor_parallel_size,
73
+ distributed_executor_backend=distributed_executor_backend,
74
+ enforce_eager=True) as vllm_model:
75
+ vllm_outputs_per_image = [
76
+ vllm_model.generate_greedy_logprobs(prompts,
77
+ max_tokens,
78
+ num_logprobs=num_logprobs,
79
+ images=vllm_images)
80
+ for prompts, vllm_images in inputs_per_image
81
+ ]
82
+
83
+ with hf_runner(model, dtype=dtype) as hf_model:
84
+ hf_model.model.get_output_embeddings = lambda: \
85
+ hf_model.model.language_model.get_output_embeddings()
86
+ eos_token_id = hf_model.processor.tokenizer.eos_token_id
87
+ hf_outputs_per_image = [
88
+ hf_model.generate_greedy_logprobs_limit(prompts,
89
+ max_tokens,
90
+ num_logprobs=num_logprobs,
91
+ images=hf_images,
92
+ eos_token_id=eos_token_id)
93
+ for prompts, hf_images in inputs_per_image
94
+ ]
95
+
96
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
97
+ vllm_outputs_per_image):
98
+ check_logprobs_close(
99
+ outputs_0_lst=hf_outputs,
100
+ outputs_1_lst=[
101
+ vllm_to_hf_output(vllm_output) for vllm_output in vllm_outputs
102
+ ],
103
+ name_0="hf",
104
+ name_1="vllm",
105
+ )
106
+
107
+
108
+target_dtype = "half"
109
+if is_cpu():
110
+ target_dtype = "bfloat16"
111
+
112
+
113
+@pytest.mark.parametrize("model", models)
114
+@pytest.mark.parametrize(
115
+ "size_factors",
116
+ [
117
+ # No image
118
+ [],
119
+ # Single-scale
120
+ [0.25],
121
+ # Single-scale, batched
122
+ [0.25, 0.25, 0.25],
123
+ # Multi-scale
124
+ [0.25, 0.2, 0.15],
125
+ ],
126
+)
127
+@pytest.mark.parametrize("dtype", [target_dtype])
128
+@pytest.mark.parametrize("max_tokens", [128])
129
+@pytest.mark.parametrize("num_logprobs", [10])
130
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
131
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
132
+ run_test(
133
+ hf_runner,
134
+ vllm_runner,
135
+ image_assets,
136
+ model,
137
+ size_factors=size_factors,
138
+ dtype=dtype,
139
+ max_tokens=max_tokens,
140
+ num_logprobs=num_logprobs,
141
+ tensor_parallel_size=1,
142
+ )
vllm/model_executor/models/__init__.py
+2
@@ -23,6 +23,7 @@ _GENERATION_MODELS = {
23
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
25
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
26
+ "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
27
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
28
"Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
29
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
@@ -49,6 +50,7 @@ _GENERATION_MODELS = {
50
"OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
51
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
52
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
53
+ "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
54
"PaliGemmaForConditionalGeneration":
55
("paligemma", "PaliGemmaForConditionalGeneration"),
56
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
vllm/model_executor/models/fuyu.py
new
+328
@@ -0,0 +1,328 @@
1
+# coding=utf-8
2
+# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/fuyu/modeling_fuyu.py
3
+# Copyright 2023 The vLLM team.
4
+# Copyright 2023 HuggingFace Inc. team. All rights reserved.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+""" PyTorch Fuyu model."""
18
+import math
19
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
20
+
21
+import torch
22
+import torch.nn as nn
23
+import torch.utils.checkpoint
24
+from PIL import Image
25
+from transformers import FuyuConfig, FuyuImageProcessor
26
+
27
+from vllm.attention import AttentionMetadata
28
+from vllm.config import CacheConfig, MultiModalConfig
29
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30
+from vllm.logger import init_logger
31
+from vllm.model_executor.layers.linear import ColumnParallelLinear
32
+from vllm.model_executor.layers.quantization.base_config import (
33
+ QuantizationConfig)
34
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35
+from vllm.model_executor.models.persimmon import PersimmonForCausalLM
36
+from vllm.model_executor.sampling_metadata import SamplingMetadata
37
+from vllm.multimodal import MULTIMODAL_REGISTRY
38
+from vllm.multimodal.base import MultiModalInputs
39
+from vllm.multimodal.image import (cached_get_image_processor,
40
+ cached_get_tokenizer)
41
+from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
42
+
43
+from .interfaces import SupportsVision
44
+from .utils import merge_vision_embeddings
45
+
46
+logger = init_logger(__name__)
47
+
48
+# Cannot find the following 2 numbers from hf config.
49
+_IMAGE_TOKEN_ID = 71011
50
+_NEWLINE_TOKEN_ID = 71019
51
+
52
+MAX_IMAGE_FEATURE_SIZE_HEIGHT = 1080
53
+MAX_IMAGE_FEATURE_SIZE_WIDTH = 1920
54
+
55
+
56
+class FuyuImagePixelInputs(TypedDict):
57
+ type: Literal["pixel_values"]
58
+ data: torch.Tensor
59
+ """
60
+ Shape:
61
+ (batch_size, num_patches, patch_size_x * patch_size_y * num_channels)
62
+ """
63
+
64
+
65
+def _calculate_num_image_tokens(
66
+ height: int,
67
+ width: int,
68
+) -> Tuple[int, int]:
69
+ """
70
+ calculate number of image tokens needed for a given image size
71
+ The expected Fuyu image prompts is in format:
72
+ (image_token * ncols + newline_token) * nrows
73
+ args:
74
+ image_size: Tuple[int, int] - (width, height) of the image
75
+ returns:
76
+ ncols: int - number of image tokens in x direction
77
+ nrows: int - number of image tokens in y direction
78
+ """
79
+ ncol = math.ceil(width / 30)
80
+ nrow = math.ceil(height / 30)
81
+ return ncol, nrow
82
+
83
+
84
+def get_max_fuyu_image_feature_size():
85
+
86
+ return _calculate_num_image_tokens(
87
+ height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
88
+ width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
89
+ )
90
+
91
+
92
+def get_max_fuyu_image_tokens(ctx: InputContext):
93
+ ncol, nrow = get_max_fuyu_image_feature_size()
94
+ return (ncol + 1) * nrow
95
+
96
+
97
+def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int):
98
+ ncol, nrow = get_max_fuyu_image_feature_size()
99
+ image_feature_size = get_max_fuyu_image_tokens(ctx)
100
+
101
+ token_ids = ([_IMAGE_TOKEN_ID] * ncol + [_NEWLINE_TOKEN_ID]) * nrow
102
+ token_ids += [0] * (seq_len - image_feature_size)
103
+ return SequenceData(token_ids)
104
+
105
+
106
+def dummy_image_for_fuyu(
107
+ image_width: int,
108
+ image_height: int,
109
+):
110
+ image = Image.new("RGB", (image_width, image_height), color=0)
111
+ return {"image": image}
112
+
113
+
114
+def dummy_data_for_fuyu(ctx: InputContext, seq_len: int):
115
+ seq_data = dummy_seq_data_for_fuyu(ctx, seq_len)
116
+ mm_data = dummy_image_for_fuyu(MAX_IMAGE_FEATURE_SIZE_WIDTH,
117
+ MAX_IMAGE_FEATURE_SIZE_HEIGHT)
118
+ return seq_data, mm_data
119
+
120
+
121
+def _fuyu_image_preprocess(image_processor: FuyuImageProcessor,
122
+ data: Image.Image):
123
+ image_encoding = image_processor.preprocess(data, return_tensors="pt")
124
+ batch_images = torch.stack([img[0] for img in image_encoding["images"]
125
+ ]).unsqueeze(1)
126
+ image_unpadded_heights = torch.tensor(
127
+ image_encoding["image_unpadded_heights"])
128
+ image_unpadded_widths = torch.tensor(
129
+ image_encoding["image_unpadded_widths"])
130
+
131
+ batch_size = len(image_encoding["images"])
132
+ image_present = torch.ones(batch_size, 1, 1)
133
+ model_image_input = image_processor.preprocess_with_tokenizer_info(
134
+ image_input=batch_images,
135
+ image_present=image_present,
136
+ image_unpadded_h=image_unpadded_heights,
137
+ image_unpadded_w=image_unpadded_widths,
138
+ image_placeholder_id=_IMAGE_TOKEN_ID,
139
+ image_newline_id=_NEWLINE_TOKEN_ID,
140
+ variable_sized=True,
141
+ )
142
+ return model_image_input
143
+
144
+
145
+def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
146
+ multi_modal_data = llm_inputs.get("multi_modal_data")
147
+ if multi_modal_data is None or "image" not in multi_modal_data:
148
+ return llm_inputs
149
+
150
+ model_config = ctx.model_config
151
+ image_data = multi_modal_data["image"]
152
+ new_multi_modal_data = {}
153
+ # process image data
154
+ if isinstance(image_data, Image.Image):
155
+ # Fuyu's image_processor can also finish token padding
156
+ image_processor: FuyuImageProcessor = cached_get_image_processor(
157
+ model_config.model)
158
+
159
+ model_image_input = _fuyu_image_preprocess(image_processor, image_data)
160
+ image_patches = torch.stack([
161
+ image_patch[0]
162
+ for image_patch in model_image_input["image_patches"]
163
+ ])
164
+ new_multi_modal_data["image"] = image_patches
165
+
166
+ elif isinstance(image_data, torch.Tensor):
167
+ raise NotImplementedError("Embeddings input is not supported yet")
168
+ else:
169
+ raise TypeError(f"Invalid image type: {type(image_data)}")
170
+
171
+ # process prompts
172
+ prompt = llm_inputs["prompt"]
173
+ prompt_token_ids = llm_inputs["prompt_token_ids"]
174
+ tokenizer = cached_get_tokenizer(model_config.model)
175
+ # dim0 is batch_size, dim1 is subseq_size which will always be 1
176
+ image_input_ids: List[List[
177
+ torch.Tensor]] = model_image_input["image_input_ids"]
178
+ image_input_ids = image_input_ids[0][0].tolist()
179
+ bos_token = tokenizer.encode("<s>", add_special_tokens=False)[1:]
180
+ boa_token = tokenizer.encode("\x04", add_special_tokens=False)[1:]
181
+
182
+ new_prompt = prompt + "\x04"
183
+ new_prompt_token_ids = image_input_ids + bos_token + prompt_token_ids[
184
+ 1:] + boa_token
185
+
186
+ return LLMInputs(prompt=new_prompt,
187
+ prompt_token_ids=new_prompt_token_ids,
188
+ multi_modal_data=new_multi_modal_data)
189
+
190
+
191
+def input_mapper_for_fuyu(ctx: InputContext, data: object):
192
+ model_config = ctx.model_config
193
+ if isinstance(data, Image.Image):
194
+ # Fuyu's image_processor can also finish token padding
195
+ image_processor: FuyuImageProcessor = cached_get_image_processor(
196
+ model_config.model)
197
+
198
+ model_image_input = _fuyu_image_preprocess(image_processor, data)
199
+ data = torch.stack([
200
+ image_patch[0]
201
+ for image_patch in model_image_input["image_patches"]
202
+ ])
203
+
204
+ # image has been processed with prompt in input processor
205
+ return MultiModalInputs({"image_patches": data})
206
+
207
+
208
+@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_fuyu)
209
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_fuyu_image_tokens)
210
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_fuyu)
211
+@INPUT_REGISTRY.register_input_processor(input_processor_for_fuyu)
212
+class FuyuForCausalLM(nn.Module, SupportsVision):
213
+
214
+ def __init__(self,
215
+ config: FuyuConfig,
216
+ multimodal_config: MultiModalConfig,
217
+ cache_config: Optional[CacheConfig] = None,
218
+ quant_config: Optional[QuantizationConfig] = None) -> None:
219
+ super().__init__()
220
+ self.config = config
221
+ self.multimodal_config = multimodal_config
222
+
223
+ self.padding_idx = config.pad_token_id
224
+ self.vocab_size = config.vocab_size
225
+ self.image_token_id = _IMAGE_TOKEN_ID
226
+ self.image_feature_size = config.patch_size**2 * config.num_channels
227
+
228
+ self.vision_embed_tokens = ColumnParallelLinear(
229
+ self.image_feature_size,
230
+ config.hidden_size,
231
+ quant_config=quant_config,
232
+ )
233
+ self.language_model = PersimmonForCausalLM(config,
234
+ cache_config=cache_config,
235
+ quant_config=quant_config)
236
+
237
+ def _parse_and_validate_image_input(self, **kwargs: object):
238
+ image_patches = kwargs.pop("image_patches", None)
239
+
240
+ if isinstance(image_patches, torch.Tensor):
241
+ expected_feature_size = self.image_feature_size
242
+ if image_patches.size(-1) != expected_feature_size:
243
+ raise ValueError(
244
+ f"Expected image patches to have the last dimension of "
245
+ f"{expected_feature_size}, got {image_patches.size(-1)}")
246
+ image_patches = image_patches.to(
247
+ self.vision_embed_tokens.weight.dtype)
248
+ return FuyuImagePixelInputs(type="pixel_values",
249
+ data=image_patches)
250
+ return None
251
+
252
+ def forward(
253
+ self,
254
+ input_ids: torch.Tensor,
255
+ positions: torch.Tensor,
256
+ kv_caches: List[torch.Tensor],
257
+ attn_metadata: AttentionMetadata,
258
+ intermediate_tensors: Optional[IntermediateTensors] = None,
259
+ **kwargs: object,
260
+ ):
261
+ image_input = self._parse_and_validate_image_input(**kwargs)
262
+
263
+ if image_input is not None:
264
+ vision_embeddings, _ = self.vision_embed_tokens(
265
+ image_input["data"])
266
+ inputs_embeds = self.language_model.model.embed_tokens(input_ids)
267
+ inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
268
+ vision_embeddings,
269
+ self.image_token_id)
270
+
271
+ else:
272
+ inputs_embeds = None
273
+
274
+ hidden_states = self.language_model(
275
+ input_ids=input_ids,
276
+ positions=positions,
277
+ kv_caches=kv_caches,
278
+ attn_metadata=attn_metadata,
279
+ inputs_embeds=inputs_embeds,
280
+ )
281
+ return hidden_states
282
+
283
+ def compute_logits(self, hidden_states: torch.Tensor,
284
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
285
+ logits = self.language_model.logits_processor(
286
+ self.language_model.lm_head, hidden_states, sampling_metadata)
287
+ return logits
288
+
289
+ def sample(
290
+ self,
291
+ logits: torch.Tensor,
292
+ sampling_metadata: SamplingMetadata,
293
+ ) -> Optional[SamplerOutput]:
294
+ next_tokens = self.language_model.sampler(logits, sampling_metadata)
295
+ return next_tokens
296
+
297
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
298
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
299
+ for name, loaded_weight in weights:
300
+ if "rotary_emb.inv_freq" in name:
301
+ continue
302
+ if ("rotary_emb.cos_cached" in name
303
+ or "rotary_emb.sin_cached" in name):
304
+ # Models trained using ColossalAI may include these tensors in
305
+ # the checkpoint. Skip them.
306
+ continue
307
+ param = params_dict[name]
308
+
309
+ if "query_key_value" in name:
310
+ # copy from vllm/model_executor/models/bloom.py
311
+ # NOTE: Fuyu's fused QKV's output_dim has the shape of
312
+ # (num_heads * 3 * head_size), while the
313
+ # required shape is (3 * num_heads * head_size).
314
+ # Thus, we need weight conversion.
315
+ output_dim = getattr(param, "output_dim", None)
316
+ num_heads = self.config.num_attention_heads
317
+ if output_dim is not None:
318
+ loaded_weight_shape = loaded_weight.shape
319
+ loaded_weight = loaded_weight.view(
320
+ loaded_weight_shape[:output_dim] + (num_heads, 3, -1) +
321
+ loaded_weight_shape[output_dim + 1:])
322
+ loaded_weight = loaded_weight.transpose(
323
+ output_dim, output_dim + 1)
324
+ loaded_weight = loaded_weight.reshape(loaded_weight_shape)
325
+
326
+ weight_loader = getattr(param, "weight_loader",
327
+ default_weight_loader)
328
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/persimmon.py
new
+333
@@ -0,0 +1,333 @@
1
+# coding=utf-8
2
+# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/persimmon/modeling_persimmon.py
3
+# Copyright 2023 The vLLM team.
4
+# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5
+#
6
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7
+# and OPT implementations in this library. It has been modified from its
8
+# original forms to accommodate minor architectural differences compared
9
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10
+#
11
+# Licensed under the Apache License, Version 2.0 (the "License");
12
+# you may not use this file except in compliance with the License.
13
+# You may obtain a copy of the License at
14
+#
15
+# http://www.apache.org/licenses/LICENSE-2.0
16
+#
17
+# Unless required by applicable law or agreed to in writing, software
18
+# distributed under the License is distributed on an "AS IS" BASIS,
19
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20
+# See the License for the specific language governing permissions and
21
+# limitations under the License.
22
+"""Inference-only persimmon model compatible with HuggingFace weights."""
23
+from typing import Iterable, List, Optional, Tuple
24
+
25
+import torch
26
+from torch import nn
27
+from transformers import PersimmonConfig
28
+from transformers.activations import ReLUSquaredActivation
29
+
30
+from vllm.attention import Attention, AttentionMetadata
31
+from vllm.config import CacheConfig
32
+from vllm.distributed import get_tensor_model_parallel_world_size
33
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
+ QKVParallelLinear,
35
+ RowParallelLinear)
36
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
+from vllm.model_executor.layers.quantization.base_config import (
38
+ QuantizationConfig)
39
+from vllm.model_executor.layers.rotary_embedding import get_rope
40
+from vllm.model_executor.layers.sampler import Sampler
41
+from vllm.model_executor.layers.vocab_parallel_embedding import (
42
+ ParallelLMHead, VocabParallelEmbedding)
43
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
+from vllm.sequence import IntermediateTensors, SamplerOutput
46
+
47
+
48
+class PersimmonMLP(nn.Module):
49
+
50
+ def __init__(self,
51
+ config: PersimmonConfig,
52
+ quant_config: Optional[QuantizationConfig] = None):
53
+ super().__init__()
54
+ self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
55
+ config.intermediate_size,
56
+ quant_config=quant_config)
57
+ self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
58
+ config.hidden_size,
59
+ quant_config=quant_config)
60
+ self.act = ReLUSquaredActivation()
61
+
62
+ def forward(self, hidden_states) -> torch.Tensor:
63
+ hidden_states, _ = self.dense_h_to_4h(hidden_states)
64
+ hidden_states = self.act(hidden_states)
65
+ hidden_states, _ = self.dense_4h_to_h(hidden_states)
66
+ return hidden_states
67
+
68
+
69
+class PersimmonAttention(nn.Module):
70
+
71
+ def __init__(self,
72
+ config: PersimmonConfig,
73
+ cache_config: Optional[CacheConfig] = None,
74
+ quant_config: Optional[QuantizationConfig] = None):
75
+ super().__init__()
76
+ self.config = config
77
+ tensor_parallel_world_size = get_tensor_model_parallel_world_size()
78
+
79
+ self.hidden_size = config.hidden_size
80
+ self.total_num_heads = config.num_attention_heads
81
+ self.num_heads = self.total_num_heads // tensor_parallel_world_size
82
+ self.head_dim = self.hidden_size // self.total_num_heads
83
+ self.max_position_embeddings = config.max_position_embeddings
84
+ self.rope_theta = config.rope_theta
85
+ self.partial_rotary_factor = config.partial_rotary_factor
86
+ self.is_causal = True
87
+
88
+ assert (self.head_dim * self.total_num_heads) == self.hidden_size
89
+ assert self.total_num_heads % tensor_parallel_world_size == 0
90
+
91
+ self.query_key_value = QKVParallelLinear(
92
+ self.hidden_size,
93
+ self.head_dim,
94
+ self.total_num_heads,
95
+ bias=True,
96
+ quant_config=quant_config,
97
+ )
98
+ self.dense = RowParallelLinear(
99
+ self.num_heads * self.head_dim,
100
+ self.hidden_size,
101
+ bias=True,
102
+ quant_config=quant_config,
103
+ )
104
+ self.is_qk_layernorm = config.qk_layernorm
105
+
106
+ if self.is_qk_layernorm:
107
+ self.q_layernorm = nn.LayerNorm(self.head_dim)
108
+ self.k_layernorm = nn.LayerNorm(self.head_dim)
109
+
110
+ self.rotary_emb = get_rope(
111
+ self.head_dim,
112
+ rotary_dim=int(self.partial_rotary_factor * self.head_dim),
113
+ max_position=self.max_position_embeddings,
114
+ base=self.rope_theta,
115
+ )
116
+ self.scaling = self.head_dim**-0.5
117
+ self.attn = Attention(self.num_heads,
118
+ self.head_dim,
119
+ scale=self.scaling,
120
+ cache_config=cache_config,
121
+ quant_config=quant_config)
122
+
123
+ def _split_heads(self, x: torch.Tensor) -> torch.Tensor:
124
+ # [seq_length, hidden_size] -> [seq_length, num_heads, head_dim]
125
+ seq_length = x.shape[0]
126
+ return x.view(seq_length, self.num_heads, self.head_dim)
127
+
128
+ def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
129
+ # [seq_length, num_heads, head_dim] -> [seq_length, hidden_size]
130
+ seq_length = x.shape[0]
131
+ return x.view(seq_length, self.num_heads * self.head_dim)
132
+
133
+ def forward(
134
+ self,
135
+ position_ids: torch.Tensor,
136
+ hidden_states: torch.Tensor,
137
+ kv_cache: torch.Tensor,
138
+ attn_metadata: AttentionMetadata,
139
+ ) -> torch.Tensor:
140
+ # [seq_length, 3 x hidden_size]
141
+ qkv, _ = self.query_key_value(hidden_states)
142
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
143
+
144
+ if self.is_qk_layernorm:
145
+ # [seq_length, num_heads, head_dim]
146
+ q = self._split_heads(q)
147
+ k = self._split_heads(k)
148
+
149
+ q = self.q_layernorm(q)
150
+ k = self.k_layernorm(k)
151
+
152
+ q = self._merge_heads(q)
153
+ k = self._merge_heads(k)
154
+
155
+ q, k = self.rotary_emb(position_ids, q, k)
156
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
157
+ output, _ = self.dense(attn_output)
158
+ return output
159
+
160
+
161
+class PersimmonDecoderLayer(nn.Module):
162
+
163
+ def __init__(self,
164
+ config: PersimmonConfig,
165
+ cache_config: Optional[CacheConfig] = None,
166
+ quant_config: Optional[QuantizationConfig] = None):
167
+ super().__init__()
168
+ self.hidden_size = config.hidden_size
169
+ self.self_attn = PersimmonAttention(config=config,
170
+ cache_config=cache_config,
171
+ quant_config=quant_config)
172
+ self.mlp = PersimmonMLP(config, quant_config=quant_config)
173
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
174
+ eps=config.layer_norm_eps)
175
+ self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
176
+ eps=config.layer_norm_eps)
177
+
178
+ def forward(
179
+ self,
180
+ position_ids: torch.Tensor,
181
+ hidden_states: torch.Tensor,
182
+ kv_cache: torch.Tensor,
183
+ attn_metadata: AttentionMetadata,
184
+ ) -> torch.Tensor:
185
+ residual = hidden_states
186
+
187
+ hidden_states = self.input_layernorm(hidden_states)
188
+
189
+ # Self Attention
190
+ hidden_states = self.self_attn(
191
+ position_ids=position_ids,
192
+ hidden_states=hidden_states,
193
+ kv_cache=kv_cache,
194
+ attn_metadata=attn_metadata,
195
+ )
196
+ hidden_states = residual + hidden_states
197
+
198
+ # Fully Connected
199
+ residual = hidden_states
200
+ hidden_states = self.post_attention_layernorm(hidden_states)
201
+ hidden_states = self.mlp(hidden_states)
202
+
203
+ hidden_states = hidden_states + residual
204
+
205
+ outputs = hidden_states
206
+ return outputs
207
+
208
+
209
+class PersimmonModel(nn.Module):
210
+
211
+ def __init__(self,
212
+ config: PersimmonConfig,
213
+ cache_config: Optional[CacheConfig] = None,
214
+ quant_config: Optional[QuantizationConfig] = None):
215
+ super().__init__()
216
+ self.vocab_size = config.vocab_size
217
+
218
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
219
+ config.hidden_size)
220
+ self.layers = nn.ModuleList([
221
+ PersimmonDecoderLayer(config,
222
+ cache_config=cache_config,
223
+ quant_config=quant_config)
224
+ for _ in range(config.num_hidden_layers)
225
+ ])
226
+ self.final_layernorm = nn.LayerNorm(config.hidden_size,
227
+ eps=config.layer_norm_eps)
228
+
229
+ def forward(
230
+ self,
231
+ input_ids: torch.Tensor,
232
+ positions: torch.Tensor,
233
+ kv_caches: List[torch.Tensor],
234
+ attn_metadata: AttentionMetadata,
235
+ inputs_embeds: Optional[torch.Tensor] = None,
236
+ ) -> torch.Tensor:
237
+ if inputs_embeds is not None:
238
+ hidden_states = inputs_embeds
239
+ else:
240
+ hidden_states = self.embed_tokens(input_ids)
241
+ for i in range(len(self.layers)):
242
+ hidden_states = self.layers[i](
243
+ positions,
244
+ hidden_states,
245
+ kv_caches[i],
246
+ attn_metadata,
247
+ )
248
+ hidden_states = self.final_layernorm(hidden_states)
249
+ return hidden_states
250
+
251
+
252
+class PersimmonForCausalLM(nn.Module):
253
+
254
+ def __init__(self,
255
+ config,
256
+ cache_config: Optional[CacheConfig] = None,
257
+ quant_config: Optional[QuantizationConfig] = None):
258
+ super().__init__()
259
+ self.config = config
260
+ self.vocab_size = config.vocab_size
261
+ self.model = PersimmonModel(config,
262
+ cache_config=cache_config,
263
+ quant_config=quant_config)
264
+ self.lm_head = ParallelLMHead(config.vocab_size,
265
+ config.hidden_size,
266
+ bias=False)
267
+ self.logits_processor = LogitsProcessor(config.vocab_size)
268
+ self.sampler = Sampler()
269
+
270
+ def forward(
271
+ self,
272
+ input_ids: torch.Tensor,
273
+ positions: torch.Tensor,
274
+ kv_caches: List[torch.Tensor],
275
+ attn_metadata: AttentionMetadata,
276
+ intermediate_tensors: Optional[IntermediateTensors] = None,
277
+ inputs_embeds: Optional[torch.Tensor] = None,
278
+ ):
279
+ hidden_states = self.model(
280
+ input_ids=input_ids,
281
+ positions=positions,
282
+ kv_caches=kv_caches,
283
+ attn_metadata=attn_metadata,
284
+ inputs_embeds=inputs_embeds,
285
+ )
286
+ return hidden_states
287
+
288
+ def compute_logits(self, hidden_states: torch.Tensor,
289
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
290
+ logits = self.logits_processor(self.lm_head, hidden_states,
291
+ sampling_metadata)
292
+ return logits
293
+
294
+ def sample(
295
+ self,
296
+ logits: torch.Tensor,
297
+ sampling_metadata: SamplingMetadata,
298
+ ) -> Optional[SamplerOutput]:
299
+ next_tokens = self.sampler(logits, sampling_metadata)
300
+ return next_tokens
301
+
302
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
303
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
304
+ for name, loaded_weight in weights:
305
+ if "rotary_emb.inv_freq" in name:
306
+ continue
307
+ if ("rotary_emb.cos_cached" in name
308
+ or "rotary_emb.sin_cached" in name):
309
+ # Models trained using ColossalAI may include these tensors in
310
+ # the checkpoint. Skip them.
311
+ continue
312
+ param = params_dict[name]
313
+
314
+ if "query_key_value" in name:
315
+ # copy from vllm/model_executor/models/bloom.py
316
+ # NOTE: Persimmon's fused QKV's output_dim has the shape of
317
+ # (num_heads * 3 * head_size), while the
318
+ # required shape is (3 * num_heads * head_size).
319
+ # Thus, we need weight conversion.
320
+ output_dim = getattr(param, "output_dim", None)
321
+ num_heads = self.config.num_attention_heads
322
+ if output_dim is not None:
323
+ loaded_weight_shape = loaded_weight.shape
324
+ loaded_weight = loaded_weight.view(
325
+ loaded_weight_shape[:output_dim] + (num_heads, 3, -1) +
326
+ loaded_weight_shape[output_dim + 1:])
327
+ loaded_weight = loaded_weight.transpose(
328
+ output_dim, output_dim + 1)
329
+ loaded_weight = loaded_weight.reshape(loaded_weight_shape)
330
+
331
+ weight_loader = getattr(param, "weight_loader",
332
+ default_weight_loader)
333
+ weight_loader(param, loaded_weight)