@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 540c0368b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Model] Initialize Fuyu-8B support (#3924)

Co-authored-by: Roger Wang <ywang@roblox.com>

Isotr0py committed Jul 14, 2024 at 13:27 UTC 540c0368b14ddd8d3efac0b182761bf6600f104f
6 files changed +844
docs/source/models/supported_models.rst
+8
@@ -137,6 +137,10 @@ Decoder-only Language Models
137 - Phi-3-Small
138 - :code:`microsoft/Phi-3-small-8k-instruct`, :code:`microsoft/Phi-3-small-128k-instruct`, etc.
139 -
140 + * - :code:`PersimmonForCausalLM`
141 + - Persimmon
142 + - :code:`adept/persimmon-8b-base`, :code:`adept/persimmon-8b-chat`, etc.
143 + -
144 * - :code:`QWenLMHeadModel`
145 - Qwen
146 - :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
@@ -178,6 +182,10 @@ Vision Language Models
182 - Models
183 - Example HuggingFace Models
184 - :ref:`LoRA <lora>`
185 + * - :code:`FuyuForCausalLM`
186 + - Fuyu
187 + - :code:`adept/fuyu-8b` etc.
188 + -
189 * - :code:`LlavaForConditionalGeneration`
190 - LLaVA-1.5
191 - :code:`llava-hf/llava-1.5-7b-hf`, :code:`llava-hf/llava-1.5-13b-hf`, etc.
examples/fuyu_example.py new
+31
@@ -0,0 +1,31 @@
1 +import requests
2 +from PIL import Image
3 +
4 +from vllm import LLM, SamplingParams
5 +
6 +
7 +def run_fuyu():
8 + llm = LLM(model="adept/fuyu-8b", max_model_len=4096)
9 +
10 + # single-image prompt
11 + prompt = "What is the highest life expectancy at of male?\n"
12 + url = "https://huggingface.co/adept/fuyu-8b/resolve/main/chart.png"
13 + image = Image.open(requests.get(url, stream=True).raw)
14 + sampling_params = SamplingParams(temperature=0, max_tokens=64)
15 +
16 + outputs = llm.generate(
17 + {
18 + "prompt": prompt,
19 + "multi_modal_data": {
20 + "image": image
21 + },
22 + },
23 + sampling_params=sampling_params)
24 +
25 + for o in outputs:
26 + generated_text = o.outputs[0].text
27 + print(generated_text)
28 +
29 +
30 +if __name__ == "__main__":
31 + run_fuyu()
tests/models/test_fuyu.py new
+142
@@ -0,0 +1,142 @@
1 +from typing import List, Optional, Tuple, Type
2 +
3 +import pytest
4 +
5 +from vllm.multimodal.utils import rescale_image_size
6 +from vllm.sequence import SampleLogprobs
7 +from vllm.utils import is_cpu
8 +
9 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10 +from .utils import check_logprobs_close
11 +
12 +pytestmark = pytest.mark.vlm
13 +
14 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
15 + "stop_sign": "What's the content of the image?\n", # noqa: E501
16 + "cherry_blossom": "What is the season?\n",
17 + "boardwalk": "What's in this image?\n",
18 +})
19 +
20 +models = ["adept/fuyu-8b"]
21 +
22 +
23 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
24 + Optional[SampleLogprobs]]):
25 + """Sanitize vllm output to be comparable with hf output."""
26 + output_ids, output_str, out_logprobs = vllm_output
27 +
28 + hf_output_str = output_str.lstrip() + "|ENDOFTEXT|"
29 +
30 + return output_ids, hf_output_str, out_logprobs
31 +
32 +
33 +def run_test(
34 + hf_runner: Type[HfRunner],
35 + vllm_runner: Type[VllmRunner],
36 + image_assets: _ImageAssets,
37 + model: str,
38 + *,
39 + size_factors: List[float],
40 + dtype: str,
41 + max_tokens: int,
42 + num_logprobs: int,
43 + tensor_parallel_size: int,
44 + distributed_executor_backend: Optional[str] = None,
45 +):
46 + """Inference result should be the same between hf and vllm.
47 +
48 + All the image fixtures for the test is under tests/images.
49 + For huggingface runner, we provide the PIL images as input.
50 + For vllm runner, we provide MultiModalDataDict objects
51 + and corresponding vision language config as input.
52 + Note, the text input is also adjusted to abide by vllm contract.
53 + The text output is sanitized to be able to compare with hf.
54 + """
55 + images = [asset.pil_image for asset in image_assets]
56 +
57 + inputs_per_image = [(
58 + [prompt for _ in size_factors],
59 + [rescale_image_size(image, factor) for factor in size_factors],
60 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
61 +
62 + # NOTE: take care of the order. run vLLM first, and then run HF.
63 + # vLLM needs a fresh new process without cuda initialization.
64 + # if we run HF first, the cuda initialization will be done and it
65 + # will hurt multiprocessing backend with fork method (the default method).
66 +
67 + # max_model_len should be greater than image_feature_size
68 + with vllm_runner(model,
69 + max_model_len=2560,
70 + max_num_seqs=1,
71 + dtype=dtype,
72 + tensor_parallel_size=tensor_parallel_size,
73 + distributed_executor_backend=distributed_executor_backend,
74 + enforce_eager=True) as vllm_model:
75 + vllm_outputs_per_image = [
76 + vllm_model.generate_greedy_logprobs(prompts,
77 + max_tokens,
78 + num_logprobs=num_logprobs,
79 + images=vllm_images)
80 + for prompts, vllm_images in inputs_per_image
81 + ]
82 +
83 + with hf_runner(model, dtype=dtype) as hf_model:
84 + hf_model.model.get_output_embeddings = lambda: \
85 + hf_model.model.language_model.get_output_embeddings()
86 + eos_token_id = hf_model.processor.tokenizer.eos_token_id
87 + hf_outputs_per_image = [
88 + hf_model.generate_greedy_logprobs_limit(prompts,
89 + max_tokens,
90 + num_logprobs=num_logprobs,
91 + images=hf_images,
92 + eos_token_id=eos_token_id)
93 + for prompts, hf_images in inputs_per_image
94 + ]
95 +
96 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
97 + vllm_outputs_per_image):
98 + check_logprobs_close(
99 + outputs_0_lst=hf_outputs,
100 + outputs_1_lst=[
101 + vllm_to_hf_output(vllm_output) for vllm_output in vllm_outputs
102 + ],
103 + name_0="hf",
104 + name_1="vllm",
105 + )
106 +
107 +
108 +target_dtype = "half"
109 +if is_cpu():
110 + target_dtype = "bfloat16"
111 +
112 +
113 +@pytest.mark.parametrize("model", models)
114 +@pytest.mark.parametrize(
115 + "size_factors",
116 + [
117 + # No image
118 + [],
119 + # Single-scale
120 + [0.25],
121 + # Single-scale, batched
122 + [0.25, 0.25, 0.25],
123 + # Multi-scale
124 + [0.25, 0.2, 0.15],
125 + ],
126 +)
127 +@pytest.mark.parametrize("dtype", [target_dtype])
128 +@pytest.mark.parametrize("max_tokens", [128])
129 +@pytest.mark.parametrize("num_logprobs", [10])
130 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
131 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
132 + run_test(
133 + hf_runner,
134 + vllm_runner,
135 + image_assets,
136 + model,
137 + size_factors=size_factors,
138 + dtype=dtype,
139 + max_tokens=max_tokens,
140 + num_logprobs=num_logprobs,
141 + tensor_parallel_size=1,
142 + )
vllm/model_executor/models/__init__.py
+2
@@ -23,6 +23,7 @@ _GENERATION_MODELS = {
23 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24 "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
25 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
26 + "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
27 "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
28 "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
29 "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
@@ -49,6 +50,7 @@ _GENERATION_MODELS = {
50 "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
51 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
52 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
53 + "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
54 "PaliGemmaForConditionalGeneration":
55 ("paligemma", "PaliGemmaForConditionalGeneration"),
56 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
vllm/model_executor/models/fuyu.py new
+328
@@ -0,0 +1,328 @@
1 +# coding=utf-8
2 +# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/fuyu/modeling_fuyu.py
3 +# Copyright 2023 The vLLM team.
4 +# Copyright 2023 HuggingFace Inc. team. All rights reserved.
5 +#
6 +# Licensed under the Apache License, Version 2.0 (the "License");
7 +# you may not use this file except in compliance with the License.
8 +# You may obtain a copy of the License at
9 +#
10 +# http://www.apache.org/licenses/LICENSE-2.0
11 +#
12 +# Unless required by applicable law or agreed to in writing, software
13 +# distributed under the License is distributed on an "AS IS" BASIS,
14 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 +# See the License for the specific language governing permissions and
16 +# limitations under the License.
17 +""" PyTorch Fuyu model."""
18 +import math
19 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
20 +
21 +import torch
22 +import torch.nn as nn
23 +import torch.utils.checkpoint
24 +from PIL import Image
25 +from transformers import FuyuConfig, FuyuImageProcessor
26 +
27 +from vllm.attention import AttentionMetadata
28 +from vllm.config import CacheConfig, MultiModalConfig
29 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30 +from vllm.logger import init_logger
31 +from vllm.model_executor.layers.linear import ColumnParallelLinear
32 +from vllm.model_executor.layers.quantization.base_config import (
33 + QuantizationConfig)
34 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35 +from vllm.model_executor.models.persimmon import PersimmonForCausalLM
36 +from vllm.model_executor.sampling_metadata import SamplingMetadata
37 +from vllm.multimodal import MULTIMODAL_REGISTRY
38 +from vllm.multimodal.base import MultiModalInputs
39 +from vllm.multimodal.image import (cached_get_image_processor,
40 + cached_get_tokenizer)
41 +from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
42 +
43 +from .interfaces import SupportsVision
44 +from .utils import merge_vision_embeddings
45 +
46 +logger = init_logger(__name__)
47 +
48 +# Cannot find the following 2 numbers from hf config.
49 +_IMAGE_TOKEN_ID = 71011
50 +_NEWLINE_TOKEN_ID = 71019
51 +
52 +MAX_IMAGE_FEATURE_SIZE_HEIGHT = 1080
53 +MAX_IMAGE_FEATURE_SIZE_WIDTH = 1920
54 +
55 +
56 +class FuyuImagePixelInputs(TypedDict):
57 + type: Literal["pixel_values"]
58 + data: torch.Tensor
59 + """
60 + Shape:
61 + (batch_size, num_patches, patch_size_x * patch_size_y * num_channels)
62 + """
63 +
64 +
65 +def _calculate_num_image_tokens(
66 + height: int,
67 + width: int,
68 +) -> Tuple[int, int]:
69 + """
70 + calculate number of image tokens needed for a given image size
71 + The expected Fuyu image prompts is in format:
72 + (image_token * ncols + newline_token) * nrows
73 + args:
74 + image_size: Tuple[int, int] - (width, height) of the image
75 + returns:
76 + ncols: int - number of image tokens in x direction
77 + nrows: int - number of image tokens in y direction
78 + """
79 + ncol = math.ceil(width / 30)
80 + nrow = math.ceil(height / 30)
81 + return ncol, nrow
82 +
83 +
84 +def get_max_fuyu_image_feature_size():
85 +
86 + return _calculate_num_image_tokens(
87 + height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
88 + width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
89 + )
90 +
91 +
92 +def get_max_fuyu_image_tokens(ctx: InputContext):
93 + ncol, nrow = get_max_fuyu_image_feature_size()
94 + return (ncol + 1) * nrow
95 +
96 +
97 +def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int):
98 + ncol, nrow = get_max_fuyu_image_feature_size()
99 + image_feature_size = get_max_fuyu_image_tokens(ctx)
100 +
101 + token_ids = ([_IMAGE_TOKEN_ID] * ncol + [_NEWLINE_TOKEN_ID]) * nrow
102 + token_ids += [0] * (seq_len - image_feature_size)
103 + return SequenceData(token_ids)
104 +
105 +
106 +def dummy_image_for_fuyu(
107 + image_width: int,
108 + image_height: int,
109 +):
110 + image = Image.new("RGB", (image_width, image_height), color=0)
111 + return {"image": image}
112 +
113 +
114 +def dummy_data_for_fuyu(ctx: InputContext, seq_len: int):
115 + seq_data = dummy_seq_data_for_fuyu(ctx, seq_len)
116 + mm_data = dummy_image_for_fuyu(MAX_IMAGE_FEATURE_SIZE_WIDTH,
117 + MAX_IMAGE_FEATURE_SIZE_HEIGHT)
118 + return seq_data, mm_data
119 +
120 +
121 +def _fuyu_image_preprocess(image_processor: FuyuImageProcessor,
122 + data: Image.Image):
123 + image_encoding = image_processor.preprocess(data, return_tensors="pt")
124 + batch_images = torch.stack([img[0] for img in image_encoding["images"]
125 + ]).unsqueeze(1)
126 + image_unpadded_heights = torch.tensor(
127 + image_encoding["image_unpadded_heights"])
128 + image_unpadded_widths = torch.tensor(
129 + image_encoding["image_unpadded_widths"])
130 +
131 + batch_size = len(image_encoding["images"])
132 + image_present = torch.ones(batch_size, 1, 1)
133 + model_image_input = image_processor.preprocess_with_tokenizer_info(
134 + image_input=batch_images,
135 + image_present=image_present,
136 + image_unpadded_h=image_unpadded_heights,
137 + image_unpadded_w=image_unpadded_widths,
138 + image_placeholder_id=_IMAGE_TOKEN_ID,
139 + image_newline_id=_NEWLINE_TOKEN_ID,
140 + variable_sized=True,
141 + )
142 + return model_image_input
143 +
144 +
145 +def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
146 + multi_modal_data = llm_inputs.get("multi_modal_data")
147 + if multi_modal_data is None or "image" not in multi_modal_data:
148 + return llm_inputs
149 +
150 + model_config = ctx.model_config
151 + image_data = multi_modal_data["image"]
152 + new_multi_modal_data = {}
153 + # process image data
154 + if isinstance(image_data, Image.Image):
155 + # Fuyu's image_processor can also finish token padding
156 + image_processor: FuyuImageProcessor = cached_get_image_processor(
157 + model_config.model)
158 +
159 + model_image_input = _fuyu_image_preprocess(image_processor, image_data)
160 + image_patches = torch.stack([
161 + image_patch[0]
162 + for image_patch in model_image_input["image_patches"]
163 + ])
164 + new_multi_modal_data["image"] = image_patches
165 +
166 + elif isinstance(image_data, torch.Tensor):
167 + raise NotImplementedError("Embeddings input is not supported yet")
168 + else:
169 + raise TypeError(f"Invalid image type: {type(image_data)}")
170 +
171 + # process prompts
172 + prompt = llm_inputs["prompt"]
173 + prompt_token_ids = llm_inputs["prompt_token_ids"]
174 + tokenizer = cached_get_tokenizer(model_config.model)
175 + # dim0 is batch_size, dim1 is subseq_size which will always be 1
176 + image_input_ids: List[List[
177 + torch.Tensor]] = model_image_input["image_input_ids"]
178 + image_input_ids = image_input_ids[0][0].tolist()
179 + bos_token = tokenizer.encode("<s>", add_special_tokens=False)[1:]
180 + boa_token = tokenizer.encode("\x04", add_special_tokens=False)[1:]
181 +
182 + new_prompt = prompt + "\x04"
183 + new_prompt_token_ids = image_input_ids + bos_token + prompt_token_ids[
184 + 1:] + boa_token
185 +
186 + return LLMInputs(prompt=new_prompt,
187 + prompt_token_ids=new_prompt_token_ids,
188 + multi_modal_data=new_multi_modal_data)
189 +
190 +
191 +def input_mapper_for_fuyu(ctx: InputContext, data: object):
192 + model_config = ctx.model_config
193 + if isinstance(data, Image.Image):
194 + # Fuyu's image_processor can also finish token padding
195 + image_processor: FuyuImageProcessor = cached_get_image_processor(
196 + model_config.model)
197 +
198 + model_image_input = _fuyu_image_preprocess(image_processor, data)
199 + data = torch.stack([
200 + image_patch[0]
201 + for image_patch in model_image_input["image_patches"]
202 + ])
203 +
204 + # image has been processed with prompt in input processor
205 + return MultiModalInputs({"image_patches": data})
206 +
207 +
208 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_fuyu)
209 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_fuyu_image_tokens)
210 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_fuyu)
211 +@INPUT_REGISTRY.register_input_processor(input_processor_for_fuyu)
212 +class FuyuForCausalLM(nn.Module, SupportsVision):
213 +
214 + def __init__(self,
215 + config: FuyuConfig,
216 + multimodal_config: MultiModalConfig,
217 + cache_config: Optional[CacheConfig] = None,
218 + quant_config: Optional[QuantizationConfig] = None) -> None:
219 + super().__init__()
220 + self.config = config
221 + self.multimodal_config = multimodal_config
222 +
223 + self.padding_idx = config.pad_token_id
224 + self.vocab_size = config.vocab_size
225 + self.image_token_id = _IMAGE_TOKEN_ID
226 + self.image_feature_size = config.patch_size**2 * config.num_channels
227 +
228 + self.vision_embed_tokens = ColumnParallelLinear(
229 + self.image_feature_size,
230 + config.hidden_size,
231 + quant_config=quant_config,
232 + )
233 + self.language_model = PersimmonForCausalLM(config,
234 + cache_config=cache_config,
235 + quant_config=quant_config)
236 +
237 + def _parse_and_validate_image_input(self, **kwargs: object):
238 + image_patches = kwargs.pop("image_patches", None)
239 +
240 + if isinstance(image_patches, torch.Tensor):
241 + expected_feature_size = self.image_feature_size
242 + if image_patches.size(-1) != expected_feature_size:
243 + raise ValueError(
244 + f"Expected image patches to have the last dimension of "
245 + f"{expected_feature_size}, got {image_patches.size(-1)}")
246 + image_patches = image_patches.to(
247 + self.vision_embed_tokens.weight.dtype)
248 + return FuyuImagePixelInputs(type="pixel_values",
249 + data=image_patches)
250 + return None
251 +
252 + def forward(
253 + self,
254 + input_ids: torch.Tensor,
255 + positions: torch.Tensor,
256 + kv_caches: List[torch.Tensor],
257 + attn_metadata: AttentionMetadata,
258 + intermediate_tensors: Optional[IntermediateTensors] = None,
259 + **kwargs: object,
260 + ):
261 + image_input = self._parse_and_validate_image_input(**kwargs)
262 +
263 + if image_input is not None:
264 + vision_embeddings, _ = self.vision_embed_tokens(
265 + image_input["data"])
266 + inputs_embeds = self.language_model.model.embed_tokens(input_ids)
267 + inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
268 + vision_embeddings,
269 + self.image_token_id)
270 +
271 + else:
272 + inputs_embeds = None
273 +
274 + hidden_states = self.language_model(
275 + input_ids=input_ids,
276 + positions=positions,
277 + kv_caches=kv_caches,
278 + attn_metadata=attn_metadata,
279 + inputs_embeds=inputs_embeds,
280 + )
281 + return hidden_states
282 +
283 + def compute_logits(self, hidden_states: torch.Tensor,
284 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
285 + logits = self.language_model.logits_processor(
286 + self.language_model.lm_head, hidden_states, sampling_metadata)
287 + return logits
288 +
289 + def sample(
290 + self,
291 + logits: torch.Tensor,
292 + sampling_metadata: SamplingMetadata,
293 + ) -> Optional[SamplerOutput]:
294 + next_tokens = self.language_model.sampler(logits, sampling_metadata)
295 + return next_tokens
296 +
297 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
298 + params_dict = dict(self.named_parameters(remove_duplicate=False))
299 + for name, loaded_weight in weights:
300 + if "rotary_emb.inv_freq" in name:
301 + continue
302 + if ("rotary_emb.cos_cached" in name
303 + or "rotary_emb.sin_cached" in name):
304 + # Models trained using ColossalAI may include these tensors in
305 + # the checkpoint. Skip them.
306 + continue
307 + param = params_dict[name]
308 +
309 + if "query_key_value" in name:
310 + # copy from vllm/model_executor/models/bloom.py
311 + # NOTE: Fuyu's fused QKV's output_dim has the shape of
312 + # (num_heads * 3 * head_size), while the
313 + # required shape is (3 * num_heads * head_size).
314 + # Thus, we need weight conversion.
315 + output_dim = getattr(param, "output_dim", None)
316 + num_heads = self.config.num_attention_heads
317 + if output_dim is not None:
318 + loaded_weight_shape = loaded_weight.shape
319 + loaded_weight = loaded_weight.view(
320 + loaded_weight_shape[:output_dim] + (num_heads, 3, -1) +
321 + loaded_weight_shape[output_dim + 1:])
322 + loaded_weight = loaded_weight.transpose(
323 + output_dim, output_dim + 1)
324 + loaded_weight = loaded_weight.reshape(loaded_weight_shape)
325 +
326 + weight_loader = getattr(param, "weight_loader",
327 + default_weight_loader)
328 + weight_loader(param, loaded_weight)
vllm/model_executor/models/persimmon.py new
+333
@@ -0,0 +1,333 @@
1 +# coding=utf-8
2 +# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/persimmon/modeling_persimmon.py
3 +# Copyright 2023 The vLLM team.
4 +# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5 +#
6 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7 +# and OPT implementations in this library. It has been modified from its
8 +# original forms to accommodate minor architectural differences compared
9 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10 +#
11 +# Licensed under the Apache License, Version 2.0 (the "License");
12 +# you may not use this file except in compliance with the License.
13 +# You may obtain a copy of the License at
14 +#
15 +# http://www.apache.org/licenses/LICENSE-2.0
16 +#
17 +# Unless required by applicable law or agreed to in writing, software
18 +# distributed under the License is distributed on an "AS IS" BASIS,
19 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 +# See the License for the specific language governing permissions and
21 +# limitations under the License.
22 +"""Inference-only persimmon model compatible with HuggingFace weights."""
23 +from typing import Iterable, List, Optional, Tuple
24 +
25 +import torch
26 +from torch import nn
27 +from transformers import PersimmonConfig
28 +from transformers.activations import ReLUSquaredActivation
29 +
30 +from vllm.attention import Attention, AttentionMetadata
31 +from vllm.config import CacheConfig
32 +from vllm.distributed import get_tensor_model_parallel_world_size
33 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 + QKVParallelLinear,
35 + RowParallelLinear)
36 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 +from vllm.model_executor.layers.quantization.base_config import (
38 + QuantizationConfig)
39 +from vllm.model_executor.layers.rotary_embedding import get_rope
40 +from vllm.model_executor.layers.sampler import Sampler
41 +from vllm.model_executor.layers.vocab_parallel_embedding import (
42 + ParallelLMHead, VocabParallelEmbedding)
43 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 +from vllm.sequence import IntermediateTensors, SamplerOutput
46 +
47 +
48 +class PersimmonMLP(nn.Module):
49 +
50 + def __init__(self,
51 + config: PersimmonConfig,
52 + quant_config: Optional[QuantizationConfig] = None):
53 + super().__init__()
54 + self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
55 + config.intermediate_size,
56 + quant_config=quant_config)
57 + self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
58 + config.hidden_size,
59 + quant_config=quant_config)
60 + self.act = ReLUSquaredActivation()
61 +
62 + def forward(self, hidden_states) -> torch.Tensor:
63 + hidden_states, _ = self.dense_h_to_4h(hidden_states)
64 + hidden_states = self.act(hidden_states)
65 + hidden_states, _ = self.dense_4h_to_h(hidden_states)
66 + return hidden_states
67 +
68 +
69 +class PersimmonAttention(nn.Module):
70 +
71 + def __init__(self,
72 + config: PersimmonConfig,
73 + cache_config: Optional[CacheConfig] = None,
74 + quant_config: Optional[QuantizationConfig] = None):
75 + super().__init__()
76 + self.config = config
77 + tensor_parallel_world_size = get_tensor_model_parallel_world_size()
78 +
79 + self.hidden_size = config.hidden_size
80 + self.total_num_heads = config.num_attention_heads
81 + self.num_heads = self.total_num_heads // tensor_parallel_world_size
82 + self.head_dim = self.hidden_size // self.total_num_heads
83 + self.max_position_embeddings = config.max_position_embeddings
84 + self.rope_theta = config.rope_theta
85 + self.partial_rotary_factor = config.partial_rotary_factor
86 + self.is_causal = True
87 +
88 + assert (self.head_dim * self.total_num_heads) == self.hidden_size
89 + assert self.total_num_heads % tensor_parallel_world_size == 0
90 +
91 + self.query_key_value = QKVParallelLinear(
92 + self.hidden_size,
93 + self.head_dim,
94 + self.total_num_heads,
95 + bias=True,
96 + quant_config=quant_config,
97 + )
98 + self.dense = RowParallelLinear(
99 + self.num_heads * self.head_dim,
100 + self.hidden_size,
101 + bias=True,
102 + quant_config=quant_config,
103 + )
104 + self.is_qk_layernorm = config.qk_layernorm
105 +
106 + if self.is_qk_layernorm:
107 + self.q_layernorm = nn.LayerNorm(self.head_dim)
108 + self.k_layernorm = nn.LayerNorm(self.head_dim)
109 +
110 + self.rotary_emb = get_rope(
111 + self.head_dim,
112 + rotary_dim=int(self.partial_rotary_factor * self.head_dim),
113 + max_position=self.max_position_embeddings,
114 + base=self.rope_theta,
115 + )
116 + self.scaling = self.head_dim**-0.5
117 + self.attn = Attention(self.num_heads,
118 + self.head_dim,
119 + scale=self.scaling,
120 + cache_config=cache_config,
121 + quant_config=quant_config)
122 +
123 + def _split_heads(self, x: torch.Tensor) -> torch.Tensor:
124 + # [seq_length, hidden_size] -> [seq_length, num_heads, head_dim]
125 + seq_length = x.shape[0]
126 + return x.view(seq_length, self.num_heads, self.head_dim)
127 +
128 + def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
129 + # [seq_length, num_heads, head_dim] -> [seq_length, hidden_size]
130 + seq_length = x.shape[0]
131 + return x.view(seq_length, self.num_heads * self.head_dim)
132 +
133 + def forward(
134 + self,
135 + position_ids: torch.Tensor,
136 + hidden_states: torch.Tensor,
137 + kv_cache: torch.Tensor,
138 + attn_metadata: AttentionMetadata,
139 + ) -> torch.Tensor:
140 + # [seq_length, 3 x hidden_size]
141 + qkv, _ = self.query_key_value(hidden_states)
142 + q, k, v = qkv.chunk(chunks=3, dim=-1)
143 +
144 + if self.is_qk_layernorm:
145 + # [seq_length, num_heads, head_dim]
146 + q = self._split_heads(q)
147 + k = self._split_heads(k)
148 +
149 + q = self.q_layernorm(q)
150 + k = self.k_layernorm(k)
151 +
152 + q = self._merge_heads(q)
153 + k = self._merge_heads(k)
154 +
155 + q, k = self.rotary_emb(position_ids, q, k)
156 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
157 + output, _ = self.dense(attn_output)
158 + return output
159 +
160 +
161 +class PersimmonDecoderLayer(nn.Module):
162 +
163 + def __init__(self,
164 + config: PersimmonConfig,
165 + cache_config: Optional[CacheConfig] = None,
166 + quant_config: Optional[QuantizationConfig] = None):
167 + super().__init__()
168 + self.hidden_size = config.hidden_size
169 + self.self_attn = PersimmonAttention(config=config,
170 + cache_config=cache_config,
171 + quant_config=quant_config)
172 + self.mlp = PersimmonMLP(config, quant_config=quant_config)
173 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
174 + eps=config.layer_norm_eps)
175 + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
176 + eps=config.layer_norm_eps)
177 +
178 + def forward(
179 + self,
180 + position_ids: torch.Tensor,
181 + hidden_states: torch.Tensor,
182 + kv_cache: torch.Tensor,
183 + attn_metadata: AttentionMetadata,
184 + ) -> torch.Tensor:
185 + residual = hidden_states
186 +
187 + hidden_states = self.input_layernorm(hidden_states)
188 +
189 + # Self Attention
190 + hidden_states = self.self_attn(
191 + position_ids=position_ids,
192 + hidden_states=hidden_states,
193 + kv_cache=kv_cache,
194 + attn_metadata=attn_metadata,
195 + )
196 + hidden_states = residual + hidden_states
197 +
198 + # Fully Connected
199 + residual = hidden_states
200 + hidden_states = self.post_attention_layernorm(hidden_states)
201 + hidden_states = self.mlp(hidden_states)
202 +
203 + hidden_states = hidden_states + residual
204 +
205 + outputs = hidden_states
206 + return outputs
207 +
208 +
209 +class PersimmonModel(nn.Module):
210 +
211 + def __init__(self,
212 + config: PersimmonConfig,
213 + cache_config: Optional[CacheConfig] = None,
214 + quant_config: Optional[QuantizationConfig] = None):
215 + super().__init__()
216 + self.vocab_size = config.vocab_size
217 +
218 + self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
219 + config.hidden_size)
220 + self.layers = nn.ModuleList([
221 + PersimmonDecoderLayer(config,
222 + cache_config=cache_config,
223 + quant_config=quant_config)
224 + for _ in range(config.num_hidden_layers)
225 + ])
226 + self.final_layernorm = nn.LayerNorm(config.hidden_size,
227 + eps=config.layer_norm_eps)
228 +
229 + def forward(
230 + self,
231 + input_ids: torch.Tensor,
232 + positions: torch.Tensor,
233 + kv_caches: List[torch.Tensor],
234 + attn_metadata: AttentionMetadata,
235 + inputs_embeds: Optional[torch.Tensor] = None,
236 + ) -> torch.Tensor:
237 + if inputs_embeds is not None:
238 + hidden_states = inputs_embeds
239 + else:
240 + hidden_states = self.embed_tokens(input_ids)
241 + for i in range(len(self.layers)):
242 + hidden_states = self.layers[i](
243 + positions,
244 + hidden_states,
245 + kv_caches[i],
246 + attn_metadata,
247 + )
248 + hidden_states = self.final_layernorm(hidden_states)
249 + return hidden_states
250 +
251 +
252 +class PersimmonForCausalLM(nn.Module):
253 +
254 + def __init__(self,
255 + config,
256 + cache_config: Optional[CacheConfig] = None,
257 + quant_config: Optional[QuantizationConfig] = None):
258 + super().__init__()
259 + self.config = config
260 + self.vocab_size = config.vocab_size
261 + self.model = PersimmonModel(config,
262 + cache_config=cache_config,
263 + quant_config=quant_config)
264 + self.lm_head = ParallelLMHead(config.vocab_size,
265 + config.hidden_size,
266 + bias=False)
267 + self.logits_processor = LogitsProcessor(config.vocab_size)
268 + self.sampler = Sampler()
269 +
270 + def forward(
271 + self,
272 + input_ids: torch.Tensor,
273 + positions: torch.Tensor,
274 + kv_caches: List[torch.Tensor],
275 + attn_metadata: AttentionMetadata,
276 + intermediate_tensors: Optional[IntermediateTensors] = None,
277 + inputs_embeds: Optional[torch.Tensor] = None,
278 + ):
279 + hidden_states = self.model(
280 + input_ids=input_ids,
281 + positions=positions,
282 + kv_caches=kv_caches,
283 + attn_metadata=attn_metadata,
284 + inputs_embeds=inputs_embeds,
285 + )
286 + return hidden_states
287 +
288 + def compute_logits(self, hidden_states: torch.Tensor,
289 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
290 + logits = self.logits_processor(self.lm_head, hidden_states,
291 + sampling_metadata)
292 + return logits
293 +
294 + def sample(
295 + self,
296 + logits: torch.Tensor,
297 + sampling_metadata: SamplingMetadata,
298 + ) -> Optional[SamplerOutput]:
299 + next_tokens = self.sampler(logits, sampling_metadata)
300 + return next_tokens
301 +
302 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
303 + params_dict = dict(self.named_parameters(remove_duplicate=False))
304 + for name, loaded_weight in weights:
305 + if "rotary_emb.inv_freq" in name:
306 + continue
307 + if ("rotary_emb.cos_cached" in name
308 + or "rotary_emb.sin_cached" in name):
309 + # Models trained using ColossalAI may include these tensors in
310 + # the checkpoint. Skip them.
311 + continue
312 + param = params_dict[name]
313 +
314 + if "query_key_value" in name:
315 + # copy from vllm/model_executor/models/bloom.py
316 + # NOTE: Persimmon's fused QKV's output_dim has the shape of
317 + # (num_heads * 3 * head_size), while the
318 + # required shape is (3 * num_heads * head_size).
319 + # Thus, we need weight conversion.
320 + output_dim = getattr(param, "output_dim", None)
321 + num_heads = self.config.num_attention_heads
322 + if output_dim is not None:
323 + loaded_weight_shape = loaded_weight.shape
324 + loaded_weight = loaded_weight.view(
325 + loaded_weight_shape[:output_dim] + (num_heads, 3, -1) +
326 + loaded_weight_shape[output_dim + 1:])
327 + loaded_weight = loaded_weight.transpose(
328 + output_dim, output_dim + 1)
329 + loaded_weight = loaded_weight.reshape(loaded_weight_shape)
330 +
331 + weight_loader = getattr(param, "weight_loader",
332 + default_weight_loader)
333 + weight_loader(param, loaded_weight)