@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / de60a3fb93
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Added DeciLM-7b and DeciLM-7b-instruct (#2062)

avideci committed Dec 19, 2023 at 12:29 UTC de60a3fb93957dce6b242299b5d163f02ef7f383
5 files changed +129
README.md
+1
@@ -54,6 +54,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
54 - Baichuan & Baichuan2 (`baichuan-inc/Baichuan2-13B-Chat`, `baichuan-inc/Baichuan-7B`, etc.)
55 - BLOOM (`bigscience/bloom`, `bigscience/bloomz`, etc.)
56 - ChatGLM (`THUDM/chatglm2-6b`, `THUDM/chatglm3-6b`, etc.)
57 +- DeciLM (`Deci/DeciLM-7B`, `Deci/DeciLM-7B-instruct`, etc.)
58 - Falcon (`tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc.)
59 - GPT-2 (`gpt2`, `gpt2-xl`, etc.)
60 - GPT BigCode (`bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, etc.)
docs/source/models/supported_models.rst
+3
@@ -23,6 +23,9 @@ Alongside each architecture, we include some popular models that use it.
23 * - :code:`ChatGLMModel`
24 - ChatGLM
25 - :code:`THUDM/chatglm2-6b`, :code:`THUDM/chatglm3-6b`, etc.
26 + * - :code:`DeciLMForCausalLM`
27 + - DeciLM
28 + - :code:`Deci/DeciLM-7B`, :code:`Deci/DeciLM-7B-instruct`, etc.
29 * - :code:`BloomForCausalLM`
30 - BLOOM, BLOOMZ, BLOOMChat
31 - :code:`bigscience/bloom`, :code:`bigscience/bloomz`, etc.
tests/models/test_models.py
+1
@@ -8,6 +8,7 @@ MODELS = [
8 "facebook/opt-125m",
9 "meta-llama/Llama-2-7b-hf",
10 "mistralai/Mistral-7B-v0.1",
11 + "Deci/DeciLM-7b",
12 "tiiuae/falcon-7b",
13 "gpt2",
14 "bigcode/tiny_starcoder_py",
vllm/model_executor/models/__init__.py
+1
@@ -17,6 +17,7 @@ _MODELS = {
17 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
18 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19 "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20 + "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
21 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
22 "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
23 "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
vllm/model_executor/models/decilm.py new
+123
@@ -0,0 +1,123 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 DeciAI Research Team. All rights reserved.
5 +# Copyright 2023 The vLLM team.
6 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7 +#
8 +# This code is based on MistralAI GPT-NeoX library and the GPT-NeoX
9 +# and OPT implementations in this library. It has been modified from its
10 +# original forms to accommodate minor architectural differences compared
11 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12 +#
13 +# Licensed under the Apache License, Version 2.0 (the "License");
14 +# you may not use this file except in compliance with the License.
15 +# You may obtain a copy of the License at
16 +#
17 +# http://www.apache.org/licenses/LICENSE-2.0
18 +#
19 +# Unless required by applicable law or agreed to in writing, software
20 +# distributed under the License is distributed on an "AS IS" BASIS,
21 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 +# See the License for the specific language governing permissions and
23 +# limitations under the License.
24 +"""Inference-only DeciLM model compatible with HuggingFace weights."""
25 +
26 +from typing import Optional
27 +
28 +import torch
29 +from transformers import PretrainedConfig
30 +
31 +from vllm.model_executor.layers.linear import LinearMethodBase
32 +from vllm.model_executor.models.llama import LlamaForCausalLM
33 +from vllm.model_executor.weight_utils import (default_weight_loader,
34 + hf_model_weights_iterator)
35 +
36 +
37 +class DeciLMForCausalLM(LlamaForCausalLM):
38 + """
39 + Implementation for https://huggingface.co/Deci/DeciLM-7b-instruct.
40 + Based on the llama executor.
41 +
42 + The main difference is that DeciLM uses Variable Grouped Query Attention.
43 + The constant number of GQA heads in the decoder is overriden with a value
44 + per layer.
45 +
46 + Usually, in the HuggingFace implementation, instead of
47 + "config.num_key_value_heads", we use
48 + "config.num_key_value_heads_per_layer[i]" which varies.
49 +
50 + Currently, PagedAttention does not work well with variable GQA, so we
51 + normalize the weights upon loading, and use uniform GQA with the max value
52 + instead.
53 + """
54 +
55 + def __init__(
56 + self,
57 + config: Optional[PretrainedConfig] = None,
58 + linear_method: Optional[LinearMethodBase] = None,
59 + ) -> None:
60 + config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
61 + delattr(config, "num_key_value_heads_per_layer")
62 + super().__init__(config=config, linear_method=linear_method)
63 +
64 + def load_weights(self,
65 + model_name_or_path: str,
66 + cache_dir: Optional[str] = None,
67 + load_format: str = "auto",
68 + revision: Optional[str] = None):
69 + stacked_params_mapping = [
70 + # (param_name, shard_name, shard_id)
71 + ("qkv_proj", "q_proj", "q"),
72 + ("qkv_proj", "k_proj", "k"),
73 + ("qkv_proj", "v_proj", "v"),
74 + ("gate_up_proj", "gate_proj", 0),
75 + ("gate_up_proj", "up_proj", 1),
76 + ]
77 + params_dict = dict(self.named_parameters())
78 + for name, loaded_weight in hf_model_weights_iterator(
79 + model_name_or_path, cache_dir, load_format, revision):
80 + if "rotary_emb.inv_freq" in name:
81 + continue
82 +
83 + if "k_proj" in name or "v_proj" in name:
84 + loaded_weight = self._degroup_weight(loaded_weight)
85 +
86 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
87 + if weight_name not in name:
88 + continue
89 + name = name.replace(weight_name, param_name)
90 + # Skip loading extra bias for GPTQ models.
91 + if name.endswith(".bias") and name not in params_dict:
92 + continue
93 + param = params_dict[name]
94 + weight_loader = param.weight_loader
95 + weight_loader(param, loaded_weight, shard_id)
96 + break
97 + else:
98 + # Skip loading extra bias for GPTQ models.
99 + if name.endswith(".bias") and name not in params_dict:
100 + continue
101 + param = params_dict[name]
102 + weight_loader = getattr(param, "weight_loader",
103 + default_weight_loader)
104 + weight_loader(param, loaded_weight)
105 +
106 + def _degroup_weight(self, loaded_weight: torch.Tensor) -> torch.Tensor:
107 + hidden_size = self.config.hidden_size
108 + head_size = self.config.hidden_size // self.config.num_attention_heads
109 + target_num_kv_heads = self.config.num_key_value_heads
110 + num_kv_heads = loaded_weight.shape[0] // head_size
111 + n_repeats = target_num_kv_heads / num_kv_heads
112 + assert n_repeats == int(n_repeats)
113 +
114 + n_repeats = int(n_repeats)
115 + loaded_weight = loaded_weight.view(num_kv_heads, head_size,
116 + hidden_size)
117 + loaded_weight = torch.repeat_interleave(loaded_weight,
118 + repeats=n_repeats,
119 + dim=0)
120 + loaded_weight = loaded_weight.reshape(target_num_kv_heads * head_size,
121 + hidden_size)
122 +
123 + return loaded_weight