A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Model] Add OLMoE (#7922)
Niklas Muennighoff committed
Sep 20, 2024 at 09:31 UTC
3b63de9353ce51ba6c1c167ae8d4b87b8bcf9c9e
3 files changed
+414
docs/source/models/supported_models.rst
+4
@@ -127,6 +127,10 @@ Decoder-only Language Models
127
- Nemotron-3, Nemotron-4, Minitron
128
- :code:`nvidia/Minitron-8B-Base`, :code:`mgoin/Nemotron-4-340B-Base-hf-FP8`, etc.
129
- ✅︎
130
+ * - :code:`OLMoEForCausalLM`
131
+ - OLMoE
132
+ - :code:`allenai/OLMoE-1B-7B-0924`, :code:`allenai/OLMoE-1B-7B-0924-Instruct`, etc.
133
+ -
134
* - :code:`OLMoForCausalLM`
135
- OLMo
136
- :code:`allenai/OLMo-1B-hf`, :code:`allenai/OLMo-7B-hf`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -46,6 +46,7 @@ _GENERATION_MODELS = {
46
"MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
47
"NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
48
"OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
49
+ "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
50
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
51
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
52
"PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
vllm/model_executor/models/olmoe.py
new
+409
@@ -0,0 +1,409 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License");
2
+# you may not use this file except in compliance with the License.
3
+# You may obtain a copy of the License at
4
+#
5
+# http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS,
9
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10
+# See the License for the specific language governing permissions and
11
+# limitations under the License.
12
+"""Inference-only OLMoE model compatible with HuggingFace weights."""
13
+from typing import Any, Dict, Iterable, List, Optional, Tuple
14
+
15
+import torch
16
+from torch import nn
17
+from transformers import PretrainedConfig
18
+
19
+from vllm.attention import Attention, AttentionMetadata
20
+from vllm.config import CacheConfig
21
+from vllm.distributed import get_tensor_model_parallel_world_size
22
+from vllm.model_executor.layers.fused_moe import FusedMoE
23
+from vllm.model_executor.layers.layernorm import RMSNorm
24
+from vllm.model_executor.layers.linear import (QKVParallelLinear,
25
+ ReplicatedLinear,
26
+ RowParallelLinear)
27
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
28
+from vllm.model_executor.layers.quantization.base_config import (
29
+ QuantizationConfig)
30
+from vllm.model_executor.layers.rotary_embedding import get_rope
31
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
32
+from vllm.model_executor.layers.vocab_parallel_embedding import (
33
+ ParallelLMHead, VocabParallelEmbedding)
34
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35
+from vllm.model_executor.sampling_metadata import SamplingMetadata
36
+from vllm.sequence import IntermediateTensors
37
+from vllm.utils import print_warning_once
38
+
39
+
40
+class OlmoeMoE(nn.Module):
41
+ """A tensor-parallel MoE implementation for Olmoe that shards each expert
42
+ across all ranks.
43
+
44
+ Each expert's weights are sharded across all ranks and a fused MoE
45
+ kernel is used for the forward pass, and finally we reduce the outputs
46
+ across ranks.
47
+ """
48
+
49
+ def __init__(self,
50
+ num_experts: int,
51
+ top_k: int,
52
+ hidden_size: int,
53
+ intermediate_size: int,
54
+ params_dtype: Optional[torch.dtype] = None,
55
+ quant_config: Optional[QuantizationConfig] = None,
56
+ tp_size: Optional[int] = None,
57
+ prefix: str = ""):
58
+ super().__init__()
59
+ self.hidden_size = hidden_size
60
+
61
+ # Gate always runs at half / full precision for now.
62
+ self.gate = ReplicatedLinear(hidden_size,
63
+ num_experts,
64
+ bias=False,
65
+ quant_config=None)
66
+
67
+ self.experts = FusedMoE(num_experts=num_experts,
68
+ top_k=top_k,
69
+ hidden_size=hidden_size,
70
+ intermediate_size=intermediate_size,
71
+ reduce_results=True,
72
+ renormalize=False,
73
+ quant_config=quant_config,
74
+ tp_size=tp_size)
75
+
76
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
77
+ # NOTE: hidden_states can have either 1D or 2D shape.
78
+ orig_shape = hidden_states.shape
79
+ hidden_dim = hidden_states.shape[-1]
80
+ hidden_states = hidden_states.view(-1, hidden_dim)
81
+ # router_logits: (num_tokens, n_experts)
82
+ router_logits, _ = self.gate(hidden_states)
83
+ final_hidden_states = self.experts(hidden_states=hidden_states,
84
+ router_logits=router_logits)
85
+ return final_hidden_states.view(orig_shape)
86
+
87
+
88
+class OlmoeAttention(nn.Module):
89
+
90
+ def __init__(
91
+ self,
92
+ hidden_size: int,
93
+ num_heads: int,
94
+ num_kv_heads: int,
95
+ rope_theta: float = 10000,
96
+ rope_scaling: Optional[Dict[str, Any]] = None,
97
+ max_position_embeddings: int = 4096,
98
+ cache_config: Optional[CacheConfig] = None,
99
+ quant_config: Optional[QuantizationConfig] = None,
100
+ ) -> None:
101
+ super().__init__()
102
+ self.hidden_size = hidden_size
103
+ tp_size = get_tensor_model_parallel_world_size()
104
+ self.total_num_heads = num_heads
105
+ assert self.total_num_heads % tp_size == 0
106
+ self.num_heads = self.total_num_heads // tp_size
107
+ self.total_num_kv_heads = num_kv_heads
108
+ if self.total_num_kv_heads >= tp_size:
109
+ # Number of KV heads is greater than TP size, so we partition
110
+ # the KV heads across multiple tensor parallel GPUs.
111
+ assert self.total_num_kv_heads % tp_size == 0
112
+ else:
113
+ # Number of KV heads is less than TP size, so we replicate
114
+ # the KV heads across multiple tensor parallel GPUs.
115
+ assert tp_size % self.total_num_kv_heads == 0
116
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
117
+ self.head_dim = hidden_size // self.total_num_heads
118
+ self.q_size = self.num_heads * self.head_dim
119
+ self.kv_size = self.num_kv_heads * self.head_dim
120
+ self.scaling = self.head_dim**-0.5
121
+ self.rope_theta = rope_theta
122
+ self.max_position_embeddings = max_position_embeddings
123
+
124
+ self.qkv_proj = QKVParallelLinear(
125
+ hidden_size,
126
+ self.head_dim,
127
+ self.total_num_heads,
128
+ self.total_num_kv_heads,
129
+ bias=False,
130
+ quant_config=quant_config,
131
+ )
132
+ self.q_norm = RMSNorm(hidden_size, eps=1e-5)
133
+ self.k_norm = RMSNorm(hidden_size, eps=1e-5)
134
+ self.o_proj = RowParallelLinear(
135
+ self.total_num_heads * self.head_dim,
136
+ hidden_size,
137
+ bias=False,
138
+ quant_config=quant_config,
139
+ )
140
+
141
+ self.rotary_emb = get_rope(
142
+ self.head_dim,
143
+ rotary_dim=self.head_dim,
144
+ max_position=max_position_embeddings,
145
+ base=rope_theta,
146
+ rope_scaling=rope_scaling,
147
+ is_neox_style=True,
148
+ )
149
+ self.attn = Attention(self.num_heads,
150
+ self.head_dim,
151
+ self.scaling,
152
+ num_kv_heads=self.num_kv_heads,
153
+ cache_config=cache_config,
154
+ quant_config=quant_config)
155
+
156
+ def forward(
157
+ self,
158
+ positions: torch.Tensor,
159
+ hidden_states: torch.Tensor,
160
+ kv_cache: torch.Tensor,
161
+ attn_metadata: AttentionMetadata,
162
+ ) -> torch.Tensor:
163
+ qkv, _ = self.qkv_proj(hidden_states)
164
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
165
+ q, k = self.q_norm(q.contiguous()), self.k_norm(k.contiguous())
166
+ q, k = self.rotary_emb(positions, q, k)
167
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
168
+ output, _ = self.o_proj(attn_output)
169
+ return output
170
+
171
+
172
+class OlmoeDecoderLayer(nn.Module):
173
+
174
+ def __init__(
175
+ self,
176
+ config: PretrainedConfig,
177
+ layer_idx: int,
178
+ cache_config: Optional[CacheConfig] = None,
179
+ quant_config: Optional[QuantizationConfig] = None,
180
+ ) -> None:
181
+ super().__init__()
182
+ self.hidden_size = config.hidden_size
183
+ rope_theta = getattr(config, "rope_theta", 10000)
184
+ rope_scaling = getattr(config, "rope_scaling", None)
185
+ max_position_embeddings = getattr(config, "max_position_embeddings",
186
+ 4096)
187
+
188
+ self.self_attn = OlmoeAttention(
189
+ hidden_size=self.hidden_size,
190
+ num_heads=config.num_attention_heads,
191
+ num_kv_heads=config.num_key_value_heads,
192
+ rope_theta=rope_theta,
193
+ rope_scaling=rope_scaling,
194
+ max_position_embeddings=max_position_embeddings,
195
+ cache_config=cache_config,
196
+ quant_config=quant_config,
197
+ )
198
+
199
+ self.mlp = OlmoeMoE(
200
+ num_experts=config.num_experts,
201
+ top_k=config.num_experts_per_tok,
202
+ hidden_size=config.hidden_size,
203
+ intermediate_size=config.intermediate_size,
204
+ quant_config=quant_config,
205
+ )
206
+ self.input_layernorm = RMSNorm(config.hidden_size, eps=1e-5)
207
+ self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=1e-5)
208
+
209
+ def forward(
210
+ self,
211
+ positions: torch.Tensor,
212
+ hidden_states: torch.Tensor,
213
+ kv_cache: torch.Tensor,
214
+ attn_metadata: AttentionMetadata,
215
+ residual: Optional[torch.Tensor],
216
+ ) -> torch.Tensor:
217
+ # Self Attention
218
+ if residual is None:
219
+ residual = hidden_states
220
+ hidden_states = self.input_layernorm(hidden_states)
221
+ else:
222
+ hidden_states, residual = self.input_layernorm(
223
+ hidden_states, residual)
224
+
225
+ hidden_states = self.self_attn(
226
+ positions=positions,
227
+ hidden_states=hidden_states,
228
+ kv_cache=kv_cache,
229
+ attn_metadata=attn_metadata,
230
+ )
231
+
232
+ # Fully Connected
233
+ hidden_states, residual = self.post_attention_layernorm(
234
+ hidden_states, residual)
235
+ hidden_states = self.mlp(hidden_states)
236
+ return hidden_states, residual
237
+
238
+
239
+class OlmoeModel(nn.Module):
240
+
241
+ def __init__(
242
+ self,
243
+ config: PretrainedConfig,
244
+ cache_config: Optional[CacheConfig] = None,
245
+ quant_config: Optional[QuantizationConfig] = None,
246
+ ) -> None:
247
+ super().__init__()
248
+ self.padding_idx = config.pad_token_id
249
+ self.vocab_size = config.vocab_size
250
+
251
+ self.embed_tokens = VocabParallelEmbedding(
252
+ config.vocab_size,
253
+ config.hidden_size,
254
+ )
255
+ self.layers = nn.ModuleList([
256
+ OlmoeDecoderLayer(config,
257
+ layer_idx,
258
+ cache_config,
259
+ quant_config=quant_config)
260
+ for layer_idx in range(config.num_hidden_layers)
261
+ ])
262
+ self.norm = RMSNorm(config.hidden_size, eps=1e-5)
263
+
264
+ def forward(
265
+ self,
266
+ input_ids: torch.Tensor,
267
+ positions: torch.Tensor,
268
+ kv_caches: List[torch.Tensor],
269
+ attn_metadata: AttentionMetadata,
270
+ ) -> torch.Tensor:
271
+ hidden_states = self.embed_tokens(input_ids)
272
+ residual = None
273
+ for i in range(len(self.layers)):
274
+ layer = self.layers[i]
275
+ hidden_states, residual = layer(positions, hidden_states,
276
+ kv_caches[i], attn_metadata,
277
+ residual)
278
+ hidden_states, _ = self.norm(hidden_states, residual)
279
+ return hidden_states
280
+
281
+
282
+class OlmoeForCausalLM(nn.Module):
283
+
284
+ fall_back_to_pt_during_load = False
285
+
286
+ def __init__(
287
+ self,
288
+ config: PretrainedConfig,
289
+ cache_config: Optional[CacheConfig] = None,
290
+ quant_config: Optional[QuantizationConfig] = None,
291
+ ) -> None:
292
+ super().__init__()
293
+ self.config = config
294
+ self.quant_config = quant_config
295
+ self.model = OlmoeModel(config, cache_config, quant_config)
296
+ self.lm_head = ParallelLMHead(config.vocab_size,
297
+ config.hidden_size,
298
+ quant_config=quant_config)
299
+ self.logits_processor = LogitsProcessor(config.vocab_size)
300
+ self.sampler = Sampler()
301
+
302
+ def forward(
303
+ self,
304
+ input_ids: torch.Tensor,
305
+ positions: torch.Tensor,
306
+ kv_caches: List[torch.Tensor],
307
+ attn_metadata: AttentionMetadata,
308
+ intermediate_tensors: Optional[IntermediateTensors] = None,
309
+ ) -> torch.Tensor:
310
+ hidden_states = self.model(input_ids, positions, kv_caches,
311
+ attn_metadata)
312
+ return hidden_states
313
+
314
+ def compute_logits(self, hidden_states: torch.Tensor,
315
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
316
+ logits = self.logits_processor(self.lm_head, hidden_states,
317
+ sampling_metadata)
318
+ return logits
319
+
320
+ def sample(
321
+ self,
322
+ logits: Optional[torch.Tensor],
323
+ sampling_metadata: SamplingMetadata,
324
+ ) -> Optional[SamplerOutput]:
325
+ next_tokens = self.sampler(logits, sampling_metadata)
326
+ return next_tokens
327
+
328
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
329
+ stacked_params_mapping = [
330
+ # (param_name, shard_name, shard_id)
331
+ ("qkv_proj", "q_proj", "q"),
332
+ ("qkv_proj", "k_proj", "k"),
333
+ ("qkv_proj", "v_proj", "v"),
334
+ ("gate_up_proj", "gate_proj", 0),
335
+ ("gate_up_proj", "up_proj", 1),
336
+ ]
337
+
338
+ # Params for weights, fp8 weight scales, fp8 activation scales
339
+ # (param_name, weight_name, expert_id, shard_id)
340
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
341
+ ckpt_gate_proj_name="gate_proj",
342
+ ckpt_down_proj_name="down_proj",
343
+ ckpt_up_proj_name="up_proj",
344
+ num_experts=self.config.num_experts)
345
+
346
+ params_dict = dict(self.named_parameters())
347
+ for name, loaded_weight in weights:
348
+ if "rotary_emb.inv_freq" in name:
349
+ continue
350
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
351
+ # Skip non-stacked layers and experts (experts handled below).
352
+ if weight_name not in name:
353
+ continue
354
+ # We have mlp.experts[0].gate_proj in the checkpoint.
355
+ # Since we handle the experts below in expert_params_mapping,
356
+ # we need to skip here BEFORE we update the name, otherwise
357
+ # name will be updated to mlp.experts[0].gate_up_proj, which
358
+ # will then be updated below in expert_params_mapping
359
+ # for mlp.experts[0].gate_gate_up_proj, which breaks load.
360
+ if "mlp.experts" in name:
361
+ continue
362
+ name = name.replace(weight_name, param_name)
363
+ # Skip loading extra bias for GPTQ models.
364
+ if name.endswith(".bias") and name not in params_dict:
365
+ continue
366
+ if name not in params_dict:
367
+ continue
368
+
369
+ param = params_dict[name]
370
+ weight_loader = param.weight_loader
371
+ weight_loader(param, loaded_weight, shard_id)
372
+ break
373
+ else:
374
+ for mapping in expert_params_mapping:
375
+ param_name, weight_name, expert_id, shard_id = mapping
376
+ if weight_name not in name:
377
+ continue
378
+ name = name.replace(weight_name, param_name)
379
+ param = params_dict[name]
380
+ weight_loader = param.weight_loader
381
+ weight_loader(param,
382
+ loaded_weight,
383
+ name,
384
+ shard_id=shard_id,
385
+ expert_id=expert_id)
386
+ break
387
+ else:
388
+ # Skip loading extra bias for GPTQ models.
389
+ if name.endswith(".bias") and name not in params_dict:
390
+ continue
391
+ # Remapping the name of FP8 kv-scale.
392
+ if name.endswith("kv_scale"):
393
+ remapped_kv_scale_name = name.replace(
394
+ ".kv_scale", ".attn.kv_scale")
395
+ if remapped_kv_scale_name not in params_dict:
396
+ print_warning_once(
397
+ "Found kv scale in the checkpoint "
398
+ f"(e.g. {name}), but not found the expected "
399
+ f"name in the model "
400
+ f"(e.g. {remapped_kv_scale_name}). "
401
+ "kv-scale is not loaded.")
402
+ continue
403
+ else:
404
+ name = remapped_kv_scale_name
405
+
406
+ param = params_dict[name]
407
+ weight_loader = getattr(param, "weight_loader",
408
+ default_weight_loader)
409
+ weight_loader(param, loaded_weight)