@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b63de9353
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Model] Add OLMoE (#7922)

Niklas Muennighoff committed Sep 20, 2024 at 09:31 UTC 3b63de9353ce51ba6c1c167ae8d4b87b8bcf9c9e
3 files changed +414
docs/source/models/supported_models.rst
+4
@@ -127,6 +127,10 @@ Decoder-only Language Models
127 - Nemotron-3, Nemotron-4, Minitron
128 - :code:`nvidia/Minitron-8B-Base`, :code:`mgoin/Nemotron-4-340B-Base-hf-FP8`, etc.
129 - ✅︎
130 + * - :code:`OLMoEForCausalLM`
131 + - OLMoE
132 + - :code:`allenai/OLMoE-1B-7B-0924`, :code:`allenai/OLMoE-1B-7B-0924-Instruct`, etc.
133 + -
134 * - :code:`OLMoForCausalLM`
135 - OLMo
136 - :code:`allenai/OLMo-1B-hf`, :code:`allenai/OLMo-7B-hf`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -46,6 +46,7 @@ _GENERATION_MODELS = {
46 "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
47 "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
48 "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
49 + "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
50 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
51 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
52 "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
vllm/model_executor/models/olmoe.py new
+409
@@ -0,0 +1,409 @@
1 +# Licensed under the Apache License, Version 2.0 (the "License");
2 +# you may not use this file except in compliance with the License.
3 +# You may obtain a copy of the License at
4 +#
5 +# http://www.apache.org/licenses/LICENSE-2.0
6 +#
7 +# Unless required by applicable law or agreed to in writing, software
8 +# distributed under the License is distributed on an "AS IS" BASIS,
9 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 +# See the License for the specific language governing permissions and
11 +# limitations under the License.
12 +"""Inference-only OLMoE model compatible with HuggingFace weights."""
13 +from typing import Any, Dict, Iterable, List, Optional, Tuple
14 +
15 +import torch
16 +from torch import nn
17 +from transformers import PretrainedConfig
18 +
19 +from vllm.attention import Attention, AttentionMetadata
20 +from vllm.config import CacheConfig
21 +from vllm.distributed import get_tensor_model_parallel_world_size
22 +from vllm.model_executor.layers.fused_moe import FusedMoE
23 +from vllm.model_executor.layers.layernorm import RMSNorm
24 +from vllm.model_executor.layers.linear import (QKVParallelLinear,
25 + ReplicatedLinear,
26 + RowParallelLinear)
27 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
28 +from vllm.model_executor.layers.quantization.base_config import (
29 + QuantizationConfig)
30 +from vllm.model_executor.layers.rotary_embedding import get_rope
31 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
32 +from vllm.model_executor.layers.vocab_parallel_embedding import (
33 + ParallelLMHead, VocabParallelEmbedding)
34 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35 +from vllm.model_executor.sampling_metadata import SamplingMetadata
36 +from vllm.sequence import IntermediateTensors
37 +from vllm.utils import print_warning_once
38 +
39 +
40 +class OlmoeMoE(nn.Module):
41 + """A tensor-parallel MoE implementation for Olmoe that shards each expert
42 + across all ranks.
43 +
44 + Each expert's weights are sharded across all ranks and a fused MoE
45 + kernel is used for the forward pass, and finally we reduce the outputs
46 + across ranks.
47 + """
48 +
49 + def __init__(self,
50 + num_experts: int,
51 + top_k: int,
52 + hidden_size: int,
53 + intermediate_size: int,
54 + params_dtype: Optional[torch.dtype] = None,
55 + quant_config: Optional[QuantizationConfig] = None,
56 + tp_size: Optional[int] = None,
57 + prefix: str = ""):
58 + super().__init__()
59 + self.hidden_size = hidden_size
60 +
61 + # Gate always runs at half / full precision for now.
62 + self.gate = ReplicatedLinear(hidden_size,
63 + num_experts,
64 + bias=False,
65 + quant_config=None)
66 +
67 + self.experts = FusedMoE(num_experts=num_experts,
68 + top_k=top_k,
69 + hidden_size=hidden_size,
70 + intermediate_size=intermediate_size,
71 + reduce_results=True,
72 + renormalize=False,
73 + quant_config=quant_config,
74 + tp_size=tp_size)
75 +
76 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
77 + # NOTE: hidden_states can have either 1D or 2D shape.
78 + orig_shape = hidden_states.shape
79 + hidden_dim = hidden_states.shape[-1]
80 + hidden_states = hidden_states.view(-1, hidden_dim)
81 + # router_logits: (num_tokens, n_experts)
82 + router_logits, _ = self.gate(hidden_states)
83 + final_hidden_states = self.experts(hidden_states=hidden_states,
84 + router_logits=router_logits)
85 + return final_hidden_states.view(orig_shape)
86 +
87 +
88 +class OlmoeAttention(nn.Module):
89 +
90 + def __init__(
91 + self,
92 + hidden_size: int,
93 + num_heads: int,
94 + num_kv_heads: int,
95 + rope_theta: float = 10000,
96 + rope_scaling: Optional[Dict[str, Any]] = None,
97 + max_position_embeddings: int = 4096,
98 + cache_config: Optional[CacheConfig] = None,
99 + quant_config: Optional[QuantizationConfig] = None,
100 + ) -> None:
101 + super().__init__()
102 + self.hidden_size = hidden_size
103 + tp_size = get_tensor_model_parallel_world_size()
104 + self.total_num_heads = num_heads
105 + assert self.total_num_heads % tp_size == 0
106 + self.num_heads = self.total_num_heads // tp_size
107 + self.total_num_kv_heads = num_kv_heads
108 + if self.total_num_kv_heads >= tp_size:
109 + # Number of KV heads is greater than TP size, so we partition
110 + # the KV heads across multiple tensor parallel GPUs.
111 + assert self.total_num_kv_heads % tp_size == 0
112 + else:
113 + # Number of KV heads is less than TP size, so we replicate
114 + # the KV heads across multiple tensor parallel GPUs.
115 + assert tp_size % self.total_num_kv_heads == 0
116 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
117 + self.head_dim = hidden_size // self.total_num_heads
118 + self.q_size = self.num_heads * self.head_dim
119 + self.kv_size = self.num_kv_heads * self.head_dim
120 + self.scaling = self.head_dim**-0.5
121 + self.rope_theta = rope_theta
122 + self.max_position_embeddings = max_position_embeddings
123 +
124 + self.qkv_proj = QKVParallelLinear(
125 + hidden_size,
126 + self.head_dim,
127 + self.total_num_heads,
128 + self.total_num_kv_heads,
129 + bias=False,
130 + quant_config=quant_config,
131 + )
132 + self.q_norm = RMSNorm(hidden_size, eps=1e-5)
133 + self.k_norm = RMSNorm(hidden_size, eps=1e-5)
134 + self.o_proj = RowParallelLinear(
135 + self.total_num_heads * self.head_dim,
136 + hidden_size,
137 + bias=False,
138 + quant_config=quant_config,
139 + )
140 +
141 + self.rotary_emb = get_rope(
142 + self.head_dim,
143 + rotary_dim=self.head_dim,
144 + max_position=max_position_embeddings,
145 + base=rope_theta,
146 + rope_scaling=rope_scaling,
147 + is_neox_style=True,
148 + )
149 + self.attn = Attention(self.num_heads,
150 + self.head_dim,
151 + self.scaling,
152 + num_kv_heads=self.num_kv_heads,
153 + cache_config=cache_config,
154 + quant_config=quant_config)
155 +
156 + def forward(
157 + self,
158 + positions: torch.Tensor,
159 + hidden_states: torch.Tensor,
160 + kv_cache: torch.Tensor,
161 + attn_metadata: AttentionMetadata,
162 + ) -> torch.Tensor:
163 + qkv, _ = self.qkv_proj(hidden_states)
164 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
165 + q, k = self.q_norm(q.contiguous()), self.k_norm(k.contiguous())
166 + q, k = self.rotary_emb(positions, q, k)
167 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
168 + output, _ = self.o_proj(attn_output)
169 + return output
170 +
171 +
172 +class OlmoeDecoderLayer(nn.Module):
173 +
174 + def __init__(
175 + self,
176 + config: PretrainedConfig,
177 + layer_idx: int,
178 + cache_config: Optional[CacheConfig] = None,
179 + quant_config: Optional[QuantizationConfig] = None,
180 + ) -> None:
181 + super().__init__()
182 + self.hidden_size = config.hidden_size
183 + rope_theta = getattr(config, "rope_theta", 10000)
184 + rope_scaling = getattr(config, "rope_scaling", None)
185 + max_position_embeddings = getattr(config, "max_position_embeddings",
186 + 4096)
187 +
188 + self.self_attn = OlmoeAttention(
189 + hidden_size=self.hidden_size,
190 + num_heads=config.num_attention_heads,
191 + num_kv_heads=config.num_key_value_heads,
192 + rope_theta=rope_theta,
193 + rope_scaling=rope_scaling,
194 + max_position_embeddings=max_position_embeddings,
195 + cache_config=cache_config,
196 + quant_config=quant_config,
197 + )
198 +
199 + self.mlp = OlmoeMoE(
200 + num_experts=config.num_experts,
201 + top_k=config.num_experts_per_tok,
202 + hidden_size=config.hidden_size,
203 + intermediate_size=config.intermediate_size,
204 + quant_config=quant_config,
205 + )
206 + self.input_layernorm = RMSNorm(config.hidden_size, eps=1e-5)
207 + self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=1e-5)
208 +
209 + def forward(
210 + self,
211 + positions: torch.Tensor,
212 + hidden_states: torch.Tensor,
213 + kv_cache: torch.Tensor,
214 + attn_metadata: AttentionMetadata,
215 + residual: Optional[torch.Tensor],
216 + ) -> torch.Tensor:
217 + # Self Attention
218 + if residual is None:
219 + residual = hidden_states
220 + hidden_states = self.input_layernorm(hidden_states)
221 + else:
222 + hidden_states, residual = self.input_layernorm(
223 + hidden_states, residual)
224 +
225 + hidden_states = self.self_attn(
226 + positions=positions,
227 + hidden_states=hidden_states,
228 + kv_cache=kv_cache,
229 + attn_metadata=attn_metadata,
230 + )
231 +
232 + # Fully Connected
233 + hidden_states, residual = self.post_attention_layernorm(
234 + hidden_states, residual)
235 + hidden_states = self.mlp(hidden_states)
236 + return hidden_states, residual
237 +
238 +
239 +class OlmoeModel(nn.Module):
240 +
241 + def __init__(
242 + self,
243 + config: PretrainedConfig,
244 + cache_config: Optional[CacheConfig] = None,
245 + quant_config: Optional[QuantizationConfig] = None,
246 + ) -> None:
247 + super().__init__()
248 + self.padding_idx = config.pad_token_id
249 + self.vocab_size = config.vocab_size
250 +
251 + self.embed_tokens = VocabParallelEmbedding(
252 + config.vocab_size,
253 + config.hidden_size,
254 + )
255 + self.layers = nn.ModuleList([
256 + OlmoeDecoderLayer(config,
257 + layer_idx,
258 + cache_config,
259 + quant_config=quant_config)
260 + for layer_idx in range(config.num_hidden_layers)
261 + ])
262 + self.norm = RMSNorm(config.hidden_size, eps=1e-5)
263 +
264 + def forward(
265 + self,
266 + input_ids: torch.Tensor,
267 + positions: torch.Tensor,
268 + kv_caches: List[torch.Tensor],
269 + attn_metadata: AttentionMetadata,
270 + ) -> torch.Tensor:
271 + hidden_states = self.embed_tokens(input_ids)
272 + residual = None
273 + for i in range(len(self.layers)):
274 + layer = self.layers[i]
275 + hidden_states, residual = layer(positions, hidden_states,
276 + kv_caches[i], attn_metadata,
277 + residual)
278 + hidden_states, _ = self.norm(hidden_states, residual)
279 + return hidden_states
280 +
281 +
282 +class OlmoeForCausalLM(nn.Module):
283 +
284 + fall_back_to_pt_during_load = False
285 +
286 + def __init__(
287 + self,
288 + config: PretrainedConfig,
289 + cache_config: Optional[CacheConfig] = None,
290 + quant_config: Optional[QuantizationConfig] = None,
291 + ) -> None:
292 + super().__init__()
293 + self.config = config
294 + self.quant_config = quant_config
295 + self.model = OlmoeModel(config, cache_config, quant_config)
296 + self.lm_head = ParallelLMHead(config.vocab_size,
297 + config.hidden_size,
298 + quant_config=quant_config)
299 + self.logits_processor = LogitsProcessor(config.vocab_size)
300 + self.sampler = Sampler()
301 +
302 + def forward(
303 + self,
304 + input_ids: torch.Tensor,
305 + positions: torch.Tensor,
306 + kv_caches: List[torch.Tensor],
307 + attn_metadata: AttentionMetadata,
308 + intermediate_tensors: Optional[IntermediateTensors] = None,
309 + ) -> torch.Tensor:
310 + hidden_states = self.model(input_ids, positions, kv_caches,
311 + attn_metadata)
312 + return hidden_states
313 +
314 + def compute_logits(self, hidden_states: torch.Tensor,
315 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
316 + logits = self.logits_processor(self.lm_head, hidden_states,
317 + sampling_metadata)
318 + return logits
319 +
320 + def sample(
321 + self,
322 + logits: Optional[torch.Tensor],
323 + sampling_metadata: SamplingMetadata,
324 + ) -> Optional[SamplerOutput]:
325 + next_tokens = self.sampler(logits, sampling_metadata)
326 + return next_tokens
327 +
328 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
329 + stacked_params_mapping = [
330 + # (param_name, shard_name, shard_id)
331 + ("qkv_proj", "q_proj", "q"),
332 + ("qkv_proj", "k_proj", "k"),
333 + ("qkv_proj", "v_proj", "v"),
334 + ("gate_up_proj", "gate_proj", 0),
335 + ("gate_up_proj", "up_proj", 1),
336 + ]
337 +
338 + # Params for weights, fp8 weight scales, fp8 activation scales
339 + # (param_name, weight_name, expert_id, shard_id)
340 + expert_params_mapping = FusedMoE.make_expert_params_mapping(
341 + ckpt_gate_proj_name="gate_proj",
342 + ckpt_down_proj_name="down_proj",
343 + ckpt_up_proj_name="up_proj",
344 + num_experts=self.config.num_experts)
345 +
346 + params_dict = dict(self.named_parameters())
347 + for name, loaded_weight in weights:
348 + if "rotary_emb.inv_freq" in name:
349 + continue
350 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
351 + # Skip non-stacked layers and experts (experts handled below).
352 + if weight_name not in name:
353 + continue
354 + # We have mlp.experts[0].gate_proj in the checkpoint.
355 + # Since we handle the experts below in expert_params_mapping,
356 + # we need to skip here BEFORE we update the name, otherwise
357 + # name will be updated to mlp.experts[0].gate_up_proj, which
358 + # will then be updated below in expert_params_mapping
359 + # for mlp.experts[0].gate_gate_up_proj, which breaks load.
360 + if "mlp.experts" in name:
361 + continue
362 + name = name.replace(weight_name, param_name)
363 + # Skip loading extra bias for GPTQ models.
364 + if name.endswith(".bias") and name not in params_dict:
365 + continue
366 + if name not in params_dict:
367 + continue
368 +
369 + param = params_dict[name]
370 + weight_loader = param.weight_loader
371 + weight_loader(param, loaded_weight, shard_id)
372 + break
373 + else:
374 + for mapping in expert_params_mapping:
375 + param_name, weight_name, expert_id, shard_id = mapping
376 + if weight_name not in name:
377 + continue
378 + name = name.replace(weight_name, param_name)
379 + param = params_dict[name]
380 + weight_loader = param.weight_loader
381 + weight_loader(param,
382 + loaded_weight,
383 + name,
384 + shard_id=shard_id,
385 + expert_id=expert_id)
386 + break
387 + else:
388 + # Skip loading extra bias for GPTQ models.
389 + if name.endswith(".bias") and name not in params_dict:
390 + continue
391 + # Remapping the name of FP8 kv-scale.
392 + if name.endswith("kv_scale"):
393 + remapped_kv_scale_name = name.replace(
394 + ".kv_scale", ".attn.kv_scale")
395 + if remapped_kv_scale_name not in params_dict:
396 + print_warning_once(
397 + "Found kv scale in the checkpoint "
398 + f"(e.g. {name}), but not found the expected "
399 + f"name in the model "
400 + f"(e.g. {remapped_kv_scale_name}). "
401 + "kv-scale is not loaded.")
402 + continue
403 + else:
404 + name = remapped_kv_scale_name
405 +
406 + param = params_dict[name]
407 + weight_loader = getattr(param, "weight_loader",
408 + default_weight_loader)
409 + weight_loader(param, loaded_weight)