@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 54951ac4bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix incorrect output on OLMo models in Tensor Parallelism (#3869)

Isotr0py committed Apr 6, 2024 at 03:02 UTC 54951ac4bfb7f4224cb8f5ffc89b214c950107d8
1 file changed +12 -19
vllm/model_executor/models/olmo.py
+12 -19
@@ -39,14 +39,15 @@
39 from typing import List, Optional, Tuple
40
41 import torch
42 -import torch.nn.functional as F
42 # this model must need this dependency
43 from hf_olmo import OLMoConfig
44 from torch import nn
45
46 from vllm.attention import Attention, AttentionMetadata
47 +from vllm.model_executor.layers.activation import SiluAndMul
48 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
49 LinearMethodBase,
50 + MergedColumnParallelLinear,
51 QKVParallelLinear,
52 RowParallelLinear)
53 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -62,17 +63,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
63 from vllm.sequence import SamplerOutput
64
65
65 -class SwiGLU(nn.Module):
66 -
67 - def forward(self, x: torch.Tensor) -> torch.Tensor:
68 - x, gate = x.chunk(2, dim=-1)
69 - return F.silu(gate) * x
70 -
71 - @property
72 - def output_multiplier(self) -> float:
73 - return 0.5
74 -
75 -
66 class OlmoAttention(nn.Module):
67 """
68 This is the attention block where the output is computed as
@@ -174,17 +164,16 @@ class OlmoMLP(nn.Module):
164 bias=False)
165
166 # Feed-forward input projection.
177 - self.ff_proj = ColumnParallelLinear(
167 + self.ff_proj = MergedColumnParallelLinear(
168 config.d_model,
179 - self.hidden_size,
169 + [self.hidden_size // 2] * 2,
170 bias=config.include_bias,
171 linear_method=linear_method,
172 )
173
174 # Activation function.
185 - # self.act = SiluAndMul()
186 - # self.act.output_multiplier = 0.5
187 - self.act = SwiGLU()
175 + self.act = SiluAndMul()
176 + self.act.output_multiplier = 0.5
177 assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
178
179 # Feed-forward output projection.
@@ -374,8 +363,12 @@ class OLMoForCausalLM(nn.Module):
363 if ".att" in name:
364 name = name.replace(".att", ".attn.att")
365 # mlp
377 - if ".ff" in name and "transformer.ff_out" not in name:
378 - name = name.replace(".ff", ".mlp.ff")
366 + if ".ff_proj" in name:
367 + name = name.replace(".ff_proj", ".mlp.ff_proj")
368 + # Reverse the weight for the MergeColumnParallelLinear
369 + loaded_weight = torch.concat(loaded_weight.chunk(2)[::-1])
370 + if ".ff_out" in name and "transformer.ff_out" not in name:
371 + name = name.replace(".ff_out", ".mlp.ff_out")
372 # there is no bias in olmo
373 param = params_dict[name]
374 weight_loader = getattr(param, "weight_loader",