A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix incorrect output on OLMo models in Tensor Parallelism (#3869)
Isotr0py committed
Apr 6, 2024 at 03:02 UTC
54951ac4bfb7f4224cb8f5ffc89b214c950107d8
1 file changed
+12
-19
vllm/model_executor/models/olmo.py
+12
-19
@@ -39,14 +39,15 @@
39
from typing import List, Optional, Tuple
40
41
import torch
42
-import torch.nn.functional as F
42
# this model must need this dependency
43
from hf_olmo import OLMoConfig
44
from torch import nn
45
46
from vllm.attention import Attention, AttentionMetadata
47
+from vllm.model_executor.layers.activation import SiluAndMul
48
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
49
LinearMethodBase,
50
+ MergedColumnParallelLinear,
51
QKVParallelLinear,
52
RowParallelLinear)
53
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -62,17 +63,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
63
from vllm.sequence import SamplerOutput
64
65
65
-class SwiGLU(nn.Module):
66
-
67
- def forward(self, x: torch.Tensor) -> torch.Tensor:
68
- x, gate = x.chunk(2, dim=-1)
69
- return F.silu(gate) * x
70
-
71
- @property
72
- def output_multiplier(self) -> float:
73
- return 0.5
74
-
75
-
66
class OlmoAttention(nn.Module):
67
"""
68
This is the attention block where the output is computed as
@@ -174,17 +164,16 @@ class OlmoMLP(nn.Module):
164
bias=False)
165
166
# Feed-forward input projection.
177
- self.ff_proj = ColumnParallelLinear(
167
+ self.ff_proj = MergedColumnParallelLinear(
168
config.d_model,
179
- self.hidden_size,
169
+ [self.hidden_size // 2] * 2,
170
bias=config.include_bias,
171
linear_method=linear_method,
172
)
173
174
# Activation function.
185
- # self.act = SiluAndMul()
186
- # self.act.output_multiplier = 0.5
187
- self.act = SwiGLU()
175
+ self.act = SiluAndMul()
176
+ self.act.output_multiplier = 0.5
177
assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
178
179
# Feed-forward output projection.
@@ -374,8 +363,12 @@ class OLMoForCausalLM(nn.Module):
363
if ".att" in name:
364
name = name.replace(".att", ".attn.att")
365
# mlp
377
- if ".ff" in name and "transformer.ff_out" not in name:
378
- name = name.replace(".ff", ".mlp.ff")
366
+ if ".ff_proj" in name:
367
+ name = name.replace(".ff_proj", ".mlp.ff_proj")
368
+ # Reverse the weight for the MergeColumnParallelLinear
369
+ loaded_weight = torch.concat(loaded_weight.chunk(2)[::-1])
370
+ if ".ff_out" in name and "transformer.ff_out" not in name:
371
+ name = name.replace(".ff_out", ".mlp.ff_out")
372
# there is no bias in olmo
373
param = params_dict[name]
374
weight_loader = getattr(param, "weight_loader",