@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 42f5e7c52a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Kernel] Support MulAndSilu (#11624)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Jan 15, 2025 at 10:29 UTC 42f5e7c52a5852e20937001332572c8cb8115af0
7 files changed +83 -36
csrc/activation_kernels.cu
+25 -7
@@ -9,8 +9,16 @@
9
10 namespace vllm {
11
12 +template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
13 + bool act_first>
14 +__device__ __forceinline__ scalar_t compute(const scalar_t& x,
15 + const scalar_t& y) {
16 + return act_first ? ACT_FN(x) * y : x * ACT_FN(y);
17 +}
18 // Activation and gating kernel template.
13 -template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
19 +
20 +template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
21 + bool act_first>
22 __global__ void act_and_mul_kernel(
23 scalar_t* __restrict__ out, // [..., d]
24 const scalar_t* __restrict__ input, // [..., 2, d]
@@ -19,7 +27,7 @@ __global__ void act_and_mul_kernel(
27 for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
28 const scalar_t x = VLLM_LDG(&input[token_idx * 2 * d + idx]);
29 const scalar_t y = VLLM_LDG(&input[token_idx * 2 * d + d + idx]);
22 - out[token_idx * d + idx] = ACT_FN(x) * y;
30 + out[token_idx * d + idx] = compute<scalar_t, ACT_FN, act_first>(x, y);
31 }
32 }
33
@@ -55,7 +63,9 @@ __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
63 } // namespace vllm
64
65 // Launch activation and gating kernel.
58 -#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
66 +// Use ACT_FIRST (bool) indicating whether to apply the activation function
67 +// first.
68 +#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL, ACT_FIRST) \
69 int d = input.size(-1) / 2; \
70 int64_t num_tokens = input.numel() / input.size(-1); \
71 dim3 grid(num_tokens); \
@@ -64,7 +74,7 @@ __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
74 const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
75 VLLM_DISPATCH_FLOATING_TYPES( \
76 input.scalar_type(), "act_and_mul_kernel", [&] { \
67 - vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>> \
77 + vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>, ACT_FIRST> \
78 <<<grid, block, 0, stream>>>(out.data_ptr<scalar_t>(), \
79 input.data_ptr<scalar_t>(), d); \
80 });
@@ -72,19 +82,27 @@ __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
82 void silu_and_mul(torch::Tensor& out, // [..., d]
83 torch::Tensor& input) // [..., 2 * d]
84 {
75 - LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel);
85 + LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, true);
86 +}
87 +
88 +void mul_and_silu(torch::Tensor& out, // [..., d]
89 + torch::Tensor& input) // [..., 2 * d]
90 +{
91 + // The difference between mul_and_silu and silu_and_mul is that mul_and_silu
92 + // applies the silu to the latter half of the input.
93 + LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, false);
94 }
95
96 void gelu_and_mul(torch::Tensor& out, // [..., d]
97 torch::Tensor& input) // [..., 2 * d]
98 {
81 - LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel);
99 + LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel, true);
100 }
101
102 void gelu_tanh_and_mul(torch::Tensor& out, // [..., d]
103 torch::Tensor& input) // [..., 2 * d]
104 {
87 - LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel);
105 + LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel, true);
106 }
107
108 namespace vllm {
csrc/ops.h
+2
@@ -86,6 +86,8 @@ void batched_rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
86
87 void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
88
89 +void mul_and_silu(torch::Tensor& out, torch::Tensor& input);
90 +
91 void gelu_and_mul(torch::Tensor& out, torch::Tensor& input);
92
93 void gelu_tanh_and_mul(torch::Tensor& out, torch::Tensor& input);
csrc/torch_bindings.cpp
+3
@@ -55,6 +55,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
55 ops.def("silu_and_mul(Tensor! out, Tensor input) -> ()");
56 ops.impl("silu_and_mul", torch::kCUDA, &silu_and_mul);
57
58 + ops.def("mul_and_silu(Tensor! out, Tensor input) -> ()");
59 + ops.impl("mul_and_silu", torch::kCUDA, &mul_and_silu);
60 +
61 // Activation function used in GeGLU with `none` approximation.
62 ops.def("gelu_and_mul(Tensor! out, Tensor input) -> ()");
63 ops.impl("gelu_and_mul", torch::kCUDA, &gelu_and_mul);
tests/kernels/test_activation.py
+13 -7
@@ -6,8 +6,9 @@ import torch
6
7 from tests.kernels.utils import opcheck
8 from vllm.model_executor.layers.activation import (FastGELU, FatreluAndMul,
9 - GeluAndMul, NewGELU,
10 - QuickGELU, SiluAndMul)
9 + GeluAndMul, MulAndSilu,
10 + NewGELU, QuickGELU,
11 + SiluAndMul)
12 from vllm.platforms import current_platform
13
14 from .allclose_default import get_default_atol, get_default_rtol
@@ -21,8 +22,9 @@ CUDA_DEVICES = [
22 ]
23
24
24 -@pytest.mark.parametrize("activation",
25 - ["silu", "gelu", "gelu_tanh", "fatrelu"])
25 +@pytest.mark.parametrize(
26 + "activation",
27 + ["silu_and_mul", "mul_and_silu", "gelu", "gelu_tanh", "fatrelu"])
28 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
29 @pytest.mark.parametrize("d", D)
30 @pytest.mark.parametrize("dtype", DTYPES)
@@ -40,9 +42,12 @@ def test_act_and_mul(
42 current_platform.seed_everything(seed)
43 torch.set_default_device(device)
44 x = torch.randn(num_tokens, 2 * d, dtype=dtype)
43 - if activation == "silu":
45 + if activation == "silu_and_mul":
46 layer = SiluAndMul()
47 fn = torch.ops._C.silu_and_mul
48 + if activation == "mul_and_silu":
49 + layer = MulAndSilu()
50 + fn = torch.ops._C.mul_and_silu
51 elif activation == "gelu":
52 layer = GeluAndMul(approximate="none")
53 fn = torch.ops._C.gelu_and_mul
@@ -55,8 +60,9 @@ def test_act_and_mul(
60 fn = torch.ops._C.fatrelu_and_mul
61 out = layer(x)
62 ref_out = layer.forward_native(x)
58 - # The SiLU, GELU and FatReLU implementations are equivalent to the native
59 - # PyTorch implementations, so we can do exact comparison.
63 + # The SiluAndMul, MulAndSilu, GELU and FatReLU implementations are
64 + # equivalent to the native PyTorch implementations, so we can do exact
65 + # comparison.
66 torch.testing.assert_close(out, ref_out, atol=0.0, rtol=0.0)
67
68 d = x.shape[-1] // 2
vllm/model_executor/layers/activation.py
+35
@@ -87,6 +87,41 @@ class SiluAndMul(CustomOp):
87 return out
88
89
90 +@CustomOp.register("mul_and_silu")
91 +class MulAndSilu(CustomOp):
92 + """An activation function for SwiGLU.
93 +
94 + The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2.
95 +
96 + Shapes:
97 + x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
98 + return: (num_tokens, d) or (batch_size, seq_len, d)
99 + """
100 +
101 + def __init__(self):
102 + super().__init__()
103 + if current_platform.is_cuda_alike() or current_platform.is_cpu():
104 + self.op = torch.ops._C.mul_and_silu
105 + elif current_platform.is_xpu():
106 + from vllm._ipex_ops import ipex_ops
107 + self.op = ipex_ops.silu_and_mul
108 +
109 + def forward_native(self, x: torch.Tensor) -> torch.Tensor:
110 + """PyTorch-native implementation equivalent to forward()."""
111 + d = x.shape[-1] // 2
112 + return x[..., :d] * F.silu(x[..., d:])
113 +
114 + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
115 + d = x.shape[-1] // 2
116 + output_shape = (x.shape[:-1] + (d, ))
117 + out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
118 + self.op(out, x)
119 + return out
120 +
121 + # TODO implement forward_xpu for MulAndSilu
122 + # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
123 +
124 +
125 @CustomOp.register("gelu_and_mul")
126 class GeluAndMul(CustomOp):
127 """An activation function for GeGLU.
vllm/model_executor/models/molmo.py
+3 -11
@@ -23,7 +23,8 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
23 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
24 InputContext, token_inputs)
25 from vllm.model_executor import SamplingMetadata
26 -from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
26 +from vllm.model_executor.layers.activation import (MulAndSilu, QuickGELU,
27 + SiluAndMul)
28 from vllm.model_executor.layers.layernorm import RMSNorm
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 MergedColumnParallelLinear,
@@ -462,15 +463,6 @@ class MolmoAttention(nn.Module):
463 return output
464
465
465 -class SwiGLU(nn.Module):
466 -
467 - def forward(self, x: torch.Tensor) -> torch.Tensor:
468 - x, gate = x.chunk(2, dim=-1)
469 - # Note that the order is reversed compared to
470 - # SiluAndMul.
471 - return x * F.silu(gate)
472 -
473 -
466 class LanuageModelMLP(nn.Module):
467 """Molmo's LLM mlp."""
468
@@ -489,7 +481,7 @@ class LanuageModelMLP(nn.Module):
481 quant_config=quant_config,
482 )
483 # Activation function.
492 - self.act_fn = SwiGLU()
484 + self.act_fn = MulAndSilu()
485 # Feed-forward output projection.
486 self.down_proj = RowParallelLinear(
487 self.intermediate_size,
vllm/model_executor/models/ultravox.py
+2 -11
@@ -16,7 +16,7 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
16 from vllm import envs
17 from vllm.attention import AttentionMetadata
18 from vllm.config import VllmConfig
19 -from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
19 +from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
20 from vllm.model_executor.layers.layernorm import RMSNorm
21 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22 from vllm.model_executor.model_loader.loader import DefaultModelLoader
@@ -248,15 +248,6 @@ class StackAudioFrames(nn.Module):
248 return audio_embeds
249
250
251 -class FlippedSiluAndMul(SiluAndMul):
252 - """Ultravox is trained with SwiGLU with flipped halves."""
253 -
254 - def forward(self, x: torch.Tensor):
255 - a, b = x.chunk(2, dim=-1)
256 - flipped = torch.cat((b, a), dim=-1)
257 - return super().forward(flipped)
258 -
259 -
251 class UltravoxProjector(nn.Module):
252
253 def __init__(self, config: UltravoxConfig):
@@ -269,7 +260,7 @@ class UltravoxProjector(nn.Module):
260 dim = self.hidden_dim
261
262 if config.projector_act == "swiglu":
272 - self.act = FlippedSiluAndMul()
263 + self.act = MulAndSilu()
264 dim = dim // 2
265 else:
266 self.act = get_act_fn(config.projector_act)