@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4a151dd453
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add activation registry (#126)

Woosuk Kwon committed May 25, 2023 at 00:09 UTC 4a151dd45308f812d125b6ea239b1730b7f647c4
5 files changed +22 -13
cacheflow/entrypoints/llm.py
+1 -1
@@ -61,7 +61,7 @@ class LLM:
61 while self.llm_server.has_unfinished_requests():
62 step_outputs = self.llm_server.step()
63 for output in step_outputs:
64 - if output.done:
64 + if output.finished():
65 outputs.append(output)
66 if use_tqdm:
67 pbar.update(1)
cacheflow/model_executor/layers/activation.py
+15
@@ -4,6 +4,21 @@ import torch.nn as nn
4
5 from cacheflow import activation_ops
6
7 +_ACTIVATION_REGISTRY = {
8 + "gelu": nn.GELU(),
9 + "gelu_new": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
10 + "gelu_fast": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
11 + "relu": nn.ReLU(),
12 +}
13 +
14 +
15 +def get_act_fn(act_fn: str) -> nn.Module:
16 + """Get an activation function by name."""
17 + act_fn = act_fn.lower()
18 + if act_fn in _ACTIVATION_REGISTRY:
19 + return _ACTIVATION_REGISTRY[act_fn]
20 + raise ValueError(f"Activation function {act_fn!r} is not supported.")
21 +
22
23 class SiluAndMul(nn.Module):
24 """An activation function for SwiGLU.
cacheflow/model_executor/models/gpt2.py
+2 -6
@@ -27,6 +27,7 @@ from torch import nn
27 from transformers import GPT2Config
28
29 from cacheflow.model_executor.input_metadata import InputMetadata
30 +from cacheflow.model_executor.layers.activation import get_act_fn
31 from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
32 from cacheflow.model_executor.layers.sampler import Sampler
33 from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
@@ -92,12 +93,7 @@ class GPT2MLP(nn.Module):
93 self.c_proj = RowParallelLinear(intermediate_size, hidden_size,
94 bias=True, input_is_parallel=True,
95 perform_initialization=False)
95 -
96 - act_fn = config.activation_function
97 - if act_fn != "gelu_new":
98 - raise ValueError(f"Unsupported activation: {act_fn}. "
99 - "GPT-2 only supports gelu_new for now.")
100 - self.act = torch.nn.GELU(approximate="tanh")
96 + self.act = get_act_fn(config.activation_function)
97
98 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
99 hidden_states, _ = self.c_fc(hidden_states)
cacheflow/model_executor/models/gpt_neox.py
+2 -4
@@ -26,6 +26,7 @@ from torch import nn
26 from transformers import GPTNeoXConfig
27
28 from cacheflow.model_executor.input_metadata import InputMetadata
29 +from cacheflow.model_executor.layers.activation import get_act_fn
30 from cacheflow.model_executor.layers.attention import GPTNeoXCacheFlowAttention
31 from cacheflow.model_executor.layers.sampler import Sampler
32 from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
@@ -94,10 +95,7 @@ class GPTNeoXMLP(nn.Module):
95 self.dense_4h_to_h = RowParallelLinear(config.intermediate_size, config.hidden_size,
96 input_is_parallel=True,
97 perform_initialization=False)
97 - if config.hidden_act != 'gelu':
98 - raise ValueError(f'Unsupported activation: {config.hidden_act}. '
99 - 'Only gelu is supported for now.')
100 - self.act = torch.nn.GELU()
98 + self.act = get_act_fn(config.hidden_act)
99
100 def forward(self, hidden_states):
101 hidden_states, _ = self.dense_h_to_4h(hidden_states)
cacheflow/model_executor/models/opt.py
+2 -2
@@ -26,6 +26,7 @@ from torch import nn
26 from transformers import OPTConfig
27
28 from cacheflow.model_executor.input_metadata import InputMetadata
29 +from cacheflow.model_executor.layers.activation import get_act_fn
30 from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
31 from cacheflow.model_executor.layers.sampler import Sampler
32 from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
@@ -105,8 +106,7 @@ class OPTDecoderLayer(nn.Module):
106 bias=config.enable_bias,
107 )
108 self.do_layer_norm_before = config.do_layer_norm_before
108 - assert config.activation_function == 'relu'
109 - self.activation_fn = nn.ReLU()
109 + self.activation_fn = get_act_fn(config.activation_function)
110
111 self.self_attn_layer_norm = nn.LayerNorm(
112 self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)