A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[torch.compile] Fine-grained CustomOp enabling mechanism (#9300)
Luka Govedič committed
Oct 17, 2024 at 14:36 UTC
0f41fbe5a370c0b87bb9a038be592c9272d46364
8 files changed
+220
-21
tests/model_executor/test_enabled_custom_ops.py
new
+92
@@ -0,0 +1,92 @@
1
+import os
2
+from typing import List
3
+
4
+import pytest
5
+
6
+from vllm.model_executor.custom_op import CustomOp
7
+from vllm.model_executor.layers.activation import (GeluAndMul,
8
+ ReLUSquaredActivation,
9
+ SiluAndMul)
10
+from vllm.model_executor.layers.layernorm import RMSNorm
11
+
12
+
13
+# Registered subclass for test
14
+@CustomOp.register("relu3")
15
+class Relu3(ReLUSquaredActivation):
16
+ pass
17
+
18
+
19
+@pytest.mark.parametrize(
20
+ "env, torch_level, ops_enabled, default_on",
21
+ [
22
+ # Default values based on compile level
23
+ ("", 0, [True] * 4, True),
24
+ ("", 1, [True] * 4, True),
25
+ ("", 2, [True] * 4, True), # All by default
26
+ ("", 3, [False] * 4, False),
27
+ ("", 4, [False] * 4, False), # None by default
28
+ # Explicitly enabling/disabling
29
+ #
30
+ # Default: all
31
+ #
32
+ # All but SiluAndMul
33
+ ("+rms_norm,-silu_and_mul", 0, [1, 0, 1, 1], True),
34
+ # Only ReLU3
35
+ ("none,-rms_norm,+relu3", 0, [0, 0, 0, 1], False),
36
+ # All but SiluAndMul
37
+ ("all,-silu_and_mul", 1, [1, 0, 1, 1], True),
38
+ # All but ReLU3 (even if ReLU2 is on)
39
+ ("-relu3,relu2", 1, [1, 1, 1, 0], True),
40
+ # GeluAndMul and SiluAndMul
41
+ ("none,-relu3,+gelu_and_mul,+silu_and_mul", 2, [0, 1, 1, 0], False),
42
+ # All but RMSNorm
43
+ ("-rms_norm", 2, [0, 1, 1, 1], True),
44
+ #
45
+ # Default: none
46
+ #
47
+ # Only ReLU3
48
+ ("-silu_and_mul,+relu3", 3, [0, 0, 0, 1], False),
49
+ # All but RMSNorm
50
+ ("all,-rms_norm", 4, [0, 1, 1, 1], True),
51
+ ])
52
+def test_enabled_ops(env: str, torch_level: int, ops_enabled: List[int],
53
+ default_on: bool):
54
+ os.environ["VLLM_CUSTOM_OPS"] = env
55
+ os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(torch_level)
56
+
57
+ # Reset default_on (computed once):
58
+ CustomOp.default_on.cache_clear()
59
+
60
+ assert CustomOp.default_on() == default_on
61
+
62
+ ops_enabled = [bool(x) for x in ops_enabled]
63
+
64
+ assert RMSNorm(1024).enabled() == ops_enabled[0]
65
+ assert CustomOp.op_registry["rms_norm"].enabled() == ops_enabled[0]
66
+
67
+ assert SiluAndMul().enabled() == ops_enabled[1]
68
+ assert CustomOp.op_registry["silu_and_mul"].enabled() == ops_enabled[1]
69
+
70
+ assert GeluAndMul().enabled() == ops_enabled[2]
71
+ assert CustomOp.op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
72
+
73
+ # If registered, subclasses should follow their own name
74
+ assert Relu3().enabled() == ops_enabled[3]
75
+ assert CustomOp.op_registry["relu3"].enabled() == ops_enabled[3]
76
+
77
+ # Unregistered subclass
78
+ class SiluAndMul2(SiluAndMul):
79
+ pass
80
+
81
+ # Subclasses should not require registration
82
+ assert SiluAndMul2().enabled() == SiluAndMul().enabled()
83
+
84
+
85
+@pytest.mark.parametrize(
86
+ "env", ["all,none", "all,+rms_norm,all", "+rms_norm,-rms_norm"])
87
+def test_enabled_ops_invalid(env: str):
88
+ os.environ["VLLM_CUSTOM_OPS"] = env
89
+ CustomOp.default_on.cache_clear()
90
+
91
+ with pytest.raises(AssertionError):
92
+ RMSNorm(1024).enabled()
vllm/envs.py
+12
-1
@@ -65,6 +65,7 @@ if TYPE_CHECKING:
65
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66
VLLM_SKIP_P2P_CHECK: bool = False
67
VLLM_TORCH_COMPILE_LEVEL: int = 0
68
+ VLLM_CUSTOM_OPS: List[str] = []
69
VLLM_DISABLED_KERNELS: List[str] = []
70
71
@@ -205,7 +206,17 @@ environment_variables: Dict[str, Callable[[], Any]] = {
206
os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
207
"VLLM_TORCH_COMPILE_LEVEL":
208
lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")),
208
-
209
+ # Fine-grained control over which custom ops to enable/disable.
210
+ # Use 'all' to enable all, 'none' to disable all.
211
+ # Also specify a list of custom op names to enable (prefixed with a '+'),
212
+ # or disable (prefixed with a '-').
213
+ # Examples:
214
+ # - 'all,-op1' to enable all except op1
215
+ # - 'none,+op1,+op2' to enable only op1 and op2
216
+ # By default, all custom ops are enabled when running without Inductor
217
+ # and disabled when running with Inductor (compile_level >= Inductor).
218
+ "VLLM_CUSTOM_OPS":
219
+ lambda: os.environ.get("VLLM_CUSTOM_OPS", "").replace(" ", "").split(","),
220
# local rank of the process in the distributed setting, used to determine
221
# the GPU device id
222
"LOCAL_RANK":
vllm/model_executor/custom_op.py
+64
-4
@@ -1,14 +1,24 @@
1
+from functools import lru_cache
2
+from typing import Dict, Type
3
+
4
import torch.nn as nn
5
6
import vllm.envs as envs
7
from vllm.compilation.levels import CompilationLevel
8
+from vllm.logger import init_logger
9
from vllm.platforms import current_platform
6
-from vllm.utils import is_cpu, is_hip, is_xpu
10
+from vllm.utils import is_cpu, is_hip, is_xpu, print_warning_once
11
+
12
+logger = init_logger(__name__)
13
14
15
class CustomOp(nn.Module):
16
+ """
17
+ Base class for custom ops.
18
+ Dispatches the forward method to the appropriate backend.
19
+ """
20
11
- def __init__(self, *args, **kwargs):
21
+ def __init__(self):
22
super().__init__()
23
self._forward_method = self.dispatch_forward()
24
@@ -17,7 +27,6 @@ class CustomOp(nn.Module):
27
28
def forward_native(self, *args, **kwargs):
29
"""PyTorch-native implementation of the forward method.
20
-
30
This method is optional. If implemented, it can be used with compilers
31
such as torch.compile or PyTorch XLA. Also, it can be used for testing
32
purposes.
@@ -56,7 +65,11 @@ class CustomOp(nn.Module):
65
# NOTE(woosuk): Here we assume that vLLM was built for only one
66
# specific backend. Currently, we do not support dynamic dispatching.
67
59
- if envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.INDUCTOR:
68
+ enabled = self.enabled()
69
+ logger.debug("custom op %s %s", self.__class__.name,
70
+ "enabled" if enabled else "disabled")
71
+
72
+ if not enabled:
73
return self.forward_native
74
75
if is_hip():
@@ -69,3 +82,50 @@ class CustomOp(nn.Module):
82
return self.forward_xpu
83
else:
84
return self.forward_cuda
85
+
86
+ @classmethod
87
+ def enabled(cls) -> bool:
88
+ # if no name, then it was not registered
89
+ if not hasattr(cls, "name"):
90
+ print_warning_once(
91
+ f"Custom op {cls.__name__} was not registered, "
92
+ f"which means it won't appear in the op registry. "
93
+ f"It will be enabled/disabled based on the global settings.")
94
+ return CustomOp.default_on()
95
+
96
+ enabled = f"+{cls.name}" in envs.VLLM_CUSTOM_OPS
97
+ disabled = f"-{cls.name}" in envs.VLLM_CUSTOM_OPS
98
+ assert not (enabled
99
+ and disabled), f"Cannot enable and disable {cls.name}"
100
+
101
+ return (CustomOp.default_on() or enabled) and not disabled
102
+
103
+ # On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR
104
+ # Specifying 'all' or 'none' in VLLM_CUSTOM_OPS takes precedence.
105
+ @staticmethod
106
+ @lru_cache()
107
+ def default_on() -> bool:
108
+ count_none = envs.VLLM_CUSTOM_OPS.count("none")
109
+ count_all = envs.VLLM_CUSTOM_OPS.count("all")
110
+ assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
111
+ return envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR and \
112
+ not count_none > 0 or count_all > 0
113
+
114
+ # Dictionary of all custom ops (classes, indexed by registered name).
115
+ # To check if an op with a name is enabled, call .enabled() on the class.
116
+ # Examples:
117
+ # - MyOp.enabled()
118
+ # - op_registry["my_op"].enabled()
119
+ op_registry: Dict[str, Type['CustomOp']] = {}
120
+
121
+ # Decorator to register custom ops.
122
+ @classmethod
123
+ def register(cls, name: str):
124
+
125
+ def decorator(op_cls):
126
+ assert name not in cls.op_registry, f"Duplicate op name: {name}"
127
+ op_cls.name = name
128
+ cls.op_registry[name] = op_cls
129
+ return op_cls
130
+
131
+ return decorator
vllm/model_executor/layers/activation.py
+25
-11
@@ -11,11 +11,13 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
11
from vllm.model_executor.custom_op import CustomOp
12
from vllm.model_executor.layers.quantization import QuantizationConfig
13
from vllm.model_executor.utils import set_weight_attrs
14
+from vllm.utils import LazyDict
15
16
17
+@CustomOp.register("fatrelu_and_mul")
18
class FatreluAndMul(CustomOp):
19
"""An activation function for FATReLU.
18
-
20
+
21
The function computes x -> FATReLU(x[:d]) * x[d:] where
22
d = x.shape[-1] // 2.
23
This is used in openbmb/MiniCPM-S-1B-sft.
@@ -40,6 +42,7 @@ class FatreluAndMul(CustomOp):
42
return self.forward_native(x)
43
44
45
+@CustomOp.register("silu_and_mul")
46
class SiluAndMul(CustomOp):
47
"""An activation function for SwiGLU.
48
@@ -74,6 +77,7 @@ class SiluAndMul(CustomOp):
77
return out
78
79
80
+@CustomOp.register("gelu_and_mul")
81
class GeluAndMul(CustomOp):
82
"""An activation function for GeGLU.
83
@@ -123,6 +127,7 @@ class GeluAndMul(CustomOp):
127
return f'approximate={repr(self.approximate)}'
128
129
130
+@CustomOp.register("gelu_new")
131
class NewGELU(CustomOp):
132
133
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
@@ -144,6 +149,7 @@ class NewGELU(CustomOp):
149
return ops.gelu_new(x)
150
151
152
+@CustomOp.register("gelu_fast")
153
class FastGELU(CustomOp):
154
155
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
@@ -164,8 +170,8 @@ class FastGELU(CustomOp):
170
return ops.gelu_fast(x)
171
172
173
+@CustomOp.register("quick_gelu")
174
class QuickGELU(CustomOp):
168
-
175
# https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90
176
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
177
"""PyTorch-native implementation equivalent to forward()."""
@@ -189,6 +195,7 @@ class QuickGELU(CustomOp):
195
# def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
196
197
198
+@CustomOp.register("relu2")
199
class ReLUSquaredActivation(CustomOp):
200
"""
201
Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2
@@ -244,15 +251,22 @@ class ScaledActivation(nn.Module):
251
param_data.copy_(loaded_weight)
252
253
247
-_ACTIVATION_REGISTRY = {
248
- "gelu": nn.GELU(),
249
- "gelu_fast": FastGELU(),
250
- "gelu_new": NewGELU(),
251
- "gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
252
- "relu": nn.ReLU(),
253
- "relu2": ReLUSquaredActivation(),
254
- "quick_gelu": QuickGELU(),
255
-}
254
+_ACTIVATION_REGISTRY = LazyDict({
255
+ "gelu":
256
+ lambda: nn.GELU(),
257
+ "gelu_fast":
258
+ lambda: FastGELU(),
259
+ "gelu_new":
260
+ lambda: NewGELU(),
261
+ "gelu_pytorch_tanh":
262
+ lambda: nn.GELU(approximate="tanh"),
263
+ "relu":
264
+ lambda: nn.ReLU(),
265
+ "relu2":
266
+ lambda: ReLUSquaredActivation(),
267
+ "quick_gelu":
268
+ lambda: QuickGELU(),
269
+})
270
271
272
def get_act_fn(
vllm/model_executor/layers/fused_moe/layer.py
+1
-4
@@ -37,13 +37,13 @@ class FusedMoEMethodBase(QuantizeMethodBase):
37
raise NotImplementedError
38
39
40
+@CustomOp.register("unquantized_fused_moe")
41
class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
42
"""MoE method without quantization."""
43
44
def create_weights(self, layer: torch.nn.Module, num_experts: int,
45
hidden_size: int, intermediate_size: int,
46
params_dtype: torch.dtype, **extra_weight_attrs):
46
-
47
# Fused gate_up_proj (column parallel)
48
w13_weight = torch.nn.Parameter(torch.empty(num_experts,
49
2 * intermediate_size,
@@ -74,7 +74,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
74
num_expert_group: Optional[int] = None,
75
custom_routing_function: Optional[Callable] = None
76
) -> torch.Tensor:
77
-
77
return self.forward(x=x,
78
layer=layer,
79
router_logits=router_logits,
@@ -97,7 +96,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
96
num_expert_group: Optional[int] = None,
97
custom_routing_function: Optional[Callable] = None
98
) -> torch.Tensor:
100
-
99
from vllm.model_executor.layers.fused_moe.fused_moe import (
100
fused_experts)
101
@@ -134,7 +132,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
132
num_expert_group: Optional[int] = None,
133
custom_routing_function: Optional[Callable] = None
134
) -> torch.Tensor:
137
-
135
from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe
136
assert not use_grouped_topk
137
assert num_expert_group is None
vllm/model_executor/layers/layernorm.py
+2
@@ -7,6 +7,7 @@ import torch.nn as nn
7
from vllm.model_executor.custom_op import CustomOp
8
9
10
+@CustomOp.register("rms_norm")
11
class RMSNorm(CustomOp):
12
"""Root mean square normalization.
13
@@ -122,6 +123,7 @@ class RMSNorm(CustomOp):
123
return s
124
125
126
+@CustomOp.register("gemma_rms_norm")
127
class GemmaRMSNorm(CustomOp):
128
"""RMS normalization for Gemma.
129
vllm/model_executor/layers/rotary_embedding.py
+2
-1
@@ -72,6 +72,7 @@ def _apply_rotary_emb(
72
return torch.stack((o1, o2), dim=-1).flatten(-2)
73
74
75
+@CustomOp.register("rotary_embedding")
76
class RotaryEmbedding(CustomOp):
77
"""Original rotary positional embedding."""
78
@@ -468,7 +469,7 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
469
self.long_factor = long_factor
470
471
scale = self.max_position_embeddings / \
471
- self.original_max_position_embeddings
472
+ self.original_max_position_embeddings
473
if scale <= 1.0:
474
scaling_factor = 1.0
475
else:
vllm/utils.py
+22
@@ -17,6 +17,7 @@ import uuid
17
import warnings
18
import weakref
19
from asyncio import FIRST_COMPLETED, ensure_future
20
+from collections.abc import Mapping
21
from functools import lru_cache, partial, wraps
22
from platform import uname
23
from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
@@ -1442,3 +1443,24 @@ class AtomicCounter:
1443
@property
1444
def value(self):
1445
return self._value
1446
+
1447
+
1448
+# Adapted from: https://stackoverflow.com/a/47212782/5082708
1449
+class LazyDict(Mapping, Generic[T]):
1450
+
1451
+ def __init__(self, factory: Dict[str, Callable[[], T]]):
1452
+ self._factory = factory
1453
+ self._dict: Dict[str, T] = {}
1454
+
1455
+ def __getitem__(self, key) -> T:
1456
+ if key not in self._dict:
1457
+ if key not in self._factory:
1458
+ raise KeyError(key)
1459
+ self._dict[key] = self._factory[key]()
1460
+ return self._dict[key]
1461
+
1462
+ def __iter__(self):
1463
+ return iter(self._factory)
1464
+
1465
+ def __len__(self):
1466
+ return len(self._factory)