@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f41fbe5a3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[torch.compile] Fine-grained CustomOp enabling mechanism (#9300)

Luka Govedič committed Oct 17, 2024 at 14:36 UTC 0f41fbe5a370c0b87bb9a038be592c9272d46364
8 files changed +220 -21
tests/model_executor/test_enabled_custom_ops.py new
+92
@@ -0,0 +1,92 @@
1 +import os
2 +from typing import List
3 +
4 +import pytest
5 +
6 +from vllm.model_executor.custom_op import CustomOp
7 +from vllm.model_executor.layers.activation import (GeluAndMul,
8 + ReLUSquaredActivation,
9 + SiluAndMul)
10 +from vllm.model_executor.layers.layernorm import RMSNorm
11 +
12 +
13 +# Registered subclass for test
14 +@CustomOp.register("relu3")
15 +class Relu3(ReLUSquaredActivation):
16 + pass
17 +
18 +
19 +@pytest.mark.parametrize(
20 + "env, torch_level, ops_enabled, default_on",
21 + [
22 + # Default values based on compile level
23 + ("", 0, [True] * 4, True),
24 + ("", 1, [True] * 4, True),
25 + ("", 2, [True] * 4, True), # All by default
26 + ("", 3, [False] * 4, False),
27 + ("", 4, [False] * 4, False), # None by default
28 + # Explicitly enabling/disabling
29 + #
30 + # Default: all
31 + #
32 + # All but SiluAndMul
33 + ("+rms_norm,-silu_and_mul", 0, [1, 0, 1, 1], True),
34 + # Only ReLU3
35 + ("none,-rms_norm,+relu3", 0, [0, 0, 0, 1], False),
36 + # All but SiluAndMul
37 + ("all,-silu_and_mul", 1, [1, 0, 1, 1], True),
38 + # All but ReLU3 (even if ReLU2 is on)
39 + ("-relu3,relu2", 1, [1, 1, 1, 0], True),
40 + # GeluAndMul and SiluAndMul
41 + ("none,-relu3,+gelu_and_mul,+silu_and_mul", 2, [0, 1, 1, 0], False),
42 + # All but RMSNorm
43 + ("-rms_norm", 2, [0, 1, 1, 1], True),
44 + #
45 + # Default: none
46 + #
47 + # Only ReLU3
48 + ("-silu_and_mul,+relu3", 3, [0, 0, 0, 1], False),
49 + # All but RMSNorm
50 + ("all,-rms_norm", 4, [0, 1, 1, 1], True),
51 + ])
52 +def test_enabled_ops(env: str, torch_level: int, ops_enabled: List[int],
53 + default_on: bool):
54 + os.environ["VLLM_CUSTOM_OPS"] = env
55 + os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(torch_level)
56 +
57 + # Reset default_on (computed once):
58 + CustomOp.default_on.cache_clear()
59 +
60 + assert CustomOp.default_on() == default_on
61 +
62 + ops_enabled = [bool(x) for x in ops_enabled]
63 +
64 + assert RMSNorm(1024).enabled() == ops_enabled[0]
65 + assert CustomOp.op_registry["rms_norm"].enabled() == ops_enabled[0]
66 +
67 + assert SiluAndMul().enabled() == ops_enabled[1]
68 + assert CustomOp.op_registry["silu_and_mul"].enabled() == ops_enabled[1]
69 +
70 + assert GeluAndMul().enabled() == ops_enabled[2]
71 + assert CustomOp.op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
72 +
73 + # If registered, subclasses should follow their own name
74 + assert Relu3().enabled() == ops_enabled[3]
75 + assert CustomOp.op_registry["relu3"].enabled() == ops_enabled[3]
76 +
77 + # Unregistered subclass
78 + class SiluAndMul2(SiluAndMul):
79 + pass
80 +
81 + # Subclasses should not require registration
82 + assert SiluAndMul2().enabled() == SiluAndMul().enabled()
83 +
84 +
85 +@pytest.mark.parametrize(
86 + "env", ["all,none", "all,+rms_norm,all", "+rms_norm,-rms_norm"])
87 +def test_enabled_ops_invalid(env: str):
88 + os.environ["VLLM_CUSTOM_OPS"] = env
89 + CustomOp.default_on.cache_clear()
90 +
91 + with pytest.raises(AssertionError):
92 + RMSNorm(1024).enabled()
vllm/envs.py
+12 -1
@@ -65,6 +65,7 @@ if TYPE_CHECKING:
65 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66 VLLM_SKIP_P2P_CHECK: bool = False
67 VLLM_TORCH_COMPILE_LEVEL: int = 0
68 + VLLM_CUSTOM_OPS: List[str] = []
69 VLLM_DISABLED_KERNELS: List[str] = []
70
71
@@ -205,7 +206,17 @@ environment_variables: Dict[str, Callable[[], Any]] = {
206 os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
207 "VLLM_TORCH_COMPILE_LEVEL":
208 lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")),
208 -
209 + # Fine-grained control over which custom ops to enable/disable.
210 + # Use 'all' to enable all, 'none' to disable all.
211 + # Also specify a list of custom op names to enable (prefixed with a '+'),
212 + # or disable (prefixed with a '-').
213 + # Examples:
214 + # - 'all,-op1' to enable all except op1
215 + # - 'none,+op1,+op2' to enable only op1 and op2
216 + # By default, all custom ops are enabled when running without Inductor
217 + # and disabled when running with Inductor (compile_level >= Inductor).
218 + "VLLM_CUSTOM_OPS":
219 + lambda: os.environ.get("VLLM_CUSTOM_OPS", "").replace(" ", "").split(","),
220 # local rank of the process in the distributed setting, used to determine
221 # the GPU device id
222 "LOCAL_RANK":
vllm/model_executor/custom_op.py
+64 -4
@@ -1,14 +1,24 @@
1 +from functools import lru_cache
2 +from typing import Dict, Type
3 +
4 import torch.nn as nn
5
6 import vllm.envs as envs
7 from vllm.compilation.levels import CompilationLevel
8 +from vllm.logger import init_logger
9 from vllm.platforms import current_platform
6 -from vllm.utils import is_cpu, is_hip, is_xpu
10 +from vllm.utils import is_cpu, is_hip, is_xpu, print_warning_once
11 +
12 +logger = init_logger(__name__)
13
14
15 class CustomOp(nn.Module):
16 + """
17 + Base class for custom ops.
18 + Dispatches the forward method to the appropriate backend.
19 + """
20
11 - def __init__(self, *args, **kwargs):
21 + def __init__(self):
22 super().__init__()
23 self._forward_method = self.dispatch_forward()
24
@@ -17,7 +27,6 @@ class CustomOp(nn.Module):
27
28 def forward_native(self, *args, **kwargs):
29 """PyTorch-native implementation of the forward method.
20 -
30 This method is optional. If implemented, it can be used with compilers
31 such as torch.compile or PyTorch XLA. Also, it can be used for testing
32 purposes.
@@ -56,7 +65,11 @@ class CustomOp(nn.Module):
65 # NOTE(woosuk): Here we assume that vLLM was built for only one
66 # specific backend. Currently, we do not support dynamic dispatching.
67
59 - if envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.INDUCTOR:
68 + enabled = self.enabled()
69 + logger.debug("custom op %s %s", self.__class__.name,
70 + "enabled" if enabled else "disabled")
71 +
72 + if not enabled:
73 return self.forward_native
74
75 if is_hip():
@@ -69,3 +82,50 @@ class CustomOp(nn.Module):
82 return self.forward_xpu
83 else:
84 return self.forward_cuda
85 +
86 + @classmethod
87 + def enabled(cls) -> bool:
88 + # if no name, then it was not registered
89 + if not hasattr(cls, "name"):
90 + print_warning_once(
91 + f"Custom op {cls.__name__} was not registered, "
92 + f"which means it won't appear in the op registry. "
93 + f"It will be enabled/disabled based on the global settings.")
94 + return CustomOp.default_on()
95 +
96 + enabled = f"+{cls.name}" in envs.VLLM_CUSTOM_OPS
97 + disabled = f"-{cls.name}" in envs.VLLM_CUSTOM_OPS
98 + assert not (enabled
99 + and disabled), f"Cannot enable and disable {cls.name}"
100 +
101 + return (CustomOp.default_on() or enabled) and not disabled
102 +
103 + # On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR
104 + # Specifying 'all' or 'none' in VLLM_CUSTOM_OPS takes precedence.
105 + @staticmethod
106 + @lru_cache()
107 + def default_on() -> bool:
108 + count_none = envs.VLLM_CUSTOM_OPS.count("none")
109 + count_all = envs.VLLM_CUSTOM_OPS.count("all")
110 + assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
111 + return envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR and \
112 + not count_none > 0 or count_all > 0
113 +
114 + # Dictionary of all custom ops (classes, indexed by registered name).
115 + # To check if an op with a name is enabled, call .enabled() on the class.
116 + # Examples:
117 + # - MyOp.enabled()
118 + # - op_registry["my_op"].enabled()
119 + op_registry: Dict[str, Type['CustomOp']] = {}
120 +
121 + # Decorator to register custom ops.
122 + @classmethod
123 + def register(cls, name: str):
124 +
125 + def decorator(op_cls):
126 + assert name not in cls.op_registry, f"Duplicate op name: {name}"
127 + op_cls.name = name
128 + cls.op_registry[name] = op_cls
129 + return op_cls
130 +
131 + return decorator
vllm/model_executor/layers/activation.py
+25 -11
@@ -11,11 +11,13 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
11 from vllm.model_executor.custom_op import CustomOp
12 from vllm.model_executor.layers.quantization import QuantizationConfig
13 from vllm.model_executor.utils import set_weight_attrs
14 +from vllm.utils import LazyDict
15
16
17 +@CustomOp.register("fatrelu_and_mul")
18 class FatreluAndMul(CustomOp):
19 """An activation function for FATReLU.
18 -
20 +
21 The function computes x -> FATReLU(x[:d]) * x[d:] where
22 d = x.shape[-1] // 2.
23 This is used in openbmb/MiniCPM-S-1B-sft.
@@ -40,6 +42,7 @@ class FatreluAndMul(CustomOp):
42 return self.forward_native(x)
43
44
45 +@CustomOp.register("silu_and_mul")
46 class SiluAndMul(CustomOp):
47 """An activation function for SwiGLU.
48
@@ -74,6 +77,7 @@ class SiluAndMul(CustomOp):
77 return out
78
79
80 +@CustomOp.register("gelu_and_mul")
81 class GeluAndMul(CustomOp):
82 """An activation function for GeGLU.
83
@@ -123,6 +127,7 @@ class GeluAndMul(CustomOp):
127 return f'approximate={repr(self.approximate)}'
128
129
130 +@CustomOp.register("gelu_new")
131 class NewGELU(CustomOp):
132
133 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
@@ -144,6 +149,7 @@ class NewGELU(CustomOp):
149 return ops.gelu_new(x)
150
151
152 +@CustomOp.register("gelu_fast")
153 class FastGELU(CustomOp):
154
155 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
@@ -164,8 +170,8 @@ class FastGELU(CustomOp):
170 return ops.gelu_fast(x)
171
172
173 +@CustomOp.register("quick_gelu")
174 class QuickGELU(CustomOp):
168 -
175 # https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90
176 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
177 """PyTorch-native implementation equivalent to forward()."""
@@ -189,6 +195,7 @@ class QuickGELU(CustomOp):
195 # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
196
197
198 +@CustomOp.register("relu2")
199 class ReLUSquaredActivation(CustomOp):
200 """
201 Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2
@@ -244,15 +251,22 @@ class ScaledActivation(nn.Module):
251 param_data.copy_(loaded_weight)
252
253
247 -_ACTIVATION_REGISTRY = {
248 - "gelu": nn.GELU(),
249 - "gelu_fast": FastGELU(),
250 - "gelu_new": NewGELU(),
251 - "gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
252 - "relu": nn.ReLU(),
253 - "relu2": ReLUSquaredActivation(),
254 - "quick_gelu": QuickGELU(),
255 -}
254 +_ACTIVATION_REGISTRY = LazyDict({
255 + "gelu":
256 + lambda: nn.GELU(),
257 + "gelu_fast":
258 + lambda: FastGELU(),
259 + "gelu_new":
260 + lambda: NewGELU(),
261 + "gelu_pytorch_tanh":
262 + lambda: nn.GELU(approximate="tanh"),
263 + "relu":
264 + lambda: nn.ReLU(),
265 + "relu2":
266 + lambda: ReLUSquaredActivation(),
267 + "quick_gelu":
268 + lambda: QuickGELU(),
269 +})
270
271
272 def get_act_fn(
vllm/model_executor/layers/fused_moe/layer.py
+1 -4
@@ -37,13 +37,13 @@ class FusedMoEMethodBase(QuantizeMethodBase):
37 raise NotImplementedError
38
39
40 +@CustomOp.register("unquantized_fused_moe")
41 class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
42 """MoE method without quantization."""
43
44 def create_weights(self, layer: torch.nn.Module, num_experts: int,
45 hidden_size: int, intermediate_size: int,
46 params_dtype: torch.dtype, **extra_weight_attrs):
46 -
47 # Fused gate_up_proj (column parallel)
48 w13_weight = torch.nn.Parameter(torch.empty(num_experts,
49 2 * intermediate_size,
@@ -74,7 +74,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
74 num_expert_group: Optional[int] = None,
75 custom_routing_function: Optional[Callable] = None
76 ) -> torch.Tensor:
77 -
77 return self.forward(x=x,
78 layer=layer,
79 router_logits=router_logits,
@@ -97,7 +96,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
96 num_expert_group: Optional[int] = None,
97 custom_routing_function: Optional[Callable] = None
98 ) -> torch.Tensor:
100 -
99 from vllm.model_executor.layers.fused_moe.fused_moe import (
100 fused_experts)
101
@@ -134,7 +132,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
132 num_expert_group: Optional[int] = None,
133 custom_routing_function: Optional[Callable] = None
134 ) -> torch.Tensor:
137 -
135 from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe
136 assert not use_grouped_topk
137 assert num_expert_group is None
vllm/model_executor/layers/layernorm.py
+2
@@ -7,6 +7,7 @@ import torch.nn as nn
7 from vllm.model_executor.custom_op import CustomOp
8
9
10 +@CustomOp.register("rms_norm")
11 class RMSNorm(CustomOp):
12 """Root mean square normalization.
13
@@ -122,6 +123,7 @@ class RMSNorm(CustomOp):
123 return s
124
125
126 +@CustomOp.register("gemma_rms_norm")
127 class GemmaRMSNorm(CustomOp):
128 """RMS normalization for Gemma.
129
vllm/model_executor/layers/rotary_embedding.py
+2 -1
@@ -72,6 +72,7 @@ def _apply_rotary_emb(
72 return torch.stack((o1, o2), dim=-1).flatten(-2)
73
74
75 +@CustomOp.register("rotary_embedding")
76 class RotaryEmbedding(CustomOp):
77 """Original rotary positional embedding."""
78
@@ -468,7 +469,7 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
469 self.long_factor = long_factor
470
471 scale = self.max_position_embeddings / \
471 - self.original_max_position_embeddings
472 + self.original_max_position_embeddings
473 if scale <= 1.0:
474 scaling_factor = 1.0
475 else:
vllm/utils.py
+22
@@ -17,6 +17,7 @@ import uuid
17 import warnings
18 import weakref
19 from asyncio import FIRST_COMPLETED, ensure_future
20 +from collections.abc import Mapping
21 from functools import lru_cache, partial, wraps
22 from platform import uname
23 from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
@@ -1442,3 +1443,24 @@ class AtomicCounter:
1443 @property
1444 def value(self):
1445 return self._value
1446 +
1447 +
1448 +# Adapted from: https://stackoverflow.com/a/47212782/5082708
1449 +class LazyDict(Mapping, Generic[T]):
1450 +
1451 + def __init__(self, factory: Dict[str, Callable[[], T]]):
1452 + self._factory = factory
1453 + self._dict: Dict[str, T] = {}
1454 +
1455 + def __getitem__(self, key) -> T:
1456 + if key not in self._dict:
1457 + if key not in self._factory:
1458 + raise KeyError(key)
1459 + self._dict[key] = self._factory[key]()
1460 + return self._dict[key]
1461 +
1462 + def __iter__(self):
1463 + return iter(self._factory)
1464 +
1465 + def __len__(self):
1466 + return len(self._factory)