| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import functools |
| 4 | import inspect |
| 5 | |
| 6 | import torch |
| 7 | import torch.nn as nn |
| 8 | |
| 9 | from vllm.config import get_cached_compilation_config |
| 10 | from vllm.logger import init_logger |
| 11 | from vllm.model_executor.utils import maybe_disable_graph_partition |
| 12 | from vllm.platforms import current_platform |
| 13 | |
| 14 | logger = init_logger(__name__) |
| 15 | |
| 16 | # Dictionary of all custom ops (classes, indexed by registered name). |
| 17 | # To check if an op with a name is enabled, call .enabled() on the class. |
| 18 | # Examples: |
| 19 | # - MyOp.enabled() |
| 20 | # - op_registry["my_op"].enabled() |
| 21 | op_registry: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {} |
| 22 | op_registry_oot: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {} |
| 23 | |
| 24 | |
| 25 | def maybe_get_oot_by_class(class_type: type) -> type: |
| 26 | class_name = class_type.__name__ |
| 27 | if class_name in op_registry_oot: |
| 28 | return op_registry_oot[class_name] |
| 29 | return class_type |
| 30 | |
| 31 | |
| 32 | class PluggableLayer(nn.Module): |
| 33 | """Base class for pluggable layers. |
| 34 | |
| 35 | A PluggableLayer is a *module-composing* abstraction: it may instantiate other |
| 36 | ``torch.nn.Module`` objects as sub-layers, and its functionality depends on |
| 37 | these sub-layers following a generalized invocation sequence. Also, it is stateful |
| 38 | and may hold parameters or buffers. |
| 39 | |
| 40 | Unlike :class:`CustomOp`, PluggableLayer does NOT provide per-platform |
| 41 | ``forward_*`` dispatch. Instead, it supports out-of-tree (OOT) replacement |
| 42 | of the entire layer class at instantiation time, allowing customized |
| 43 | initialization and submodule composition. |
| 44 | """ |
| 45 | |
| 46 | def __new__(cls, *args, **kwargs): |
| 47 | try: |
| 48 | layer_class_name = cls.__name__ |
| 49 | except AttributeError: |
| 50 | raise TypeError( |
| 51 | f"Cannot instantiate '{cls.__name__}': its 'name' attribute " |
| 52 | f"was not set, possibly because it was not decorated with " |
| 53 | f"@PluggableLayer.register, or it's the PluggableLayer itself." |
| 54 | ) from None |
| 55 | |
| 56 | if layer_class_name not in op_registry_oot: |
| 57 | layer_cls_to_instantiate = cls |
| 58 | else: |
| 59 | layer_cls_to_instantiate = op_registry_oot[layer_class_name] |
| 60 | logger.debug( |
| 61 | "Instantiating pluggable layer: %s using %s", |
| 62 | layer_class_name, |
| 63 | str(layer_cls_to_instantiate), |
| 64 | ) |
| 65 | return super().__new__(layer_cls_to_instantiate) |
| 66 | |
| 67 | # Decorator to register pluggable layers. |
| 68 | @classmethod |
| 69 | def register(cls, name: str): |
| 70 | def decorator(op_cls): |
| 71 | assert name not in op_registry, f"Duplicate op name: {name}" |
| 72 | op_cls.name = name |
| 73 | op_registry[name] = op_cls |
| 74 | return op_cls |
| 75 | |
| 76 | return decorator |
| 77 | |
| 78 | # Decorator to register out-of-tree(oot) pluggable layers. |
| 79 | # For OOT pluggable layers: |
| 80 | # if in-tree layer class is registered with an oot_custom_layer, |
| 81 | # the oot_custom_layer will be used instead. |
| 82 | @classmethod |
| 83 | def register_oot(cls, _decorated_layer_cls=None, name: str | None = None): |
| 84 | def decorator(layer_cls): |
| 85 | reg_name = name if name is not None else cls.__name__ |
| 86 | assert reg_name not in op_registry_oot, f"Duplicate layer name: {reg_name}" |
| 87 | layer_cls.name = reg_name |
| 88 | op_registry_oot[reg_name] = layer_cls |
| 89 | return layer_cls |
| 90 | |
| 91 | if _decorated_layer_cls is None: |
| 92 | # Called with parentheses: @PluggableLayer.register_oot() |
| 93 | # or @PluggableLayer.register_oot(name="...") |
| 94 | return decorator |
| 95 | elif isinstance(_decorated_layer_cls, type): # Check if it's a class |
| 96 | # Called without parentheses: @PluggableLayer.register_oot |
| 97 | return decorator(_decorated_layer_cls) |
| 98 | else: |
| 99 | raise TypeError("Decorator can only be applied to classes.") |
| 100 | |
| 101 | |
| 102 | class CustomOp(nn.Module): |
| 103 | """Base class for custom ops. |
| 104 | Dispatches the forward method to the appropriate backend. |
| 105 | """ |
| 106 | |
| 107 | def __new__(cls, *args, **kwargs): |
| 108 | try: |
| 109 | op_name = cls.__name__ |
| 110 | except AttributeError: |
| 111 | raise TypeError( |
| 112 | f"Cannot instantiate '{cls.__name__}': its 'name' attribute " |
| 113 | f"was not set, possibly because it was not decorated with " |
| 114 | f"@CustomOp.register, or it's the CustomOp base class itself." |
| 115 | ) from None |
| 116 | |
| 117 | if op_name not in op_registry_oot: |
| 118 | op_cls_to_instantiate = cls |
| 119 | else: |
| 120 | op_cls_to_instantiate = op_registry_oot[op_name] |
| 121 | logger.debug( |
| 122 | "Instantiating custom op: %s using %s", |
| 123 | op_name, |
| 124 | str(op_cls_to_instantiate), |
| 125 | ) |
| 126 | return super().__new__(op_cls_to_instantiate) |
| 127 | |
| 128 | def __init__(self, *, enforce_enable: bool = False, compile_native: bool = False): |
| 129 | super().__init__() |
| 130 | self._enforce_enable = enforce_enable |
| 131 | self._forward_method = self.dispatch_forward(compile_native=compile_native) |
| 132 | |
| 133 | def forward(self, *args, **kwargs): |
| 134 | return self._forward_method(*args, **kwargs) |
| 135 | |
| 136 | def forward_native(self, *args, **kwargs): |
| 137 | """PyTorch-native implementation of the forward method. |
| 138 | This method is optional. If implemented, it can be used with compilers |
| 139 | such as torch.compile or PyTorch XLA. Also, it can be used for testing |
| 140 | purposes. |
| 141 | """ |
| 142 | raise NotImplementedError |
| 143 | |
| 144 | def forward_cuda(self, *args, **kwargs): |
| 145 | raise NotImplementedError |
| 146 | |
| 147 | def forward_hip(self, *args, **kwargs): |
| 148 | # By default, we assume that HIP ops are compatible with CUDA ops. |
| 149 | return self.forward_cuda(*args, **kwargs) |
| 150 | |
| 151 | def forward_xpu(self, *args, **kwargs): |
| 152 | # By default, we assume that XPU ops are compatible with the |
| 153 | # PyTorch-native implementation. |
| 154 | return self.forward_native(*args, **kwargs) |
| 155 | |
| 156 | def forward_cpu(self, *args, **kwargs): |
| 157 | # By default, we assume that CPU ops are compatible with the |
| 158 | # PyTorch-native implementation. |
| 159 | return self.forward_native(*args, **kwargs) |
| 160 | |
| 161 | def forward_tpu(self, *args, **kwargs): |
| 162 | # By default, we assume that TPU ops are compatible with the |
| 163 | # PyTorch-native implementation. |
| 164 | # NOTE(woosuk): This is a placeholder for future extensions. |
| 165 | return self.forward_native(*args, **kwargs) |
| 166 | |
| 167 | def forward_oot(self, *args, **kwargs): |
| 168 | # By default, we assume that OOT ops are compatible with the |
| 169 | # PyTorch-native implementation. |
| 170 | return self.forward_native(*args, **kwargs) |
| 171 | |
| 172 | def dispatch_forward(self, compile_native: bool): |
| 173 | # NOTE(woosuk): Here we assume that vLLM was built for only one |
| 174 | # specific backend. Currently, we do not support dynamic dispatching. |
| 175 | compilation_config = get_cached_compilation_config() |
| 176 | |
| 177 | # NOTE(shen-shanshan): CustomOp object can be enforce enabled, e.g., |
| 178 | # enable device-specific kernels in ViT models when enabling graph |
| 179 | # mode. By default, it will follow the compilation_config to determine |
| 180 | # whether enable itself. |
| 181 | # This enforce_enable mechanism will be removed after we adding a |
| 182 | # separate compilation_config for multi-modal part. |
| 183 | enabled = self._enforce_enable or self.enabled() |
| 184 | if enabled: |
| 185 | compilation_config.enabled_custom_ops.update([self.__class__.name]) |
| 186 | else: |
| 187 | compilation_config.disabled_custom_ops.update([self.__class__.name]) |
| 188 | |
| 189 | if not enabled: |
| 190 | # Compile forward_native to avoid eager torch ops if inside |
| 191 | # opaque torch custom op (e.g. fused_moe, unified_attention, etc.) |
| 192 | return self.maybe_compile(self.forward_native, enable=compile_native) |
| 193 | |
| 194 | if current_platform.is_rocm(): |
| 195 | return self.forward_hip |
| 196 | elif current_platform.is_cpu(): |
| 197 | return self.forward_cpu |
| 198 | elif current_platform.is_tpu(): |
| 199 | return self.forward_tpu |
| 200 | elif current_platform.is_xpu(): |
| 201 | return self.forward_xpu |
| 202 | elif current_platform.is_out_of_tree(): |
| 203 | return self.forward_oot |
| 204 | else: |
| 205 | return self.forward_cuda |
| 206 | |
| 207 | def maybe_compile(self, fn, *, enable: bool = True): |
| 208 | """Compile fn if compilation enabled. |
| 209 | Useful for CustomOp instances called from within a torch custom op, |
| 210 | meaning the forward call is hidden from the model-level torch.compile. |
| 211 | |
| 212 | NOTE: this does not enable fusion across ops, so opaque custom ops |
| 213 | should still be unwrapped wherever possible. |
| 214 | """ |
| 215 | from vllm.config.compilation import CompilationMode |
| 216 | |
| 217 | # Do not compile if compilation disabled |
| 218 | if not enable: |
| 219 | return fn |
| 220 | |
| 221 | # Do not compile if global compilation disabled |
| 222 | compilation_config = get_cached_compilation_config() |
| 223 | if compilation_config.mode == CompilationMode.NONE: |
| 224 | return fn |
| 225 | |
| 226 | # If eager backend is used, do not compile either |
| 227 | if compilation_config.backend == "eager": |
| 228 | return fn |
| 229 | |
| 230 | compile_options = maybe_disable_graph_partition( |
| 231 | current_platform.simple_compile_backend |
| 232 | ) |
| 233 | backend = current_platform.simple_compile_backend |
| 234 | |
| 235 | dynamic_arg_dims = getattr(self.__class__, "_dynamic_arg_dims", None) |
| 236 | if dynamic_arg_dims is not None: |
| 237 | compiled_fn = torch.compile( |
| 238 | fn, |
| 239 | dynamic=False, |
| 240 | backend=backend, |
| 241 | options=compile_options, |
| 242 | ) |
| 243 | sig = inspect.signature(fn) |
| 244 | |
| 245 | @functools.wraps(fn) |
| 246 | def wrapper(*args, **kwargs): |
| 247 | bound = sig.bind(*args, **kwargs) |
| 248 | bound.apply_defaults() |
| 249 | for name, dims in dynamic_arg_dims.items(): |
| 250 | arg = bound.arguments.get(name) |
| 251 | if arg is not None and isinstance(arg, torch.Tensor): |
| 252 | dims_list = [dims] if isinstance(dims, int) else dims |
| 253 | for d in dims_list: |
| 254 | real_d = arg.ndim + d if d < 0 else d |
| 255 | torch._dynamo.mark_dynamic(arg, real_d) |
| 256 | return compiled_fn(*args, **kwargs) |
| 257 | |
| 258 | return wrapper |
| 259 | |
| 260 | # dynamic=True to avoid recompilations |
| 261 | return torch.compile( |
| 262 | fn, |
| 263 | dynamic=True, |
| 264 | backend=backend, |
| 265 | options=compile_options, |
| 266 | ) |
| 267 | |
| 268 | @classmethod |
| 269 | def enabled(cls) -> bool: |
| 270 | # if no name, then it was not registered |
| 271 | compilation_config = get_cached_compilation_config() |
| 272 | custom_ops = compilation_config.custom_ops |
| 273 | if not hasattr(cls, "name"): |
| 274 | logger.warning_once( |
| 275 | "Custom op %s was not registered, which means it won't appear " |
| 276 | "in the op registry. It will be enabled/disabled based on the " |
| 277 | "global settings.", |
| 278 | cls.__name__, |
| 279 | ) |
| 280 | return CustomOp.default_on() |
| 281 | |
| 282 | enabled = f"+{cls.name}" in custom_ops |
| 283 | disabled = f"-{cls.name}" in custom_ops |
| 284 | if enabled and disabled: |
| 285 | raise ValueError( |
| 286 | "custom_ops cannot both enable and disable the same operation: " |
| 287 | f"{cls.name}. Remove either the '+' or '-' directive" |
| 288 | ) |
| 289 | |
| 290 | return (CustomOp.default_on() or enabled) and not disabled |
| 291 | |
| 292 | @staticmethod |
| 293 | def default_on() -> bool: |
| 294 | """Behavior controlled by `CompilationConfig.custom_ops`: On by default if |
| 295 | 'all', off by default if 'none'. |
| 296 | When PyTorch Inductor is used, 'none' is the default value, |
| 297 | otherwise 'all'. |
| 298 | """ |
| 299 | compilation_config = get_cached_compilation_config() |
| 300 | count_none = compilation_config.custom_ops.count("none") |
| 301 | count_all = compilation_config.custom_ops.count("all") |
| 302 | if count_none + count_all != 1: |
| 303 | raise ValueError( |
| 304 | "custom_ops must contain exactly one base mode: 'all' or 'none'" |
| 305 | ) |
| 306 | |
| 307 | return not count_none > 0 or count_all > 0 |
| 308 | |
| 309 | # Decorator to register custom ops. |
| 310 | @classmethod |
| 311 | def register( |
| 312 | cls, |
| 313 | name: str, |
| 314 | dynamic_arg_dims: dict[str, int | list[int]] | None = None, |
| 315 | ): |
| 316 | def decorator(op_cls): |
| 317 | assert name not in op_registry, f"Duplicate op name: {name}" |
| 318 | op_cls.name = name |
| 319 | op_cls._dynamic_arg_dims = dynamic_arg_dims |
| 320 | op_registry[name] = op_cls |
| 321 | return op_cls |
| 322 | |
| 323 | return decorator |
| 324 | |
| 325 | # Decorator to register out-of-tree(oot) custom ops. |
| 326 | # For OOT custom ops: |
| 327 | # if in-tree layer class is registered with an oot_custom_op layer, |
| 328 | # the oot_custom_op layer will be used instead. |
| 329 | # Example: |
| 330 | # - @UnquantizedFusedMoEMethod.register_oot |
| 331 | # class HPUUnquantizedFusedMoEMethod(UnquantizedFusedMoEMethod) |
| 332 | # or |
| 333 | # - @CustomOP.register_oot(name="UnquantizedFusedMoEMethod") |
| 334 | @classmethod |
| 335 | def register_oot(cls, _decorated_op_cls=None, name: str | None = None): |
| 336 | def decorator(op_cls): |
| 337 | reg_name = name if name is not None else cls.__name__ |
| 338 | assert reg_name not in op_registry_oot, f"Duplicate op name: {reg_name}" |
| 339 | op_cls.name = reg_name |
| 340 | op_registry_oot[reg_name] = op_cls |
| 341 | return op_cls |
| 342 | |
| 343 | if _decorated_op_cls is None: |
| 344 | # Called with parentheses: @CustomOP.register_oot() |
| 345 | # or @CustomOP.register_oot(name="...") |
| 346 | # So, _decorated_op_cls is None. |
| 347 | # We return the actual decorator function. |
| 348 | return decorator |
| 349 | elif isinstance(_decorated_op_cls, type): # Check if it's a class |
| 350 | # Called without parentheses: @CustomOP.register_oot |
| 351 | # The first argument is the class itself. |
| 352 | # We call the 'decorator' function immediately with the class. |
| 353 | return decorator(_decorated_op_cls) |
| 354 | else: |
| 355 | # Handle other unexpected cases if necessary |
| 356 | raise TypeError("Decorator can only be applied to classes.") |