| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 4 | """Contains helpers related to importing modules. |
| 5 | |
| 6 | This is similar in concept to the `importlib` module. |
| 7 | """ |
| 8 | |
| 9 | import importlib.metadata |
| 10 | import importlib.util |
| 11 | import os |
| 12 | import sys |
| 13 | from functools import cache |
| 14 | from types import ModuleType |
| 15 | from typing import Any |
| 16 | |
| 17 | import regex as re |
| 18 | from typing_extensions import Never |
| 19 | |
| 20 | from vllm.logger import init_logger |
| 21 | |
| 22 | logger = init_logger(__name__) |
| 23 | |
| 24 | |
| 25 | def import_pynvml(): |
| 26 | """Historical comments: |
| 27 | |
| 28 | libnvml.so is the library behind nvidia-smi, and |
| 29 | pynvml is a Python wrapper around it. We use it to get GPU |
| 30 | status without initializing CUDA context in the current process. |
| 31 | Historically, there are two packages that provide pynvml: |
| 32 | - `nvidia-ml-py` (https://pypi.org/project/nvidia-ml-py/): The official |
| 33 | wrapper. It is a dependency of vLLM, and is installed when users |
| 34 | install vLLM. It provides a Python module named `pynvml`. |
| 35 | - `pynvml` (https://pypi.org/project/pynvml/): An unofficial wrapper. |
| 36 | Prior to version 12.0, it also provides a Python module `pynvml`, |
| 37 | and therefore conflicts with the official one. What's worse, |
| 38 | the module is a Python package, and has higher priority than |
| 39 | the official one which is a standalone Python file. |
| 40 | This causes errors when both of them are installed. |
| 41 | Starting from version 12.0, it migrates to a new module |
| 42 | named `pynvml_utils` to avoid the conflict. |
| 43 | It is so confusing that many packages in the community use the |
| 44 | unofficial one by mistake, and we have to handle this case. |
| 45 | For example, `nvcr.io/nvidia/pytorch:24.12-py3` uses the unofficial |
| 46 | one, and it will cause errors, see the issue |
| 47 | https://github.com/vllm-project/vllm/issues/12847 for example. |
| 48 | After all the troubles, we decide to copy the official `pynvml` |
| 49 | module to our codebase, and use it directly. |
| 50 | """ |
| 51 | import vllm.third_party.pynvml as pynvml |
| 52 | |
| 53 | return pynvml |
| 54 | |
| 55 | |
| 56 | @cache |
| 57 | def import_triton_kernels(): |
| 58 | """For convenience, prioritize triton_kernels that is available in |
| 59 | `site-packages`. Use `vllm.third_party.triton_kernels` as a fall-back. |
| 60 | """ |
| 61 | if _has_module("triton_kernels"): |
| 62 | import triton_kernels |
| 63 | |
| 64 | logger.debug_once( |
| 65 | f"Loading module triton_kernels from {triton_kernels.__file__}.", |
| 66 | ) |
| 67 | elif _has_module("vllm.third_party.triton_kernels"): |
| 68 | import vllm.third_party.triton_kernels as triton_kernels |
| 69 | |
| 70 | logger.debug_once( |
| 71 | f"Loading module triton_kernels from {triton_kernels.__file__}.", |
| 72 | ) |
| 73 | sys.modules["triton_kernels"] = triton_kernels |
| 74 | else: |
| 75 | logger.info_once( |
| 76 | "triton_kernels unavailable in this build. " |
| 77 | "Please consider installing triton_kernels from " |
| 78 | "https://github.com/triton-lang/triton/tree/main/python/triton_kernels" |
| 79 | ) |
| 80 | |
| 81 | |
| 82 | def import_plugin(plugin_path: str) -> ModuleType | None: |
| 83 | """Import a user-defined plugin. |
| 84 | |
| 85 | Plugin can be either: |
| 86 | * a module in site-packages |
| 87 | * a Python file specified by its path |
| 88 | """ |
| 89 | try: |
| 90 | return importlib.import_module(plugin_path) |
| 91 | except ModuleNotFoundError: |
| 92 | module_name = os.path.splitext(os.path.basename(plugin_path))[0] |
| 93 | try: |
| 94 | return import_from_path(module_name, plugin_path) |
| 95 | except Exception: |
| 96 | logger.exception( |
| 97 | "Failed to load module '%s' from %s.", module_name, plugin_path |
| 98 | ) |
| 99 | return None |
| 100 | |
| 101 | |
| 102 | def import_from_path(module_name: str, file_path: str | os.PathLike) -> ModuleType: |
| 103 | """Import a Python file according to its file path. |
| 104 | |
| 105 | Based on the official recipe: |
| 106 | https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly |
| 107 | """ |
| 108 | spec = importlib.util.spec_from_file_location(module_name, file_path) |
| 109 | if spec is None: |
| 110 | raise ModuleNotFoundError(f"No module named {module_name!r}") |
| 111 | |
| 112 | assert spec.loader is not None |
| 113 | |
| 114 | module = importlib.util.module_from_spec(spec) |
| 115 | sys.modules[module_name] = module |
| 116 | spec.loader.exec_module(module) |
| 117 | return module |
| 118 | |
| 119 | |
| 120 | def resolve_obj_by_qualname(qualname: str) -> Any: |
| 121 | """Resolve an object by its fully-qualified class name.""" |
| 122 | module_name, obj_name = qualname.rsplit(".", 1) |
| 123 | module = importlib.import_module(module_name) |
| 124 | return getattr(module, obj_name) |
| 125 | |
| 126 | |
| 127 | @cache |
| 128 | def get_vllm_optional_dependencies(): |
| 129 | metadata = importlib.metadata.metadata("vllm") |
| 130 | requirements = metadata.get_all("Requires-Dist", []) |
| 131 | extras = metadata.get_all("Provides-Extra", []) |
| 132 | |
| 133 | return { |
| 134 | extra: [ |
| 135 | re.split(r";|>=|<=|==", req)[0] |
| 136 | for req in requirements |
| 137 | if req.endswith(f'extra == "{extra}"') |
| 138 | ] |
| 139 | for extra in extras |
| 140 | } |
| 141 | |
| 142 | |
| 143 | class _PlaceholderBase: |
| 144 | """Disallows downstream usage of placeholder modules. |
| 145 | |
| 146 | We need to explicitly override each dunder method because |
| 147 | [`__getattr__`][vllm.utils.import_utils._PlaceholderBase.__getattr__] |
| 148 | is not called when they are accessed. |
| 149 | |
| 150 | Info: |
| 151 | [Special method lookup](https://docs.python.org/3/reference/datamodel.html#special-lookup) |
| 152 | """ |
| 153 | |
| 154 | def __getattr__(self, key: str) -> Never: |
| 155 | """The main class should implement this to throw an error |
| 156 | for attribute accesses representing downstream usage. |
| 157 | """ |
| 158 | raise NotImplementedError |
| 159 | |
| 160 | # [Basic customization] |
| 161 | |
| 162 | def __lt__(self, other: object): |
| 163 | return self.__getattr__("__lt__") |
| 164 | |
| 165 | def __le__(self, other: object): |
| 166 | return self.__getattr__("__le__") |
| 167 | |
| 168 | def __eq__(self, other: object): |
| 169 | return self.__getattr__("__eq__") |
| 170 | |
| 171 | def __ne__(self, other: object): |
| 172 | return self.__getattr__("__ne__") |
| 173 | |
| 174 | def __gt__(self, other: object): |
| 175 | return self.__getattr__("__gt__") |
| 176 | |
| 177 | def __ge__(self, other: object): |
| 178 | return self.__getattr__("__ge__") |
| 179 | |
| 180 | def __hash__(self): |
| 181 | return self.__getattr__("__hash__") |
| 182 | |
| 183 | def __bool__(self): |
| 184 | return self.__getattr__("__bool__") |
| 185 | |
| 186 | # [Callable objects] |
| 187 | |
| 188 | def __call__(self, *args: object, **kwargs: object): |
| 189 | return self.__getattr__("__call__") |
| 190 | |
| 191 | # [Container types] |
| 192 | |
| 193 | def __len__(self): |
| 194 | return self.__getattr__("__len__") |
| 195 | |
| 196 | def __getitem__(self, key: object): |
| 197 | return self.__getattr__("__getitem__") |
| 198 | |
| 199 | def __setitem__(self, key: object, value: object): |
| 200 | return self.__getattr__("__setitem__") |
| 201 | |
| 202 | def __delitem__(self, key: object): |
| 203 | return self.__getattr__("__delitem__") |
| 204 | |
| 205 | # __missing__ is optional according to __getitem__ specification, |
| 206 | # so it is skipped |
| 207 | |
| 208 | # __iter__ and __reversed__ have a default implementation |
| 209 | # based on __len__ and __getitem__, so they are skipped. |
| 210 | |
| 211 | # [Numeric Types] |
| 212 | |
| 213 | def __add__(self, other: object): |
| 214 | return self.__getattr__("__add__") |
| 215 | |
| 216 | def __sub__(self, other: object): |
| 217 | return self.__getattr__("__sub__") |
| 218 | |
| 219 | def __mul__(self, other: object): |
| 220 | return self.__getattr__("__mul__") |
| 221 | |
| 222 | def __matmul__(self, other: object): |
| 223 | return self.__getattr__("__matmul__") |
| 224 | |
| 225 | def __truediv__(self, other: object): |
| 226 | return self.__getattr__("__truediv__") |
| 227 | |
| 228 | def __floordiv__(self, other: object): |
| 229 | return self.__getattr__("__floordiv__") |
| 230 | |
| 231 | def __mod__(self, other: object): |
| 232 | return self.__getattr__("__mod__") |
| 233 | |
| 234 | def __divmod__(self, other: object): |
| 235 | return self.__getattr__("__divmod__") |
| 236 | |
| 237 | def __pow__(self, other: object, modulo: object = ...): |
| 238 | return self.__getattr__("__pow__") |
| 239 | |
| 240 | def __lshift__(self, other: object): |
| 241 | return self.__getattr__("__lshift__") |
| 242 | |
| 243 | def __rshift__(self, other: object): |
| 244 | return self.__getattr__("__rshift__") |
| 245 | |
| 246 | def __and__(self, other: object): |
| 247 | return self.__getattr__("__and__") |
| 248 | |
| 249 | def __xor__(self, other: object): |
| 250 | return self.__getattr__("__xor__") |
| 251 | |
| 252 | def __or__(self, other: object): |
| 253 | return self.__getattr__("__or__") |
| 254 | |
| 255 | # r* and i* methods have lower priority than |
| 256 | # the methods for left operand so they are skipped |
| 257 | |
| 258 | def __neg__(self): |
| 259 | return self.__getattr__("__neg__") |
| 260 | |
| 261 | def __pos__(self): |
| 262 | return self.__getattr__("__pos__") |
| 263 | |
| 264 | def __abs__(self): |
| 265 | return self.__getattr__("__abs__") |
| 266 | |
| 267 | def __invert__(self): |
| 268 | return self.__getattr__("__invert__") |
| 269 | |
| 270 | # __complex__, __int__ and __float__ have a default implementation |
| 271 | # based on __index__, so they are skipped. |
| 272 | |
| 273 | def __index__(self): |
| 274 | return self.__getattr__("__index__") |
| 275 | |
| 276 | def __round__(self, ndigits: object = ...): |
| 277 | return self.__getattr__("__round__") |
| 278 | |
| 279 | def __trunc__(self): |
| 280 | return self.__getattr__("__trunc__") |
| 281 | |
| 282 | def __floor__(self): |
| 283 | return self.__getattr__("__floor__") |
| 284 | |
| 285 | def __ceil__(self): |
| 286 | return self.__getattr__("__ceil__") |
| 287 | |
| 288 | # [Context managers] |
| 289 | |
| 290 | def __enter__(self): |
| 291 | return self.__getattr__("__enter__") |
| 292 | |
| 293 | def __exit__(self, *args: object, **kwargs: object): |
| 294 | return self.__getattr__("__exit__") |
| 295 | |
| 296 | |
| 297 | class PlaceholderModule(_PlaceholderBase): |
| 298 | """A placeholder object to use when a module does not exist. |
| 299 | |
| 300 | This enables more informative errors when trying to access attributes |
| 301 | of a module that does not exist. |
| 302 | """ |
| 303 | |
| 304 | def __init__(self, name: str) -> None: |
| 305 | super().__init__() |
| 306 | |
| 307 | # Apply name mangling to avoid conflicting with module attributes |
| 308 | self.__name = name |
| 309 | |
| 310 | def placeholder_attr(self, attr_path: str): |
| 311 | return _PlaceholderModuleAttr(self, attr_path) |
| 312 | |
| 313 | def __getattr__(self, key: str) -> Never: |
| 314 | name = self.__name |
| 315 | |
| 316 | try: |
| 317 | importlib.import_module(name) |
| 318 | except ImportError as exc: |
| 319 | for extra, names in get_vllm_optional_dependencies().items(): |
| 320 | if name in names: |
| 321 | msg = f"Please install vllm[{extra}] for {extra} support" |
| 322 | raise ImportError(msg) from exc |
| 323 | |
| 324 | raise exc |
| 325 | |
| 326 | raise AssertionError( |
| 327 | "PlaceholderModule should not be used " |
| 328 | "when the original module can be imported" |
| 329 | ) |
| 330 | |
| 331 | |
| 332 | class _PlaceholderModuleAttr(_PlaceholderBase): |
| 333 | def __init__(self, module: PlaceholderModule, attr_path: str) -> None: |
| 334 | super().__init__() |
| 335 | |
| 336 | # Apply name mangling to avoid conflicting with module attributes |
| 337 | self.__module = module |
| 338 | self.__attr_path = attr_path |
| 339 | |
| 340 | def placeholder_attr(self, attr_path: str): |
| 341 | return _PlaceholderModuleAttr(self.__module, f"{self.__attr_path}.{attr_path}") |
| 342 | |
| 343 | def __getattr__(self, key: str) -> Never: |
| 344 | getattr(self.__module, f"{self.__attr_path}.{key}") |
| 345 | |
| 346 | raise AssertionError( |
| 347 | "PlaceholderModule should not be used " |
| 348 | "when the original module can be imported" |
| 349 | ) |
| 350 | |
| 351 | |
| 352 | class LazyLoader(ModuleType): |
| 353 | """`LazyLoader` module borrowed from [Tensorflow] |
| 354 | (https://github.com/tensorflow/tensorflow/blob/main/tensorflow/python/util/lazy_loader.py) |
| 355 | with an addition of "module caching". |
| 356 | |
| 357 | Lazily import a module, mainly to avoid pulling in large dependencies. |
| 358 | Modules such as `xgrammar` might do additional side effects, so we |
| 359 | only want to use this when it is needed, delaying all eager effects. |
| 360 | """ |
| 361 | |
| 362 | def __init__( |
| 363 | self, |
| 364 | local_name: str, |
| 365 | parent_module_globals: dict[str, Any], |
| 366 | name: str, |
| 367 | ): |
| 368 | self._local_name = local_name |
| 369 | self._parent_module_globals = parent_module_globals |
| 370 | self._module: ModuleType | None = None |
| 371 | |
| 372 | super().__init__(str(name)) |
| 373 | |
| 374 | def _load(self) -> ModuleType: |
| 375 | # Import the target module and insert it into the parent's namespace |
| 376 | try: |
| 377 | module = importlib.import_module(self.__name__) |
| 378 | self._parent_module_globals[self._local_name] = module |
| 379 | # The additional add to sys.modules |
| 380 | # ensures library is actually loaded. |
| 381 | sys.modules[self._local_name] = module |
| 382 | except ModuleNotFoundError as err: |
| 383 | raise err from None |
| 384 | |
| 385 | # Update this object's dict so that if someone keeps a |
| 386 | # reference to the LazyLoader, lookups are efficient |
| 387 | # (__getattr__ is only called on lookups that fail). |
| 388 | self.__dict__.update(module.__dict__) |
| 389 | return module |
| 390 | |
| 391 | def __getattr__(self, item: Any) -> Any: |
| 392 | if self._module is None: |
| 393 | self._module = self._load() |
| 394 | return getattr(self._module, item) |
| 395 | |
| 396 | def __dir__(self) -> list[str]: |
| 397 | if self._module is None: |
| 398 | self._module = self._load() |
| 399 | return dir(self._module) |
| 400 | |
| 401 | |
| 402 | # Optional dependency detection utilities |
| 403 | @cache |
| 404 | def _has_module(module_name: str) -> bool: |
| 405 | """Return True if *module_name* can be imported in the current environment. |
| 406 | |
| 407 | Uses ``importlib.util.find_spec`` as a fast pre-check, then performs a |
| 408 | trial import to verify that native dependencies (shared libraries, etc.) |
| 409 | are also satisfied. Any failure during the trial import is treated as the |
| 410 | module being unavailable. The result is cached so that subsequent queries |
| 411 | for the same module incur no additional overhead. |
| 412 | """ |
| 413 | try: |
| 414 | if importlib.util.find_spec(module_name) is None: |
| 415 | return False |
| 416 | importlib.import_module(module_name) |
| 417 | except Exception: |
| 418 | logger.warning( |
| 419 | "Module %s was found but failed to import", module_name, exc_info=True |
| 420 | ) |
| 421 | return False |
| 422 | return True |
| 423 | |
| 424 | |
| 425 | @cache |
| 426 | def _has_module_spec(module_name: str) -> bool: |
| 427 | """Return True if *module_name* is installed, without importing it. |
| 428 | |
| 429 | Unlike [`_has_module`][vllm.utils.import_utils._has_module], this only |
| 430 | resolves the import spec. It therefore does not pay the import cost of |
| 431 | heavyweight modules, at the price of not verifying that native |
| 432 | dependencies (shared libraries, etc.) are satisfied. The result is cached. |
| 433 | """ |
| 434 | try: |
| 435 | return importlib.util.find_spec(module_name) is not None |
| 436 | except Exception: |
| 437 | return False |
| 438 | |
| 439 | |
| 440 | def has_deep_ep() -> bool: |
| 441 | """Whether the optional `deep_ep` package is available.""" |
| 442 | return _has_module("deep_ep") |
| 443 | |
| 444 | |
| 445 | DEEPEP_V2_MIN_NCCL_VERSION_RAW = 23004 # 2.30.4 |
| 446 | |
| 447 | |
| 448 | def _get_runtime_nccl_version() -> int | None: |
| 449 | """Get the runtime NCCL version by loading the actual library. |
| 450 | |
| 451 | Returns the raw version int (e.g. 23004 for 2.30.4), or None on failure. |
| 452 | torch.cuda.nccl.version() is a compile-time constant from the PyTorch |
| 453 | wheel and does not reflect a separately installed NCCL. |
| 454 | """ |
| 455 | import ctypes |
| 456 | |
| 457 | try: |
| 458 | from vllm.utils.nccl import find_nccl_library |
| 459 | |
| 460 | lib = ctypes.CDLL(find_nccl_library()) |
| 461 | version = ctypes.c_int() |
| 462 | lib.ncclGetVersion(ctypes.byref(version)) |
| 463 | return version.value |
| 464 | except Exception: |
| 465 | return None |
| 466 | |
| 467 | |
| 468 | def _format_nccl_raw_version(raw: int) -> str: |
| 469 | s = str(raw) |
| 470 | return f"{s[0]}.{s[1:3].lstrip('0') or '0'}.{s[3:].lstrip('0') or '0'}" |
| 471 | |
| 472 | |
| 473 | def has_deep_ep_v2() -> bool: |
| 474 | """Whether deep_ep with ElasticBuffer (v2 API) is available. |
| 475 | |
| 476 | Requires both the ElasticBuffer class in the deep_ep module and |
| 477 | NCCL >= 2.30.4 (GIN backend), checked against the runtime library. |
| 478 | """ |
| 479 | if not _has_module("deep_ep"): |
| 480 | return False |
| 481 | import deep_ep # type: ignore[import-not-found] |
| 482 | |
| 483 | if not hasattr(deep_ep, "ElasticBuffer"): |
| 484 | return False |
| 485 | try: |
| 486 | nccl_ver = _get_runtime_nccl_version() |
| 487 | if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW: |
| 488 | logger.info_once( |
| 489 | "DeepEP v2 requires NCCL >= %s but found %s. " |
| 490 | "deepep_v2 backend will not be available.", |
| 491 | _format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW), |
| 492 | _format_nccl_raw_version(nccl_ver) if nccl_ver else "unknown", |
| 493 | ) |
| 494 | return False |
| 495 | except Exception: |
| 496 | return False |
| 497 | return True |
| 498 | |
| 499 | |
| 500 | def has_deep_gemm() -> bool: |
| 501 | """Whether the optional `deep_gemm` package is available. |
| 502 | |
| 503 | Prefers an externally installed ``deep_gemm`` package (so users can |
| 504 | override with a newer version), then falls back to the vendored copy |
| 505 | bundled in the vLLM wheel. |
| 506 | """ |
| 507 | return _has_module("deep_gemm") or _has_module("vllm.third_party.deep_gemm") |
| 508 | |
| 509 | |
| 510 | def has_nixl_ep() -> bool: |
| 511 | """Whether the optional `nixl_ep` package is available.""" |
| 512 | return _has_module("nixl_ep") |
| 513 | |
| 514 | |
| 515 | def has_moonep() -> bool: |
| 516 | """Whether the optional `moonep` package is available.""" |
| 517 | return _has_module("moonep") |
| 518 | |
| 519 | |
| 520 | def check_moonep_system_support() -> None: |
| 521 | """Raise if the current device cannot run MoonEP. |
| 522 | |
| 523 | MoonEP's symmetric-memory buffers require CUDA VMM plus NVSwitch multicast |
| 524 | (SHARP) on every EP rank; NVLink-only topologies without NVSwitch (e.g. |
| 525 | 4x H100 NV6) fail deep inside ``moonep.Buffer`` otherwise. |
| 526 | """ |
| 527 | if not has_moonep(): |
| 528 | raise RuntimeError( |
| 529 | "MoonEP not available. Install it from " |
| 530 | "https://github.com/MoonshotAI/MoonEP." |
| 531 | ) |
| 532 | from moonep._C import ( # type: ignore[import-not-found] |
| 533 | nvl_multicast_supported, |
| 534 | ) |
| 535 | |
| 536 | if not nvl_multicast_supported(): |
| 537 | raise RuntimeError( |
| 538 | "MoonEP requires NVSwitch multicast (SHARP) support on the " |
| 539 | "current device; it is not supported on this GPU/topology." |
| 540 | ) |
| 541 | |
| 542 | |
| 543 | def is_numba_available() -> bool: |
| 544 | """Whether the optional `numba` package is available.""" |
| 545 | return _has_module("numba") |
| 546 | |
| 547 | |
| 548 | def has_triton_kernels() -> bool: |
| 549 | """Whether the optional `triton_kernels` package is available.""" |
| 550 | is_available = _has_module("triton_kernels") or _has_module( |
| 551 | "vllm.third_party.triton_kernels" |
| 552 | ) |
| 553 | if is_available: |
| 554 | import_triton_kernels() |
| 555 | return is_available |
| 556 | |
| 557 | |
| 558 | @cache |
| 559 | def get_triton_kernels_version() -> str | None: |
| 560 | """The triton_kernels MoE-API generation ("3.5.1"/"3.6"/"3.8"), or None. |
| 561 | |
| 562 | Inferred by capability since the package exposes no usable version: 3.8 |
| 563 | replaced ``matmul_ogs`` with ``matmul``, and 3.5.1 predates ``SparseMatrix``. |
| 564 | """ |
| 565 | if not has_triton_kernels(): |
| 566 | return None |
| 567 | try: |
| 568 | import triton_kernels.matmul_ogs # noqa: F401 |
| 569 | except ImportError: |
| 570 | return "3.8" |
| 571 | try: |
| 572 | from triton_kernels.tensor import SparseMatrix # noqa: F401 |
| 573 | except ImportError: |
| 574 | return "3.5.1" |
| 575 | return "3.6" |
| 576 | |
| 577 | |
| 578 | @cache |
| 579 | def has_tilelang() -> bool: |
| 580 | """Whether the optional `tilelang` package is available. |
| 581 | |
| 582 | Only the import spec is checked: importing `tilelang` is expensive, so |
| 583 | callers must import it lazily at their point of use rather than relying |
| 584 | on this function to have imported it already. |
| 585 | """ |
| 586 | if not _has_module_spec("tilelang"): |
| 587 | return False |
| 588 | # ROCm-only guard, imported lazily to avoid loading rocm on CUDA. |
| 589 | from vllm.platforms import current_platform |
| 590 | |
| 591 | if current_platform.is_rocm(): |
| 592 | from vllm.platforms.rocm import on_gfx1250 |
| 593 | |
| 594 | # TODO: Re-enable when tilelang supports gfx1250 |
| 595 | if on_gfx1250(): |
| 596 | return False |
| 597 | return True |
| 598 | |
| 599 | |
| 600 | def has_arctic_inference() -> bool: |
| 601 | """Whether the optional `arctic_inference` package is available.""" |
| 602 | return _has_module("arctic_inference") |
| 603 | |
| 604 | |
| 605 | def has_helion() -> bool: |
| 606 | """Whether the optional `helion` package is available. |
| 607 | |
| 608 | Helion is a Python-embedded DSL for writing ML kernels. |
| 609 | See: https://github.com/pytorch/helion |
| 610 | |
| 611 | Usage: |
| 612 | if has_helion(): |
| 613 | import helion |
| 614 | import helion.language as hl |
| 615 | # use helion... |
| 616 | """ |
| 617 | return _has_module("helion") |
| 618 | |
| 619 | |
| 620 | def has_aiter() -> bool: |
| 621 | """Whether the optional `aiter` package is available.""" |
| 622 | return _has_module("aiter") |
| 623 | |
| 624 | |
| 625 | def has_mori() -> bool: |
| 626 | """Whether the optional `mori` package is available.""" |
| 627 | return _has_module("mori") |
| 628 | |
| 629 | |
| 630 | def has_fbgemm_gpu() -> bool: |
| 631 | """Whether the optional `fbgemm_gpu` package is available.""" |
| 632 | return _has_module("fbgemm_gpu") |
| 633 | |
| 634 | |
| 635 | def has_cutedsl() -> bool: |
| 636 | """Whether the optional `cutelass` package is available.""" |
| 637 | return _has_module("cutlass") |
| 638 | |
| 639 | |
| 640 | def has_humming() -> bool: |
| 641 | """Whether the optional `humming` package is available.""" |
| 642 | return _has_module("humming") |
| 643 | |
| 644 | |
| 645 | def has_quark(): |
| 646 | """Whether the optional `quark` package is available.""" |
| 647 | return _has_module("quark") |
| 648 | |
| 649 | |
| 650 | def check_torchcodec_available(): |
| 651 | """Whether the optional `torchcodec` package is available.""" |
| 652 | try: |
| 653 | import torchcodec # noqa: F401 |
| 654 | except RuntimeError as e: |
| 655 | # torchcodec will raise RuntimeError during import instead |
| 656 | # of ImportError when system ffmpeg unavailable, with a |
| 657 | # message that can leak sensitive system information. |
| 658 | # Trim it down to avoid it. |
| 659 | marker = ( |
| 660 | "The following exceptions were raised as we tried to load libtorchcodec:" |
| 661 | ) |
| 662 | message = str(e) |
| 663 | if marker in message: |
| 664 | raise RuntimeError(message.split(marker, 1)[0].rstrip()) from None |
| 665 | raise e |