@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b12e87f942
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[platforms] enable platform plugins (#11602)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 30, 2024 at 20:24 UTC b12e87f942eb7740c17ab546b964bc327afdda37
23 files changed +360 -181
.buildkite/test-pipeline.yaml
+19 -6
@@ -106,14 +106,12 @@ steps:
106 source_file_dependencies:
107 - vllm/
108 commands:
109 - - pip install -e ./plugins/vllm_add_dummy_model
109 - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_generate_multiple_loras.py --ignore=entrypoints/llm/test_guided_generate.py
110 - pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
111 - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
112 - pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
113 - pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process
114 - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py
116 - - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process
115 - pytest -v -s entrypoints/test_chat_utils.py
116 - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
117
@@ -333,8 +331,6 @@ steps:
331 - vllm/
332 - tests/models
333 commands:
336 - - pip install -e ./plugins/vllm_add_dummy_model
337 - - pytest -v -s models/test_oot_registration.py # it needs a clean process
334 - pytest -v -s models/test_registry.py
335 - pytest -v -s models/test_initialization.py
336
@@ -469,11 +465,28 @@ steps:
465 - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m 'distributed(num_gpus=2)'
466 - pytest models/decoder_only/vision_language/test_models.py -v -s -m 'distributed(num_gpus=2)'
467 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
472 - - pip install -e ./plugins/vllm_add_dummy_model
473 - - pytest -v -s distributed/test_distributed_oot.py
468 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
469 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s kv_transfer/disagg_test.py
470
471 +- label: Plugin Tests (2 GPUs) # 40min
472 + working_dir: "/vllm-workspace/tests"
473 + num_gpus: 2
474 + fast_check: true
475 + source_file_dependencies:
476 + - vllm/plugins/
477 + - tests/plugins/
478 + commands:
479 + # begin platform plugin tests, all the code in-between runs on dummy platform
480 + - pip install -e ./plugins/vllm_add_dummy_platform
481 + - pytest -v -s plugins_tests/test_platform_plugins.py
482 + - pip uninstall vllm_add_dummy_platform -y
483 + # end platform plugin tests
484 + # other tests continue here:
485 + - pip install -e ./plugins/vllm_add_dummy_model
486 + - pytest -v -s distributed/test_distributed_oot.py
487 + - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process
488 + - pytest -v -s models/test_oot_registration.py # it needs a clean process
489 +
490 - label: Multi-step Tests (4 GPUs) # 36min
491 working_dir: "/vllm-workspace/tests"
492 num_gpus: 4
docs/source/design/plugin_system.md
+4 -2
@@ -41,9 +41,11 @@ Every plugin has three parts:
41 2. **Plugin name**: The name of the plugin. This is the value in the dictionary of the `entry_points` dictionary. In the example above, the plugin name is `register_dummy_model`. Plugins can be filtered by their names using the `VLLM_PLUGINS` environment variable. To load only a specific plugin, set `VLLM_PLUGINS` to the plugin name.
42 3. **Plugin value**: The fully qualified name of the function to register in the plugin system. In the example above, the plugin value is `vllm_add_dummy_model:register`, which refers to a function named `register` in the `vllm_add_dummy_model` module.
43
44 -## What Can Plugins Do?
44 +## Types of supported plugins
45
46 -Currently, the primary use case for plugins is to register custom, out-of-the-tree models into vLLM. This is done by calling `ModelRegistry.register_model` to register the model. In the future, the plugin system may be extended to support more features, such as swapping in custom implementations for certain classes in vLLM.
46 +- **General plugins** (with group name `vllm.general_plugins`): The primary use case for these plugins is to register custom, out-of-the-tree models into vLLM. This is done by calling `ModelRegistry.register_model` to register the model inside the plugin function.
47 +
48 +- **Platform plugins** (with group name `vllm.platform_plugins`): The primary use case for these plugins is to register custom, out-of-the-tree platforms into vLLM. The plugin function should return `None` when the platform is not supported in the current environment, or the platform class's fully qualified name when the platform is supported.
49
50 ## Guidelines for Writing Plugins
51
tests/conftest.py
+1 -1
@@ -31,7 +31,6 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
31 to_enc_dec_tuple_list, zip_enc_dec_prompts)
32 from vllm.logger import init_logger
33 from vllm.outputs import RequestOutput
34 -from vllm.platforms import current_platform
34 from vllm.sampling_params import BeamSearchParams
35 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
36 identity)
@@ -242,6 +241,7 @@ _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
241 class HfRunner:
242
243 def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
244 + from vllm.platforms import current_platform
245 if x is None or isinstance(x, (bool, )):
246 return x
247
tests/kernels/test_attention_selector.py
+8 -8
@@ -5,7 +5,10 @@ import torch
5
6 from tests.kernels.utils import override_backend_env_variable
7 from vllm.attention.selector import which_attn_to_use
8 -from vllm.platforms import cpu, cuda, openvino, rocm
8 +from vllm.platforms.cpu import CpuPlatform
9 +from vllm.platforms.cuda import CudaPlatform
10 +from vllm.platforms.openvino import OpenVinoPlatform
11 +from vllm.platforms.rocm import RocmPlatform
12 from vllm.utils import STR_FLASH_ATTN_VAL, STR_INVALID_VAL
13
14
@@ -20,26 +23,23 @@ def test_env(name: str, device: str, monkeypatch):
23 override_backend_env_variable(monkeypatch, name)
24
25 if device == "cpu":
23 - with patch("vllm.attention.selector.current_platform",
24 - cpu.CpuPlatform()):
26 + with patch("vllm.attention.selector.current_platform", CpuPlatform()):
27 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
28 False)
29 assert backend.name == "TORCH_SDPA"
30 elif device == "hip":
29 - with patch("vllm.attention.selector.current_platform",
30 - rocm.RocmPlatform()):
31 + with patch("vllm.attention.selector.current_platform", RocmPlatform()):
32 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
33 False)
34 assert backend.name == "ROCM_FLASH"
35 elif device == "openvino":
36 with patch("vllm.attention.selector.current_platform",
36 - openvino.OpenVinoPlatform()):
37 + OpenVinoPlatform()):
38 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
39 False)
40 assert backend.name == "OPENVINO"
41 else:
41 - with patch("vllm.attention.selector.current_platform",
42 - cuda.CudaPlatform()):
42 + with patch("vllm.attention.selector.current_platform", CudaPlatform()):
43 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
44 False)
45 assert backend.name == name
tests/plugins/vllm_add_dummy_platform/setup.py new
+11
@@ -0,0 +1,11 @@
1 +from setuptools import setup
2 +
3 +setup(
4 + name='vllm_add_dummy_platform',
5 + version='0.1',
6 + packages=['vllm_add_dummy_platform'],
7 + entry_points={
8 + 'vllm.platform_plugins': [
9 + "dummy_platform_plugin = vllm_add_dummy_platform:dummy_platform_plugin" # noqa
10 + ]
11 + })
tests/plugins/vllm_add_dummy_platform/vllm_add_dummy_platform/__init__.py new
+5
@@ -0,0 +1,5 @@
1 +from typing import Optional
2 +
3 +
4 +def dummy_platform_plugin() -> Optional[str]:
5 + return "vllm_add_dummy_platform.dummy_platform.DummyPlatform"
tests/plugins/vllm_add_dummy_platform/vllm_add_dummy_platform/dummy_platform.py new
+5
@@ -0,0 +1,5 @@
1 +from vllm.platforms.cuda import CudaPlatform
2 +
3 +
4 +class DummyPlatform(CudaPlatform):
5 + device_name = "DummyDevice"
tests/plugins_tests/test_platform_plugins.py new
+16
@@ -0,0 +1,16 @@
1 +def test_platform_plugins():
2 + # simulate workload by running an example
3 + import runpy
4 + current_file = __file__
5 + import os
6 + example_file = os.path.join(
7 + os.path.dirname(os.path.dirname(os.path.dirname(current_file))),
8 + "examples", "offline_inference.py")
9 + runpy.run_path(example_file)
10 +
11 + # check if the plugin is loaded correctly
12 + from vllm.platforms import _init_trace, current_platform
13 + assert current_platform.device_name == "DummyDevice", (
14 + f"Expected DummyDevice, got {current_platform.device_name}, "
15 + "possibly because current_platform is imported before the plugin"
16 + f" is loaded. The first import:\n{_init_trace}")
vllm/config.py
+12 -3
@@ -22,7 +22,7 @@ from vllm.logger import init_logger
22 from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
23 get_quantization_config)
24 from vllm.model_executor.models import ModelRegistry
25 -from vllm.platforms import current_platform, interface
25 +from vllm.platforms import CpuArchEnum
26 from vllm.tracing import is_otel_available, otel_import_error_traceback
27 from vllm.transformers_utils.config import (
28 ConfigFormat, get_config, get_hf_image_processor_config,
@@ -349,6 +349,7 @@ class ModelConfig:
349 self.is_hybrid = self._init_is_hybrid()
350 self.has_inner_state = self._init_has_inner_state()
351
352 + from vllm.platforms import current_platform
353 if current_platform.is_neuron():
354 self.override_neuron_config = override_neuron_config
355 else:
@@ -589,6 +590,7 @@ class ModelConfig:
590 raise ValueError(
591 f"Unknown quantization method: {self.quantization}. Must "
592 f"be one of {supported_quantization}.")
593 + from vllm.platforms import current_platform
594 current_platform.verify_quantization(self.quantization)
595 if self.quantization not in optimized_quantization_methods:
596 logger.warning(
@@ -644,6 +646,7 @@ class ModelConfig:
646
647 # Reminder: Please update docs/source/usage/compatibility_matrix.md
648 # If the feature combo become valid
649 + from vllm.platforms import current_platform
650 if not current_platform.is_async_output_supported(self.enforce_eager):
651 logger.warning(
652 "Async output processing is not supported on the "
@@ -1012,6 +1015,7 @@ class CacheConfig:
1015 raise ValueError(
1016 "GPU memory utilization must be less than 1.0. Got "
1017 f"{self.gpu_memory_utilization}.")
1018 + from vllm.platforms import current_platform
1019 if (current_platform.is_cuda() and self.block_size is not None
1020 and self.block_size > 32):
1021 raise ValueError("CUDA Paged Attention kernel only supports "
@@ -1279,6 +1283,7 @@ class ParallelConfig:
1283 f"distributed executor backend "
1284 f"'{self.distributed_executor_backend}'.")
1285 ray_only_devices = ["tpu", "hpu"]
1286 + from vllm.platforms import current_platform
1287 if (current_platform.device_type in ray_only_devices
1288 and self.world_size > 1):
1289 if self.distributed_executor_backend is None:
@@ -1327,7 +1332,7 @@ class ParallelConfig:
1332 def _verify_args(self) -> None:
1333 # Lazy import to avoid circular import
1334 from vllm.executor.executor_base import ExecutorBase
1330 -
1335 + from vllm.platforms import current_platform
1336 if self.distributed_executor_backend not in (
1337 "ray", "mp", None) and not (isinstance(
1338 self.distributed_executor_backend, type) and issubclass(
@@ -1528,6 +1533,7 @@ class DeviceConfig:
1533 def __init__(self, device: str = "auto") -> None:
1534 if device == "auto":
1535 # Automated device type detection
1536 + from vllm.platforms import current_platform
1537 self.device_type = current_platform.device_type
1538 if not self.device_type:
1539 raise RuntimeError("Failed to infer device type")
@@ -2241,9 +2247,10 @@ def _get_and_verify_dtype(
2247 else:
2248 torch_dtype = config_dtype
2249
2250 + from vllm.platforms import current_platform
2251 if (current_platform.is_cpu()
2252 and current_platform.get_cpu_architecture()
2246 - == interface.CpuArchEnum.POWERPC
2253 + == CpuArchEnum.POWERPC
2254 and (config_dtype == torch.float16
2255 or config_dtype == torch.float32)):
2256 logger.info(
@@ -3083,6 +3090,7 @@ class VllmConfig:
3090 model_config: ModelConfig,
3091 load_config: LoadConfig) -> Optional[QuantizationConfig]:
3092 """Get the quantization config."""
3093 + from vllm.platforms import current_platform
3094 if model_config.quantization is not None:
3095 from vllm.model_executor.model_loader.weight_utils import (
3096 get_quant_config)
@@ -3145,6 +3153,7 @@ class VllmConfig:
3153 self.quant_config = VllmConfig._get_quantization_config(
3154 self.model_config, self.load_config)
3155
3156 + from vllm.platforms import current_platform
3157 if self.scheduler_config is not None and \
3158 self.model_config is not None and \
3159 self.scheduler_config.chunked_prefill_enabled and \
vllm/distributed/parallel_state.py
+2 -1
@@ -39,7 +39,6 @@ import vllm.distributed.kv_transfer.kv_transfer_agent as kv_transfer
39 import vllm.envs as envs
40 from vllm.distributed.utils import StatelessProcessGroup
41 from vllm.logger import init_logger
42 -from vllm.platforms import current_platform
42 from vllm.utils import direct_register_custom_op, supports_custom_op
43
44 if TYPE_CHECKING:
@@ -194,6 +193,7 @@ class GroupCoordinator:
193 assert self.cpu_group is not None
194 assert self.device_group is not None
195
196 + from vllm.platforms import current_platform
197 if current_platform.is_cuda_alike():
198 self.device = torch.device(f"cuda:{local_rank}")
199 else:
@@ -1188,6 +1188,7 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1188 import ray # Lazy import Ray
1189 ray.shutdown()
1190 gc.collect()
1191 + from vllm.platforms import current_platform
1192 if not current_platform.is_cpu():
1193 torch.cuda.empty_cache()
1194
vllm/engine/arg_utils.py
+1 -1
@@ -18,7 +18,6 @@ from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,
18 from vllm.executor.executor_base import ExecutorBase
19 from vllm.logger import init_logger
20 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
21 -from vllm.platforms import current_platform
21 from vllm.transformers_utils.utils import check_gguf_file
22 from vllm.usage.usage_lib import UsageContext
23 from vllm.utils import FlexibleArgumentParser, StoreBoolean
@@ -1094,6 +1093,7 @@ class EngineArgs:
1093 use_sliding_window = (model_config.get_sliding_window()
1094 is not None)
1095 use_spec_decode = self.speculative_model is not None
1096 + from vllm.platforms import current_platform
1097 if (is_gpu and not use_sliding_window and not use_spec_decode
1098 and not self.enable_lora
1099 and not self.enable_prompt_adapter
vllm/executor/ray_utils.py
+1 -1
@@ -8,7 +8,6 @@ import msgspec
8 from vllm.config import ParallelConfig
9 from vllm.executor.msgspec_utils import decode_hook, encode_hook
10 from vllm.logger import init_logger
11 -from vllm.platforms import current_platform
11 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
12 from vllm.utils import get_ip
13 from vllm.worker.worker_base import WorkerWrapperBase
@@ -229,6 +228,7 @@ def initialize_ray_cluster(
228 the default Ray cluster address.
229 """
230 assert_ray_available()
231 + from vllm.platforms import current_platform
232
233 # Connect to a ray cluster.
234 if current_platform.is_rocm() or current_platform.is_xpu():
vllm/model_executor/guided_decoding/__init__.py
+2 -1
@@ -6,7 +6,7 @@ from vllm.logger import init_logger
6 from vllm.model_executor.guided_decoding.utils import (
7 convert_lark_to_gbnf, grammar_is_likely_lark,
8 has_lmf_unsupported_json_features, has_xgrammar_unsupported_json_features)
9 -from vllm.platforms import CpuArchEnum, current_platform
9 +from vllm.platforms import CpuArchEnum
10
11 if TYPE_CHECKING:
12 from transformers import PreTrainedTokenizer
@@ -39,6 +39,7 @@ def maybe_backend_fallback(
39
40 if guided_params.backend == "xgrammar":
41 # xgrammar only has x86 wheels for linux, fallback to outlines
42 + from vllm.platforms import current_platform
43 if current_platform.get_cpu_architecture() is not CpuArchEnum.X86:
44 logger.warning("xgrammar is only supported on x86 CPUs. "
45 "Falling back to use outlines instead.")
vllm/model_executor/models/registry.py
+1 -1
@@ -18,7 +18,6 @@ import cloudpickle
18 import torch.nn as nn
19
20 from vllm.logger import init_logger
21 -from vllm.platforms import current_platform
21
22 from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
23 supports_cross_encoding, supports_multimodal,
@@ -273,6 +272,7 @@ def _try_load_model_cls(
272 model_arch: str,
273 model: _BaseRegisteredModel,
274 ) -> Optional[Type[nn.Module]]:
275 + from vllm.platforms import current_platform
276 current_platform.verify_model_arch(model_arch)
277 try:
278 return model.load_model_cls()
vllm/model_executor/utils.py
+2 -2
@@ -3,10 +3,9 @@ from typing import Any, Dict, Optional
3
4 import torch
5
6 -from vllm.platforms import current_platform
7 -
6
7 def set_random_seed(seed: int) -> None:
8 + from vllm.platforms import current_platform
9 current_platform.seed_everything(seed)
10
11
@@ -38,6 +37,7 @@ def set_weight_attrs(
37 # This sometimes causes OOM errors during model loading. To avoid this,
38 # we sync the param tensor after its weight loader is called.
39 # TODO(woosuk): Remove this hack once we have a better solution.
40 + from vllm.platforms import current_platform
41 if current_platform.is_tpu() and key == "weight_loader":
42 value = _make_synced_weight_loader(value)
43 setattr(weight, key, value)
vllm/platforms/__init__.py
+210 -110
@@ -1,123 +1,223 @@
1 +import logging
2 +import traceback
3 +from itertools import chain
4 +from typing import TYPE_CHECKING, Optional
5 +
6 +from vllm.plugins import load_plugins_by_group
7 +from vllm.utils import resolve_obj_by_qualname
8 +
9 from .interface import _Backend # noqa: F401
2 -from .interface import CpuArchEnum, Platform, PlatformEnum, UnspecifiedPlatform
10 +from .interface import CpuArchEnum, Platform, PlatformEnum
11
4 -current_platform: Platform
12 +logger = logging.getLogger(__name__)
13
6 -# NOTE: we don't use `torch.version.cuda` / `torch.version.hip` because
7 -# they only indicate the build configuration, not the runtime environment.
8 -# For example, people can install a cuda build of pytorch but run on tpu.
14
10 -is_tpu = False
11 -try:
12 - # While it's technically possible to install libtpu on a non-TPU machine,
13 - # this is a very uncommon scenario. Therefore, we assume that libtpu is
14 - # installed if and only if the machine has TPUs.
15 - import libtpu # noqa: F401
16 - is_tpu = True
17 -except Exception:
18 - pass
15 +def tpu_platform_plugin() -> Optional[str]:
16 + is_tpu = False
17 + try:
18 + # While it's technically possible to install libtpu on a
19 + # non-TPU machine, this is a very uncommon scenario. Therefore,
20 + # we assume that libtpu is installed if and only if the machine
21 + # has TPUs.
22 + import libtpu # noqa: F401
23 + is_tpu = True
24 + except Exception:
25 + pass
26 +
27 + return "vllm.platforms.tpu.TpuPlatform" if is_tpu else None
28
20 -is_cuda = False
29
22 -try:
23 - import pynvml
24 - pynvml.nvmlInit()
30 +def cuda_platform_plugin() -> Optional[str]:
31 + is_cuda = False
32 +
33 try:
26 - if pynvml.nvmlDeviceGetCount() > 0:
34 + import pynvml
35 + pynvml.nvmlInit()
36 + try:
37 + if pynvml.nvmlDeviceGetCount() > 0:
38 + is_cuda = True
39 + finally:
40 + pynvml.nvmlShutdown()
41 + except Exception:
42 + # CUDA is supported on Jetson, but NVML may not be.
43 + import os
44 +
45 + def cuda_is_jetson() -> bool:
46 + return os.path.isfile("/etc/nv_tegra_release") \
47 + or os.path.exists("/sys/class/tegra-firmware")
48 +
49 + if cuda_is_jetson():
50 is_cuda = True
28 - finally:
29 - pynvml.nvmlShutdown()
30 -except Exception:
31 - # CUDA is supported on Jetson, but NVML may not be.
32 - import os
51
34 - def cuda_is_jetson() -> bool:
35 - return os.path.isfile("/etc/nv_tegra_release") \
36 - or os.path.exists("/sys/class/tegra-firmware")
52 + return "vllm.platforms.cuda.CudaPlatform" if is_cuda else None
53 +
54 +
55 +def rocm_platform_plugin() -> Optional[str]:
56 + is_rocm = False
57 +
58 + try:
59 + import amdsmi
60 + amdsmi.amdsmi_init()
61 + try:
62 + if len(amdsmi.amdsmi_get_processor_handles()) > 0:
63 + is_rocm = True
64 + finally:
65 + amdsmi.amdsmi_shut_down()
66 + except Exception:
67 + pass
68 +
69 + return "vllm.platforms.rocm.RocmPlatform" if is_rocm else None
70 +
71 +
72 +def hpu_platform_plugin() -> Optional[str]:
73 + is_hpu = False
74 + try:
75 + from importlib import util
76 + is_hpu = util.find_spec('habana_frameworks') is not None
77 + except Exception:
78 + pass
79 +
80 + return "vllm.platforms.hpu.HpuPlatform" if is_hpu else None
81 +
82 +
83 +def xpu_platform_plugin() -> Optional[str]:
84 + is_xpu = False
85 +
86 + try:
87 + # installed IPEX if the machine has XPUs.
88 + import intel_extension_for_pytorch # noqa: F401
89 + import oneccl_bindings_for_pytorch # noqa: F401
90 + import torch
91 + if hasattr(torch, 'xpu') and torch.xpu.is_available():
92 + is_xpu = True
93 + except Exception:
94 + pass
95 +
96 + return "vllm.platforms.xpu.XPUPlatform" if is_xpu else None
97 +
98 +
99 +def cpu_platform_plugin() -> Optional[str]:
100 + is_cpu = False
101 + try:
102 + from importlib.metadata import version
103 + is_cpu = "cpu" in version("vllm")
104 + except Exception:
105 + pass
106 +
107 + return "vllm.platforms.cpu.CpuPlatform" if is_cpu else None
108 +
109 +
110 +def neuron_platform_plugin() -> Optional[str]:
111 + is_neuron = False
112 + try:
113 + import transformers_neuronx # noqa: F401
114 + is_neuron = True
115 + except ImportError:
116 + pass
117
38 - if cuda_is_jetson():
39 - is_cuda = True
118 + return "vllm.platforms.neuron.NeuronPlatform" if is_neuron else None
119
41 -is_rocm = False
120
43 -try:
44 - import amdsmi
45 - amdsmi.amdsmi_init()
121 +def openvino_platform_plugin() -> Optional[str]:
122 + is_openvino = False
123 try:
47 - if len(amdsmi.amdsmi_get_processor_handles()) > 0:
48 - is_rocm = True
49 - finally:
50 - amdsmi.amdsmi_shut_down()
51 -except Exception:
52 - pass
53 -
54 -is_hpu = False
55 -try:
56 - from importlib import util
57 - is_hpu = util.find_spec('habana_frameworks') is not None
58 -except Exception:
59 - pass
60 -
61 -is_xpu = False
62 -
63 -try:
64 - # installed IPEX if the machine has XPUs.
65 - import intel_extension_for_pytorch # noqa: F401
66 - import oneccl_bindings_for_pytorch # noqa: F401
67 - import torch
68 - if hasattr(torch, 'xpu') and torch.xpu.is_available():
69 - is_xpu = True
70 -except Exception:
71 - pass
72 -
73 -is_cpu = False
74 -try:
75 - from importlib.metadata import version
76 - is_cpu = "cpu" in version("vllm")
77 -except Exception:
78 - pass
79 -
80 -is_neuron = False
81 -try:
82 - import transformers_neuronx # noqa: F401
83 - is_neuron = True
84 -except ImportError:
85 - pass
86 -
87 -is_openvino = False
88 -try:
89 - from importlib.metadata import version
90 - is_openvino = "openvino" in version("vllm")
91 -except Exception:
92 - pass
93 -
94 -if is_tpu:
95 - # people might install pytorch built with cuda but run on tpu
96 - # so we need to check tpu first
97 - from .tpu import TpuPlatform
98 - current_platform = TpuPlatform()
99 -elif is_cuda:
100 - from .cuda import CudaPlatform
101 - current_platform = CudaPlatform()
102 -elif is_rocm:
103 - from .rocm import RocmPlatform
104 - current_platform = RocmPlatform()
105 -elif is_hpu:
106 - from .hpu import HpuPlatform
107 - current_platform = HpuPlatform()
108 -elif is_xpu:
109 - from .xpu import XPUPlatform
110 - current_platform = XPUPlatform()
111 -elif is_cpu:
112 - from .cpu import CpuPlatform
113 - current_platform = CpuPlatform()
114 -elif is_neuron:
115 - from .neuron import NeuronPlatform
116 - current_platform = NeuronPlatform()
117 -elif is_openvino:
118 - from .openvino import OpenVinoPlatform
119 - current_platform = OpenVinoPlatform()
120 -else:
121 - current_platform = UnspecifiedPlatform()
122 -
123 -__all__ = ['Platform', 'PlatformEnum', 'current_platform', 'CpuArchEnum']
124 + from importlib.metadata import version
125 + is_openvino = "openvino" in version("vllm")
126 + except Exception:
127 + pass
128 +
129 + return "vllm.platforms.openvino.OpenVinoPlatform" if is_openvino else None
130 +
131 +
132 +builtin_platform_plugins = {
133 + 'tpu': tpu_platform_plugin,
134 + 'cuda': cuda_platform_plugin,
135 + 'rocm': rocm_platform_plugin,
136 + 'hpu': hpu_platform_plugin,
137 + 'xpu': xpu_platform_plugin,
138 + 'cpu': cpu_platform_plugin,
139 + 'neuron': neuron_platform_plugin,
140 + 'openvino': openvino_platform_plugin,
141 +}
142 +
143 +
144 +def resolve_current_platform_cls_qualname() -> str:
145 + platform_plugins = load_plugins_by_group('vllm.platform_plugins')
146 +
147 + activated_plugins = []
148 +
149 + for name, func in chain(builtin_platform_plugins.items(),
150 + platform_plugins.items()):
151 + try:
152 + assert callable(func)
153 + platform_cls_qualname = func()
154 + if platform_cls_qualname is not None:
155 + activated_plugins.append(name)
156 + except Exception:
157 + pass
158 +
159 + activated_builtin_plugins = list(
160 + set(activated_plugins) & set(builtin_platform_plugins.keys()))
161 + activated_oot_plugins = list(
162 + set(activated_plugins) & set(platform_plugins.keys()))
163 +
164 + if len(activated_oot_plugins) >= 2:
165 + raise RuntimeError(
166 + "Only one platform plugin can be activated, but got: "
167 + f"{activated_oot_plugins}")
168 + elif len(activated_oot_plugins) == 1:
169 + platform_cls_qualname = platform_plugins[activated_oot_plugins[0]]()
170 + logger.info("Platform plugin %s is activated",
171 + activated_oot_plugins[0])
172 + elif len(activated_builtin_plugins) >= 2:
173 + raise RuntimeError(
174 + "Only one platform plugin can be activated, but got: "
175 + f"{activated_builtin_plugins}")
176 + elif len(activated_builtin_plugins) == 1:
177 + platform_cls_qualname = builtin_platform_plugins[
178 + activated_builtin_plugins[0]]()
179 + logger.info("Automatically detected platform %s.",
180 + activated_builtin_plugins[0])
181 + else:
182 + platform_cls_qualname = "vllm.interface.UnspecifiedPlatform"
183 + logger.info(
184 + "No platform detected, vLLM is running on UnspecifiedPlatform")
185 + return platform_cls_qualname
186 +
187 +
188 +_current_platform = None
189 +_init_trace: str = ''
190 +
191 +if TYPE_CHECKING:
192 + current_platform: Platform
193 +
194 +
195 +def __getattr__(name: str):
196 + if name == 'current_platform':
197 + # lazy init current_platform.
198 + # 1. out-of-tree platform plugins need `from vllm.platforms import
199 + # Platform` so that they can inherit `Platform` class. Therefore,
200 + # we cannot resolve `current_platform` during the import of
201 + # `vllm.platforms`.
202 + # 2. when users use out-of-tree platform plugins, they might run
203 + # `import vllm`, some vllm internal code might access
204 + # `current_platform` during the import, and we need to make sure
205 + # `current_platform` is only resolved after the plugins are loaded
206 + # (we have tests for this, if any developer violate this, they will
207 + # see the test failures).
208 + global _current_platform
209 + if _current_platform is None:
210 + platform_cls_qualname = resolve_current_platform_cls_qualname()
211 + _current_platform = resolve_obj_by_qualname(
212 + platform_cls_qualname)()
213 + global _init_trace
214 + _init_trace = "".join(traceback.format_stack())
215 + return _current_platform
216 + else:
217 + return globals()[name]
218 +
219 +
220 +__all__ = [
221 + 'Platform', 'PlatformEnum', 'current_platform', 'CpuArchEnum',
222 + "_init_trace"
223 +]
vllm/plugins/__init__.py
+41 -31
@@ -1,10 +1,10 @@
1 import logging
2 import os
3 +from typing import Callable, Dict
4
5 import torch
6
7 import vllm.envs as envs
7 -from vllm.platforms import current_platform
8
9 logger = logging.getLogger(__name__)
10
@@ -12,6 +12,39 @@ logger = logging.getLogger(__name__)
12 plugins_loaded = False
13
14
15 +def load_plugins_by_group(group: str) -> Dict[str, Callable]:
16 + import sys
17 + if sys.version_info < (3, 10):
18 + from importlib_metadata import entry_points
19 + else:
20 + from importlib.metadata import entry_points
21 +
22 + allowed_plugins = envs.VLLM_PLUGINS
23 +
24 + discovered_plugins = entry_points(group=group)
25 + if len(discovered_plugins) == 0:
26 + logger.debug("No plugins for group %s found.", group)
27 + return {}
28 + logger.info("Available plugins for group %s:", group)
29 + for plugin in discovered_plugins:
30 + logger.info("name=%s, value=%s", plugin.name, plugin.value)
31 + if allowed_plugins is None:
32 + logger.info("all available plugins for group %s will be loaded.",
33 + group)
34 + logger.info("set environment variable VLLM_PLUGINS to control"
35 + " which plugins to load.")
36 + plugins = {}
37 + for plugin in discovered_plugins:
38 + if allowed_plugins is None or plugin.name in allowed_plugins:
39 + try:
40 + func = plugin.load()
41 + plugins[plugin.name] = func
42 + logger.info("plugin %s loaded.", plugin.name)
43 + except Exception:
44 + logger.exception("Failed to load plugin %s", plugin.name)
45 + return plugins
46 +
47 +
48 def load_general_plugins():
49 """WARNING: plugins can be loaded for multiple times in different
50 processes. They should be designed in a way that they can be loaded
@@ -26,6 +59,9 @@ def load_general_plugins():
59 os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
60 # see https://github.com/vllm-project/vllm/issues/10619
61 torch._inductor.config.compile_threads = 1
62 +
63 + from vllm.platforms import current_platform
64 +
65 if current_platform.is_xpu():
66 # see https://github.com/pytorch/pytorch/blob/8cada5cbe5450e17c26fb8b358116785324537b2/torch/_dynamo/config.py#L158 # noqa
67 os.environ['TORCH_COMPILE_DISABLE'] = 'True'
@@ -47,33 +83,7 @@ def load_general_plugins():
83 if plugins_loaded:
84 return
85 plugins_loaded = True
50 - import sys
51 - if sys.version_info < (3, 10):
52 - from importlib_metadata import entry_points
53 - else:
54 - from importlib.metadata import entry_points
55 -
56 - allowed_plugins = envs.VLLM_PLUGINS
57 -
58 - discovered_plugins = entry_points(group='vllm.general_plugins')
59 - if len(discovered_plugins) == 0:
60 - logger.debug("No plugins found.")
61 - return
62 - logger.info("Available plugins:")
63 - for plugin in discovered_plugins:
64 - logger.info("name=%s, value=%s, group=%s", plugin.name, plugin.value,
65 - plugin.group)
66 - if allowed_plugins is None:
67 - logger.info("all available plugins will be loaded.")
68 - logger.info("set environment variable VLLM_PLUGINS to control"
69 - " which plugins to load.")
70 - else:
71 - logger.info("plugins to load: %s", allowed_plugins)
72 - for plugin in discovered_plugins:
73 - if allowed_plugins is None or plugin.name in allowed_plugins:
74 - try:
75 - func = plugin.load()
76 - func()
77 - logger.info("plugin %s loaded.", plugin.name)
78 - except Exception:
79 - logger.exception("Failed to load plugin %s", plugin.name)
86 + plugins = load_plugins_by_group(group='vllm.general_plugins')
87 + # general plugins, we only need to execute the loaded functions
88 + for func in plugins.values():
89 + func()
vllm/spec_decode/metrics.py
+1 -1
@@ -6,7 +6,6 @@ import torch
6
7 from vllm.model_executor.layers.spec_decode_base_sampler import (
8 SpecDecodeBaseSampler)
9 -from vllm.platforms import current_platform
9 from vllm.utils import is_pin_memory_available
10
11
@@ -94,6 +93,7 @@ class AsyncMetricsCollector:
93 def maybe_collect_rejsample_metrics(
94 self, k: int) -> Optional[SpecDecodeWorkerMetrics]:
95 # currently using cuda.Event, skip for any non_cuda_alike platform
96 + from vllm.platforms import current_platform
97 if not current_platform.is_cuda_alike():
98 return None
99
vllm/usage/usage_lib.py
+1 -1
@@ -17,7 +17,6 @@ import torch
17
18 import vllm.envs as envs
19 from vllm.connections import global_http_connection
20 -from vllm.platforms import current_platform
20 from vllm.version import __version__ as VLLM_VERSION
21
22 _config_home = envs.VLLM_CONFIG_ROOT
@@ -152,6 +151,7 @@ class UsageMessage:
151 usage_context: UsageContext,
152 extra_kvs: Dict[str, Any]) -> None:
153 # Platform information
154 + from vllm.platforms import current_platform
155 if current_platform.is_cuda_alike():
156 device_property = torch.cuda.get_device_properties(0)
157 self.gpu_count = torch.cuda.device_count()
vllm/utils.py
+6 -2
@@ -50,7 +50,6 @@ from typing_extensions import ParamSpec, TypeIs, assert_never
50
51 import vllm.envs as envs
52 from vllm.logger import enable_trace_function_call, init_logger
53 -from vllm.platforms import current_platform
53
54 if TYPE_CHECKING:
55 from vllm.config import VllmConfig
@@ -609,6 +608,7 @@ def create_kv_caches_with_random_flash(
608 seed: int = 0,
609 device: Optional[str] = "cuda",
610 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
611 + from vllm.platforms import current_platform
612 current_platform.seed_everything(seed)
613
614 torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
@@ -650,7 +650,7 @@ def create_kv_caches_with_random(
650 raise ValueError(
651 f"Does not support key cache of type fp8 with head_size {head_size}"
652 )
653 -
653 + from vllm.platforms import current_platform
654 current_platform.seed_everything(seed)
655
656 torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
@@ -703,6 +703,7 @@ def print_warning_once(msg: str) -> None:
703
704 @lru_cache(maxsize=None)
705 def is_pin_memory_available() -> bool:
706 + from vllm.platforms import current_platform
707 return current_platform.is_pin_memory_available()
708
709
@@ -713,6 +714,7 @@ class DeviceMemoryProfiler:
714
715 def current_memory_usage(self) -> float:
716 # Return the memory usage in bytes.
717 + from vllm.platforms import current_platform
718 if current_platform.is_cuda_alike():
719 torch.cuda.reset_peak_memory_stats(self.device)
720 mem = torch.cuda.max_memory_allocated(self.device)
@@ -1066,6 +1068,7 @@ def _cuda_device_count_stateless(
1068 import torch.cuda
1069 import torch.version
1070
1071 + from vllm.platforms import current_platform
1072 if not torch.cuda._is_compiled():
1073 return 0
1074 if current_platform.is_rocm():
@@ -1673,6 +1676,7 @@ def direct_register_custom_op(
1676 return
1677
1678 if not supports_custom_op():
1679 + from vllm.platforms import current_platform
1680 assert not current_platform.is_cuda_alike(), (
1681 "cuda platform needs torch>=2.4 to support custom op, "
1682 "chances are you are using an old version of pytorch "
vllm/worker/model_runner_base.py
+2 -3
@@ -12,7 +12,6 @@ from torch import is_tensor
12 from vllm.config import VllmConfig
13 from vllm.logger import init_logger
14 from vllm.model_executor.layers.sampler import SamplerOutput
15 -from vllm.platforms import current_platform
15 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
16
17 if TYPE_CHECKING:
@@ -265,13 +264,13 @@ class ModelRunnerBase(ABC, Generic[T]):
264 """
265 raise NotImplementedError
266
268 - @current_platform.inference_mode()
267 def execute_model(
268 self,
269 model_input: T,
270 kv_caches: Optional[List[torch.Tensor]],
273 - intermediate_tensors: Optional[IntermediateTensors],
271 + intermediate_tensors: Optional[IntermediateTensors] = None,
272 num_steps: int = 1,
273 + **kwargs,
274 ) -> Optional[List[SamplerOutput]]:
275 """
276 Execute the model on the given input.
vllm/worker/multi_step_model_runner.py
+1
@@ -544,6 +544,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
544 model_input.record_step_event(current_stream)
545
546 if get_pp_group().is_last_rank and self.is_driver_worker:
547 + assert isinstance(output, list)
548 assert len(
549 output
550 ) == 1, "MultiStepModelRunner requires single-step base_models"
vllm/worker/worker_base.py
+8 -6
@@ -11,7 +11,6 @@ from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
13 from vllm.model_executor.layers.sampler import SamplerOutput
14 -from vllm.platforms import current_platform
14 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
15 from vllm.utils import (enable_trace_function_call_for_thread,
16 resolve_obj_by_qualname, update_environment_variables)
@@ -44,6 +43,8 @@ class WorkerBase(ABC):
43 self.prompt_adapter_config = vllm_config.prompt_adapter_config
44 self.observability_config = vllm_config.observability_config
45 self.kv_transfer_config = vllm_config.kv_transfer_config
46 + from vllm.platforms import current_platform
47 + self.current_platform = current_platform
48
49 @abstractmethod
50 def init_device(self) -> None:
@@ -74,17 +75,17 @@ class WorkerBase(ABC):
75 """
76 raise NotImplementedError
77
77 - @current_platform.inference_mode()
78 def start_worker_execution_loop(self) -> None:
79 """Execute model loop in parallel worker.
80
81 You can stop the loop by executing a driver worker with an empty output.
82 See `stop_remote_worker_execution_loop` for more details.
83 """
84 - while True:
85 - output = self.execute_model(execute_model_req=None)
86 - if output is None:
87 - return None
84 + with self.current_platform.inference_mode():
85 + while True:
86 + output = self.execute_model(execute_model_req=None)
87 + if output is None:
88 + return None
89
90 @abstractmethod
91 def execute_model(
@@ -352,6 +353,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
353 model_execute_time = time.perf_counter() - start_time
354 if not get_pp_group().is_last_rank:
355 # output is IntermediateTensors
356 + assert isinstance(output, IntermediateTensors)
357 if (self.observability_config is not None
358 and self.observability_config.collect_model_execute_time):
359 output.tensors["model_execute_time"] = torch.tensor(