A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[platforms] enable platform plugins (#11602)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 30, 2024 at 20:24 UTC
b12e87f942eb7740c17ab546b964bc327afdda37
23 files changed
+360
-181
.buildkite/test-pipeline.yaml
+19
-6
@@ -106,14 +106,12 @@ steps:
106
source_file_dependencies:
107
- vllm/
108
commands:
109
- - pip install -e ./plugins/vllm_add_dummy_model
109
- pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_generate_multiple_loras.py --ignore=entrypoints/llm/test_guided_generate.py
110
- pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
111
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
112
- pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
113
- pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process
114
- pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py
116
- - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process
115
- pytest -v -s entrypoints/test_chat_utils.py
116
- pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
117
@@ -333,8 +331,6 @@ steps:
331
- vllm/
332
- tests/models
333
commands:
336
- - pip install -e ./plugins/vllm_add_dummy_model
337
- - pytest -v -s models/test_oot_registration.py # it needs a clean process
334
- pytest -v -s models/test_registry.py
335
- pytest -v -s models/test_initialization.py
336
@@ -469,11 +465,28 @@ steps:
465
- pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m 'distributed(num_gpus=2)'
466
- pytest models/decoder_only/vision_language/test_models.py -v -s -m 'distributed(num_gpus=2)'
467
- pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
472
- - pip install -e ./plugins/vllm_add_dummy_model
473
- - pytest -v -s distributed/test_distributed_oot.py
468
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
469
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s kv_transfer/disagg_test.py
470
471
+- label: Plugin Tests (2 GPUs) # 40min
472
+ working_dir: "/vllm-workspace/tests"
473
+ num_gpus: 2
474
+ fast_check: true
475
+ source_file_dependencies:
476
+ - vllm/plugins/
477
+ - tests/plugins/
478
+ commands:
479
+ # begin platform plugin tests, all the code in-between runs on dummy platform
480
+ - pip install -e ./plugins/vllm_add_dummy_platform
481
+ - pytest -v -s plugins_tests/test_platform_plugins.py
482
+ - pip uninstall vllm_add_dummy_platform -y
483
+ # end platform plugin tests
484
+ # other tests continue here:
485
+ - pip install -e ./plugins/vllm_add_dummy_model
486
+ - pytest -v -s distributed/test_distributed_oot.py
487
+ - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process
488
+ - pytest -v -s models/test_oot_registration.py # it needs a clean process
489
+
490
- label: Multi-step Tests (4 GPUs) # 36min
491
working_dir: "/vllm-workspace/tests"
492
num_gpus: 4
docs/source/design/plugin_system.md
+4
-2
@@ -41,9 +41,11 @@ Every plugin has three parts:
41
2. **Plugin name**: The name of the plugin. This is the value in the dictionary of the `entry_points` dictionary. In the example above, the plugin name is `register_dummy_model`. Plugins can be filtered by their names using the `VLLM_PLUGINS` environment variable. To load only a specific plugin, set `VLLM_PLUGINS` to the plugin name.
42
3. **Plugin value**: The fully qualified name of the function to register in the plugin system. In the example above, the plugin value is `vllm_add_dummy_model:register`, which refers to a function named `register` in the `vllm_add_dummy_model` module.
43
44
-## What Can Plugins Do?
44
+## Types of supported plugins
45
46
-Currently, the primary use case for plugins is to register custom, out-of-the-tree models into vLLM. This is done by calling `ModelRegistry.register_model` to register the model. In the future, the plugin system may be extended to support more features, such as swapping in custom implementations for certain classes in vLLM.
46
+- **General plugins** (with group name `vllm.general_plugins`): The primary use case for these plugins is to register custom, out-of-the-tree models into vLLM. This is done by calling `ModelRegistry.register_model` to register the model inside the plugin function.
47
+
48
+- **Platform plugins** (with group name `vllm.platform_plugins`): The primary use case for these plugins is to register custom, out-of-the-tree platforms into vLLM. The plugin function should return `None` when the platform is not supported in the current environment, or the platform class's fully qualified name when the platform is supported.
49
50
## Guidelines for Writing Plugins
51
tests/conftest.py
+1
-1
@@ -31,7 +31,6 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
31
to_enc_dec_tuple_list, zip_enc_dec_prompts)
32
from vllm.logger import init_logger
33
from vllm.outputs import RequestOutput
34
-from vllm.platforms import current_platform
34
from vllm.sampling_params import BeamSearchParams
35
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
36
identity)
@@ -242,6 +241,7 @@ _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
241
class HfRunner:
242
243
def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
244
+ from vllm.platforms import current_platform
245
if x is None or isinstance(x, (bool, )):
246
return x
247
tests/kernels/test_attention_selector.py
+8
-8
@@ -5,7 +5,10 @@ import torch
5
6
from tests.kernels.utils import override_backend_env_variable
7
from vllm.attention.selector import which_attn_to_use
8
-from vllm.platforms import cpu, cuda, openvino, rocm
8
+from vllm.platforms.cpu import CpuPlatform
9
+from vllm.platforms.cuda import CudaPlatform
10
+from vllm.platforms.openvino import OpenVinoPlatform
11
+from vllm.platforms.rocm import RocmPlatform
12
from vllm.utils import STR_FLASH_ATTN_VAL, STR_INVALID_VAL
13
14
@@ -20,26 +23,23 @@ def test_env(name: str, device: str, monkeypatch):
23
override_backend_env_variable(monkeypatch, name)
24
25
if device == "cpu":
23
- with patch("vllm.attention.selector.current_platform",
24
- cpu.CpuPlatform()):
26
+ with patch("vllm.attention.selector.current_platform", CpuPlatform()):
27
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
28
False)
29
assert backend.name == "TORCH_SDPA"
30
elif device == "hip":
29
- with patch("vllm.attention.selector.current_platform",
30
- rocm.RocmPlatform()):
31
+ with patch("vllm.attention.selector.current_platform", RocmPlatform()):
32
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
33
False)
34
assert backend.name == "ROCM_FLASH"
35
elif device == "openvino":
36
with patch("vllm.attention.selector.current_platform",
36
- openvino.OpenVinoPlatform()):
37
+ OpenVinoPlatform()):
38
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
39
False)
40
assert backend.name == "OPENVINO"
41
else:
41
- with patch("vllm.attention.selector.current_platform",
42
- cuda.CudaPlatform()):
42
+ with patch("vllm.attention.selector.current_platform", CudaPlatform()):
43
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
44
False)
45
assert backend.name == name
tests/plugins/vllm_add_dummy_platform/setup.py
new
+11
@@ -0,0 +1,11 @@
1
+from setuptools import setup
2
+
3
+setup(
4
+ name='vllm_add_dummy_platform',
5
+ version='0.1',
6
+ packages=['vllm_add_dummy_platform'],
7
+ entry_points={
8
+ 'vllm.platform_plugins': [
9
+ "dummy_platform_plugin = vllm_add_dummy_platform:dummy_platform_plugin" # noqa
10
+ ]
11
+ })
tests/plugins/vllm_add_dummy_platform/vllm_add_dummy_platform/__init__.py
new
+5
@@ -0,0 +1,5 @@
1
+from typing import Optional
2
+
3
+
4
+def dummy_platform_plugin() -> Optional[str]:
5
+ return "vllm_add_dummy_platform.dummy_platform.DummyPlatform"
tests/plugins/vllm_add_dummy_platform/vllm_add_dummy_platform/dummy_platform.py
new
+5
@@ -0,0 +1,5 @@
1
+from vllm.platforms.cuda import CudaPlatform
2
+
3
+
4
+class DummyPlatform(CudaPlatform):
5
+ device_name = "DummyDevice"
tests/plugins_tests/test_platform_plugins.py
new
+16
@@ -0,0 +1,16 @@
1
+def test_platform_plugins():
2
+ # simulate workload by running an example
3
+ import runpy
4
+ current_file = __file__
5
+ import os
6
+ example_file = os.path.join(
7
+ os.path.dirname(os.path.dirname(os.path.dirname(current_file))),
8
+ "examples", "offline_inference.py")
9
+ runpy.run_path(example_file)
10
+
11
+ # check if the plugin is loaded correctly
12
+ from vllm.platforms import _init_trace, current_platform
13
+ assert current_platform.device_name == "DummyDevice", (
14
+ f"Expected DummyDevice, got {current_platform.device_name}, "
15
+ "possibly because current_platform is imported before the plugin"
16
+ f" is loaded. The first import:\n{_init_trace}")
vllm/config.py
+12
-3
@@ -22,7 +22,7 @@ from vllm.logger import init_logger
22
from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
23
get_quantization_config)
24
from vllm.model_executor.models import ModelRegistry
25
-from vllm.platforms import current_platform, interface
25
+from vllm.platforms import CpuArchEnum
26
from vllm.tracing import is_otel_available, otel_import_error_traceback
27
from vllm.transformers_utils.config import (
28
ConfigFormat, get_config, get_hf_image_processor_config,
@@ -349,6 +349,7 @@ class ModelConfig:
349
self.is_hybrid = self._init_is_hybrid()
350
self.has_inner_state = self._init_has_inner_state()
351
352
+ from vllm.platforms import current_platform
353
if current_platform.is_neuron():
354
self.override_neuron_config = override_neuron_config
355
else:
@@ -589,6 +590,7 @@ class ModelConfig:
590
raise ValueError(
591
f"Unknown quantization method: {self.quantization}. Must "
592
f"be one of {supported_quantization}.")
593
+ from vllm.platforms import current_platform
594
current_platform.verify_quantization(self.quantization)
595
if self.quantization not in optimized_quantization_methods:
596
logger.warning(
@@ -644,6 +646,7 @@ class ModelConfig:
646
647
# Reminder: Please update docs/source/usage/compatibility_matrix.md
648
# If the feature combo become valid
649
+ from vllm.platforms import current_platform
650
if not current_platform.is_async_output_supported(self.enforce_eager):
651
logger.warning(
652
"Async output processing is not supported on the "
@@ -1012,6 +1015,7 @@ class CacheConfig:
1015
raise ValueError(
1016
"GPU memory utilization must be less than 1.0. Got "
1017
f"{self.gpu_memory_utilization}.")
1018
+ from vllm.platforms import current_platform
1019
if (current_platform.is_cuda() and self.block_size is not None
1020
and self.block_size > 32):
1021
raise ValueError("CUDA Paged Attention kernel only supports "
@@ -1279,6 +1283,7 @@ class ParallelConfig:
1283
f"distributed executor backend "
1284
f"'{self.distributed_executor_backend}'.")
1285
ray_only_devices = ["tpu", "hpu"]
1286
+ from vllm.platforms import current_platform
1287
if (current_platform.device_type in ray_only_devices
1288
and self.world_size > 1):
1289
if self.distributed_executor_backend is None:
@@ -1327,7 +1332,7 @@ class ParallelConfig:
1332
def _verify_args(self) -> None:
1333
# Lazy import to avoid circular import
1334
from vllm.executor.executor_base import ExecutorBase
1330
-
1335
+ from vllm.platforms import current_platform
1336
if self.distributed_executor_backend not in (
1337
"ray", "mp", None) and not (isinstance(
1338
self.distributed_executor_backend, type) and issubclass(
@@ -1528,6 +1533,7 @@ class DeviceConfig:
1533
def __init__(self, device: str = "auto") -> None:
1534
if device == "auto":
1535
# Automated device type detection
1536
+ from vllm.platforms import current_platform
1537
self.device_type = current_platform.device_type
1538
if not self.device_type:
1539
raise RuntimeError("Failed to infer device type")
@@ -2241,9 +2247,10 @@ def _get_and_verify_dtype(
2247
else:
2248
torch_dtype = config_dtype
2249
2250
+ from vllm.platforms import current_platform
2251
if (current_platform.is_cpu()
2252
and current_platform.get_cpu_architecture()
2246
- == interface.CpuArchEnum.POWERPC
2253
+ == CpuArchEnum.POWERPC
2254
and (config_dtype == torch.float16
2255
or config_dtype == torch.float32)):
2256
logger.info(
@@ -3083,6 +3090,7 @@ class VllmConfig:
3090
model_config: ModelConfig,
3091
load_config: LoadConfig) -> Optional[QuantizationConfig]:
3092
"""Get the quantization config."""
3093
+ from vllm.platforms import current_platform
3094
if model_config.quantization is not None:
3095
from vllm.model_executor.model_loader.weight_utils import (
3096
get_quant_config)
@@ -3145,6 +3153,7 @@ class VllmConfig:
3153
self.quant_config = VllmConfig._get_quantization_config(
3154
self.model_config, self.load_config)
3155
3156
+ from vllm.platforms import current_platform
3157
if self.scheduler_config is not None and \
3158
self.model_config is not None and \
3159
self.scheduler_config.chunked_prefill_enabled and \
vllm/distributed/parallel_state.py
+2
-1
@@ -39,7 +39,6 @@ import vllm.distributed.kv_transfer.kv_transfer_agent as kv_transfer
39
import vllm.envs as envs
40
from vllm.distributed.utils import StatelessProcessGroup
41
from vllm.logger import init_logger
42
-from vllm.platforms import current_platform
42
from vllm.utils import direct_register_custom_op, supports_custom_op
43
44
if TYPE_CHECKING:
@@ -194,6 +193,7 @@ class GroupCoordinator:
193
assert self.cpu_group is not None
194
assert self.device_group is not None
195
196
+ from vllm.platforms import current_platform
197
if current_platform.is_cuda_alike():
198
self.device = torch.device(f"cuda:{local_rank}")
199
else:
@@ -1188,6 +1188,7 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1188
import ray # Lazy import Ray
1189
ray.shutdown()
1190
gc.collect()
1191
+ from vllm.platforms import current_platform
1192
if not current_platform.is_cpu():
1193
torch.cuda.empty_cache()
1194
vllm/engine/arg_utils.py
+1
-1
@@ -18,7 +18,6 @@ from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,
18
from vllm.executor.executor_base import ExecutorBase
19
from vllm.logger import init_logger
20
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
21
-from vllm.platforms import current_platform
21
from vllm.transformers_utils.utils import check_gguf_file
22
from vllm.usage.usage_lib import UsageContext
23
from vllm.utils import FlexibleArgumentParser, StoreBoolean
@@ -1094,6 +1093,7 @@ class EngineArgs:
1093
use_sliding_window = (model_config.get_sliding_window()
1094
is not None)
1095
use_spec_decode = self.speculative_model is not None
1096
+ from vllm.platforms import current_platform
1097
if (is_gpu and not use_sliding_window and not use_spec_decode
1098
and not self.enable_lora
1099
and not self.enable_prompt_adapter
vllm/executor/ray_utils.py
+1
-1
@@ -8,7 +8,6 @@ import msgspec
8
from vllm.config import ParallelConfig
9
from vllm.executor.msgspec_utils import decode_hook, encode_hook
10
from vllm.logger import init_logger
11
-from vllm.platforms import current_platform
11
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
12
from vllm.utils import get_ip
13
from vllm.worker.worker_base import WorkerWrapperBase
@@ -229,6 +228,7 @@ def initialize_ray_cluster(
228
the default Ray cluster address.
229
"""
230
assert_ray_available()
231
+ from vllm.platforms import current_platform
232
233
# Connect to a ray cluster.
234
if current_platform.is_rocm() or current_platform.is_xpu():
vllm/model_executor/guided_decoding/__init__.py
+2
-1
@@ -6,7 +6,7 @@ from vllm.logger import init_logger
6
from vllm.model_executor.guided_decoding.utils import (
7
convert_lark_to_gbnf, grammar_is_likely_lark,
8
has_lmf_unsupported_json_features, has_xgrammar_unsupported_json_features)
9
-from vllm.platforms import CpuArchEnum, current_platform
9
+from vllm.platforms import CpuArchEnum
10
11
if TYPE_CHECKING:
12
from transformers import PreTrainedTokenizer
@@ -39,6 +39,7 @@ def maybe_backend_fallback(
39
40
if guided_params.backend == "xgrammar":
41
# xgrammar only has x86 wheels for linux, fallback to outlines
42
+ from vllm.platforms import current_platform
43
if current_platform.get_cpu_architecture() is not CpuArchEnum.X86:
44
logger.warning("xgrammar is only supported on x86 CPUs. "
45
"Falling back to use outlines instead.")
vllm/model_executor/models/registry.py
+1
-1
@@ -18,7 +18,6 @@ import cloudpickle
18
import torch.nn as nn
19
20
from vllm.logger import init_logger
21
-from vllm.platforms import current_platform
21
22
from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
23
supports_cross_encoding, supports_multimodal,
@@ -273,6 +272,7 @@ def _try_load_model_cls(
272
model_arch: str,
273
model: _BaseRegisteredModel,
274
) -> Optional[Type[nn.Module]]:
275
+ from vllm.platforms import current_platform
276
current_platform.verify_model_arch(model_arch)
277
try:
278
return model.load_model_cls()
vllm/model_executor/utils.py
+2
-2
@@ -3,10 +3,9 @@ from typing import Any, Dict, Optional
3
4
import torch
5
6
-from vllm.platforms import current_platform
7
-
6
7
def set_random_seed(seed: int) -> None:
8
+ from vllm.platforms import current_platform
9
current_platform.seed_everything(seed)
10
11
@@ -38,6 +37,7 @@ def set_weight_attrs(
37
# This sometimes causes OOM errors during model loading. To avoid this,
38
# we sync the param tensor after its weight loader is called.
39
# TODO(woosuk): Remove this hack once we have a better solution.
40
+ from vllm.platforms import current_platform
41
if current_platform.is_tpu() and key == "weight_loader":
42
value = _make_synced_weight_loader(value)
43
setattr(weight, key, value)
vllm/platforms/__init__.py
+210
-110
@@ -1,123 +1,223 @@
1
+import logging
2
+import traceback
3
+from itertools import chain
4
+from typing import TYPE_CHECKING, Optional
5
+
6
+from vllm.plugins import load_plugins_by_group
7
+from vllm.utils import resolve_obj_by_qualname
8
+
9
from .interface import _Backend # noqa: F401
2
-from .interface import CpuArchEnum, Platform, PlatformEnum, UnspecifiedPlatform
10
+from .interface import CpuArchEnum, Platform, PlatformEnum
11
4
-current_platform: Platform
12
+logger = logging.getLogger(__name__)
13
6
-# NOTE: we don't use `torch.version.cuda` / `torch.version.hip` because
7
-# they only indicate the build configuration, not the runtime environment.
8
-# For example, people can install a cuda build of pytorch but run on tpu.
14
10
-is_tpu = False
11
-try:
12
- # While it's technically possible to install libtpu on a non-TPU machine,
13
- # this is a very uncommon scenario. Therefore, we assume that libtpu is
14
- # installed if and only if the machine has TPUs.
15
- import libtpu # noqa: F401
16
- is_tpu = True
17
-except Exception:
18
- pass
15
+def tpu_platform_plugin() -> Optional[str]:
16
+ is_tpu = False
17
+ try:
18
+ # While it's technically possible to install libtpu on a
19
+ # non-TPU machine, this is a very uncommon scenario. Therefore,
20
+ # we assume that libtpu is installed if and only if the machine
21
+ # has TPUs.
22
+ import libtpu # noqa: F401
23
+ is_tpu = True
24
+ except Exception:
25
+ pass
26
+
27
+ return "vllm.platforms.tpu.TpuPlatform" if is_tpu else None
28
20
-is_cuda = False
29
22
-try:
23
- import pynvml
24
- pynvml.nvmlInit()
30
+def cuda_platform_plugin() -> Optional[str]:
31
+ is_cuda = False
32
+
33
try:
26
- if pynvml.nvmlDeviceGetCount() > 0:
34
+ import pynvml
35
+ pynvml.nvmlInit()
36
+ try:
37
+ if pynvml.nvmlDeviceGetCount() > 0:
38
+ is_cuda = True
39
+ finally:
40
+ pynvml.nvmlShutdown()
41
+ except Exception:
42
+ # CUDA is supported on Jetson, but NVML may not be.
43
+ import os
44
+
45
+ def cuda_is_jetson() -> bool:
46
+ return os.path.isfile("/etc/nv_tegra_release") \
47
+ or os.path.exists("/sys/class/tegra-firmware")
48
+
49
+ if cuda_is_jetson():
50
is_cuda = True
28
- finally:
29
- pynvml.nvmlShutdown()
30
-except Exception:
31
- # CUDA is supported on Jetson, but NVML may not be.
32
- import os
51
34
- def cuda_is_jetson() -> bool:
35
- return os.path.isfile("/etc/nv_tegra_release") \
36
- or os.path.exists("/sys/class/tegra-firmware")
52
+ return "vllm.platforms.cuda.CudaPlatform" if is_cuda else None
53
+
54
+
55
+def rocm_platform_plugin() -> Optional[str]:
56
+ is_rocm = False
57
+
58
+ try:
59
+ import amdsmi
60
+ amdsmi.amdsmi_init()
61
+ try:
62
+ if len(amdsmi.amdsmi_get_processor_handles()) > 0:
63
+ is_rocm = True
64
+ finally:
65
+ amdsmi.amdsmi_shut_down()
66
+ except Exception:
67
+ pass
68
+
69
+ return "vllm.platforms.rocm.RocmPlatform" if is_rocm else None
70
+
71
+
72
+def hpu_platform_plugin() -> Optional[str]:
73
+ is_hpu = False
74
+ try:
75
+ from importlib import util
76
+ is_hpu = util.find_spec('habana_frameworks') is not None
77
+ except Exception:
78
+ pass
79
+
80
+ return "vllm.platforms.hpu.HpuPlatform" if is_hpu else None
81
+
82
+
83
+def xpu_platform_plugin() -> Optional[str]:
84
+ is_xpu = False
85
+
86
+ try:
87
+ # installed IPEX if the machine has XPUs.
88
+ import intel_extension_for_pytorch # noqa: F401
89
+ import oneccl_bindings_for_pytorch # noqa: F401
90
+ import torch
91
+ if hasattr(torch, 'xpu') and torch.xpu.is_available():
92
+ is_xpu = True
93
+ except Exception:
94
+ pass
95
+
96
+ return "vllm.platforms.xpu.XPUPlatform" if is_xpu else None
97
+
98
+
99
+def cpu_platform_plugin() -> Optional[str]:
100
+ is_cpu = False
101
+ try:
102
+ from importlib.metadata import version
103
+ is_cpu = "cpu" in version("vllm")
104
+ except Exception:
105
+ pass
106
+
107
+ return "vllm.platforms.cpu.CpuPlatform" if is_cpu else None
108
+
109
+
110
+def neuron_platform_plugin() -> Optional[str]:
111
+ is_neuron = False
112
+ try:
113
+ import transformers_neuronx # noqa: F401
114
+ is_neuron = True
115
+ except ImportError:
116
+ pass
117
38
- if cuda_is_jetson():
39
- is_cuda = True
118
+ return "vllm.platforms.neuron.NeuronPlatform" if is_neuron else None
119
41
-is_rocm = False
120
43
-try:
44
- import amdsmi
45
- amdsmi.amdsmi_init()
121
+def openvino_platform_plugin() -> Optional[str]:
122
+ is_openvino = False
123
try:
47
- if len(amdsmi.amdsmi_get_processor_handles()) > 0:
48
- is_rocm = True
49
- finally:
50
- amdsmi.amdsmi_shut_down()
51
-except Exception:
52
- pass
53
-
54
-is_hpu = False
55
-try:
56
- from importlib import util
57
- is_hpu = util.find_spec('habana_frameworks') is not None
58
-except Exception:
59
- pass
60
-
61
-is_xpu = False
62
-
63
-try:
64
- # installed IPEX if the machine has XPUs.
65
- import intel_extension_for_pytorch # noqa: F401
66
- import oneccl_bindings_for_pytorch # noqa: F401
67
- import torch
68
- if hasattr(torch, 'xpu') and torch.xpu.is_available():
69
- is_xpu = True
70
-except Exception:
71
- pass
72
-
73
-is_cpu = False
74
-try:
75
- from importlib.metadata import version
76
- is_cpu = "cpu" in version("vllm")
77
-except Exception:
78
- pass
79
-
80
-is_neuron = False
81
-try:
82
- import transformers_neuronx # noqa: F401
83
- is_neuron = True
84
-except ImportError:
85
- pass
86
-
87
-is_openvino = False
88
-try:
89
- from importlib.metadata import version
90
- is_openvino = "openvino" in version("vllm")
91
-except Exception:
92
- pass
93
-
94
-if is_tpu:
95
- # people might install pytorch built with cuda but run on tpu
96
- # so we need to check tpu first
97
- from .tpu import TpuPlatform
98
- current_platform = TpuPlatform()
99
-elif is_cuda:
100
- from .cuda import CudaPlatform
101
- current_platform = CudaPlatform()
102
-elif is_rocm:
103
- from .rocm import RocmPlatform
104
- current_platform = RocmPlatform()
105
-elif is_hpu:
106
- from .hpu import HpuPlatform
107
- current_platform = HpuPlatform()
108
-elif is_xpu:
109
- from .xpu import XPUPlatform
110
- current_platform = XPUPlatform()
111
-elif is_cpu:
112
- from .cpu import CpuPlatform
113
- current_platform = CpuPlatform()
114
-elif is_neuron:
115
- from .neuron import NeuronPlatform
116
- current_platform = NeuronPlatform()
117
-elif is_openvino:
118
- from .openvino import OpenVinoPlatform
119
- current_platform = OpenVinoPlatform()
120
-else:
121
- current_platform = UnspecifiedPlatform()
122
-
123
-__all__ = ['Platform', 'PlatformEnum', 'current_platform', 'CpuArchEnum']
124
+ from importlib.metadata import version
125
+ is_openvino = "openvino" in version("vllm")
126
+ except Exception:
127
+ pass
128
+
129
+ return "vllm.platforms.openvino.OpenVinoPlatform" if is_openvino else None
130
+
131
+
132
+builtin_platform_plugins = {
133
+ 'tpu': tpu_platform_plugin,
134
+ 'cuda': cuda_platform_plugin,
135
+ 'rocm': rocm_platform_plugin,
136
+ 'hpu': hpu_platform_plugin,
137
+ 'xpu': xpu_platform_plugin,
138
+ 'cpu': cpu_platform_plugin,
139
+ 'neuron': neuron_platform_plugin,
140
+ 'openvino': openvino_platform_plugin,
141
+}
142
+
143
+
144
+def resolve_current_platform_cls_qualname() -> str:
145
+ platform_plugins = load_plugins_by_group('vllm.platform_plugins')
146
+
147
+ activated_plugins = []
148
+
149
+ for name, func in chain(builtin_platform_plugins.items(),
150
+ platform_plugins.items()):
151
+ try:
152
+ assert callable(func)
153
+ platform_cls_qualname = func()
154
+ if platform_cls_qualname is not None:
155
+ activated_plugins.append(name)
156
+ except Exception:
157
+ pass
158
+
159
+ activated_builtin_plugins = list(
160
+ set(activated_plugins) & set(builtin_platform_plugins.keys()))
161
+ activated_oot_plugins = list(
162
+ set(activated_plugins) & set(platform_plugins.keys()))
163
+
164
+ if len(activated_oot_plugins) >= 2:
165
+ raise RuntimeError(
166
+ "Only one platform plugin can be activated, but got: "
167
+ f"{activated_oot_plugins}")
168
+ elif len(activated_oot_plugins) == 1:
169
+ platform_cls_qualname = platform_plugins[activated_oot_plugins[0]]()
170
+ logger.info("Platform plugin %s is activated",
171
+ activated_oot_plugins[0])
172
+ elif len(activated_builtin_plugins) >= 2:
173
+ raise RuntimeError(
174
+ "Only one platform plugin can be activated, but got: "
175
+ f"{activated_builtin_plugins}")
176
+ elif len(activated_builtin_plugins) == 1:
177
+ platform_cls_qualname = builtin_platform_plugins[
178
+ activated_builtin_plugins[0]]()
179
+ logger.info("Automatically detected platform %s.",
180
+ activated_builtin_plugins[0])
181
+ else:
182
+ platform_cls_qualname = "vllm.interface.UnspecifiedPlatform"
183
+ logger.info(
184
+ "No platform detected, vLLM is running on UnspecifiedPlatform")
185
+ return platform_cls_qualname
186
+
187
+
188
+_current_platform = None
189
+_init_trace: str = ''
190
+
191
+if TYPE_CHECKING:
192
+ current_platform: Platform
193
+
194
+
195
+def __getattr__(name: str):
196
+ if name == 'current_platform':
197
+ # lazy init current_platform.
198
+ # 1. out-of-tree platform plugins need `from vllm.platforms import
199
+ # Platform` so that they can inherit `Platform` class. Therefore,
200
+ # we cannot resolve `current_platform` during the import of
201
+ # `vllm.platforms`.
202
+ # 2. when users use out-of-tree platform plugins, they might run
203
+ # `import vllm`, some vllm internal code might access
204
+ # `current_platform` during the import, and we need to make sure
205
+ # `current_platform` is only resolved after the plugins are loaded
206
+ # (we have tests for this, if any developer violate this, they will
207
+ # see the test failures).
208
+ global _current_platform
209
+ if _current_platform is None:
210
+ platform_cls_qualname = resolve_current_platform_cls_qualname()
211
+ _current_platform = resolve_obj_by_qualname(
212
+ platform_cls_qualname)()
213
+ global _init_trace
214
+ _init_trace = "".join(traceback.format_stack())
215
+ return _current_platform
216
+ else:
217
+ return globals()[name]
218
+
219
+
220
+__all__ = [
221
+ 'Platform', 'PlatformEnum', 'current_platform', 'CpuArchEnum',
222
+ "_init_trace"
223
+]
vllm/plugins/__init__.py
+41
-31
@@ -1,10 +1,10 @@
1
import logging
2
import os
3
+from typing import Callable, Dict
4
5
import torch
6
7
import vllm.envs as envs
7
-from vllm.platforms import current_platform
8
9
logger = logging.getLogger(__name__)
10
@@ -12,6 +12,39 @@ logger = logging.getLogger(__name__)
12
plugins_loaded = False
13
14
15
+def load_plugins_by_group(group: str) -> Dict[str, Callable]:
16
+ import sys
17
+ if sys.version_info < (3, 10):
18
+ from importlib_metadata import entry_points
19
+ else:
20
+ from importlib.metadata import entry_points
21
+
22
+ allowed_plugins = envs.VLLM_PLUGINS
23
+
24
+ discovered_plugins = entry_points(group=group)
25
+ if len(discovered_plugins) == 0:
26
+ logger.debug("No plugins for group %s found.", group)
27
+ return {}
28
+ logger.info("Available plugins for group %s:", group)
29
+ for plugin in discovered_plugins:
30
+ logger.info("name=%s, value=%s", plugin.name, plugin.value)
31
+ if allowed_plugins is None:
32
+ logger.info("all available plugins for group %s will be loaded.",
33
+ group)
34
+ logger.info("set environment variable VLLM_PLUGINS to control"
35
+ " which plugins to load.")
36
+ plugins = {}
37
+ for plugin in discovered_plugins:
38
+ if allowed_plugins is None or plugin.name in allowed_plugins:
39
+ try:
40
+ func = plugin.load()
41
+ plugins[plugin.name] = func
42
+ logger.info("plugin %s loaded.", plugin.name)
43
+ except Exception:
44
+ logger.exception("Failed to load plugin %s", plugin.name)
45
+ return plugins
46
+
47
+
48
def load_general_plugins():
49
"""WARNING: plugins can be loaded for multiple times in different
50
processes. They should be designed in a way that they can be loaded
@@ -26,6 +59,9 @@ def load_general_plugins():
59
os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
60
# see https://github.com/vllm-project/vllm/issues/10619
61
torch._inductor.config.compile_threads = 1
62
+
63
+ from vllm.platforms import current_platform
64
+
65
if current_platform.is_xpu():
66
# see https://github.com/pytorch/pytorch/blob/8cada5cbe5450e17c26fb8b358116785324537b2/torch/_dynamo/config.py#L158 # noqa
67
os.environ['TORCH_COMPILE_DISABLE'] = 'True'
@@ -47,33 +83,7 @@ def load_general_plugins():
83
if plugins_loaded:
84
return
85
plugins_loaded = True
50
- import sys
51
- if sys.version_info < (3, 10):
52
- from importlib_metadata import entry_points
53
- else:
54
- from importlib.metadata import entry_points
55
-
56
- allowed_plugins = envs.VLLM_PLUGINS
57
-
58
- discovered_plugins = entry_points(group='vllm.general_plugins')
59
- if len(discovered_plugins) == 0:
60
- logger.debug("No plugins found.")
61
- return
62
- logger.info("Available plugins:")
63
- for plugin in discovered_plugins:
64
- logger.info("name=%s, value=%s, group=%s", plugin.name, plugin.value,
65
- plugin.group)
66
- if allowed_plugins is None:
67
- logger.info("all available plugins will be loaded.")
68
- logger.info("set environment variable VLLM_PLUGINS to control"
69
- " which plugins to load.")
70
- else:
71
- logger.info("plugins to load: %s", allowed_plugins)
72
- for plugin in discovered_plugins:
73
- if allowed_plugins is None or plugin.name in allowed_plugins:
74
- try:
75
- func = plugin.load()
76
- func()
77
- logger.info("plugin %s loaded.", plugin.name)
78
- except Exception:
79
- logger.exception("Failed to load plugin %s", plugin.name)
86
+ plugins = load_plugins_by_group(group='vllm.general_plugins')
87
+ # general plugins, we only need to execute the loaded functions
88
+ for func in plugins.values():
89
+ func()
vllm/spec_decode/metrics.py
+1
-1
@@ -6,7 +6,6 @@ import torch
6
7
from vllm.model_executor.layers.spec_decode_base_sampler import (
8
SpecDecodeBaseSampler)
9
-from vllm.platforms import current_platform
9
from vllm.utils import is_pin_memory_available
10
11
@@ -94,6 +93,7 @@ class AsyncMetricsCollector:
93
def maybe_collect_rejsample_metrics(
94
self, k: int) -> Optional[SpecDecodeWorkerMetrics]:
95
# currently using cuda.Event, skip for any non_cuda_alike platform
96
+ from vllm.platforms import current_platform
97
if not current_platform.is_cuda_alike():
98
return None
99
vllm/usage/usage_lib.py
+1
-1
@@ -17,7 +17,6 @@ import torch
17
18
import vllm.envs as envs
19
from vllm.connections import global_http_connection
20
-from vllm.platforms import current_platform
20
from vllm.version import __version__ as VLLM_VERSION
21
22
_config_home = envs.VLLM_CONFIG_ROOT
@@ -152,6 +151,7 @@ class UsageMessage:
151
usage_context: UsageContext,
152
extra_kvs: Dict[str, Any]) -> None:
153
# Platform information
154
+ from vllm.platforms import current_platform
155
if current_platform.is_cuda_alike():
156
device_property = torch.cuda.get_device_properties(0)
157
self.gpu_count = torch.cuda.device_count()
vllm/utils.py
+6
-2
@@ -50,7 +50,6 @@ from typing_extensions import ParamSpec, TypeIs, assert_never
50
51
import vllm.envs as envs
52
from vllm.logger import enable_trace_function_call, init_logger
53
-from vllm.platforms import current_platform
53
54
if TYPE_CHECKING:
55
from vllm.config import VllmConfig
@@ -609,6 +608,7 @@ def create_kv_caches_with_random_flash(
608
seed: int = 0,
609
device: Optional[str] = "cuda",
610
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
611
+ from vllm.platforms import current_platform
612
current_platform.seed_everything(seed)
613
614
torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
@@ -650,7 +650,7 @@ def create_kv_caches_with_random(
650
raise ValueError(
651
f"Does not support key cache of type fp8 with head_size {head_size}"
652
)
653
-
653
+ from vllm.platforms import current_platform
654
current_platform.seed_everything(seed)
655
656
torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
@@ -703,6 +703,7 @@ def print_warning_once(msg: str) -> None:
703
704
@lru_cache(maxsize=None)
705
def is_pin_memory_available() -> bool:
706
+ from vllm.platforms import current_platform
707
return current_platform.is_pin_memory_available()
708
709
@@ -713,6 +714,7 @@ class DeviceMemoryProfiler:
714
715
def current_memory_usage(self) -> float:
716
# Return the memory usage in bytes.
717
+ from vllm.platforms import current_platform
718
if current_platform.is_cuda_alike():
719
torch.cuda.reset_peak_memory_stats(self.device)
720
mem = torch.cuda.max_memory_allocated(self.device)
@@ -1066,6 +1068,7 @@ def _cuda_device_count_stateless(
1068
import torch.cuda
1069
import torch.version
1070
1071
+ from vllm.platforms import current_platform
1072
if not torch.cuda._is_compiled():
1073
return 0
1074
if current_platform.is_rocm():
@@ -1673,6 +1676,7 @@ def direct_register_custom_op(
1676
return
1677
1678
if not supports_custom_op():
1679
+ from vllm.platforms import current_platform
1680
assert not current_platform.is_cuda_alike(), (
1681
"cuda platform needs torch>=2.4 to support custom op, "
1682
"chances are you are using an old version of pytorch "
vllm/worker/model_runner_base.py
+2
-3
@@ -12,7 +12,6 @@ from torch import is_tensor
12
from vllm.config import VllmConfig
13
from vllm.logger import init_logger
14
from vllm.model_executor.layers.sampler import SamplerOutput
15
-from vllm.platforms import current_platform
15
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
16
17
if TYPE_CHECKING:
@@ -265,13 +264,13 @@ class ModelRunnerBase(ABC, Generic[T]):
264
"""
265
raise NotImplementedError
266
268
- @current_platform.inference_mode()
267
def execute_model(
268
self,
269
model_input: T,
270
kv_caches: Optional[List[torch.Tensor]],
273
- intermediate_tensors: Optional[IntermediateTensors],
271
+ intermediate_tensors: Optional[IntermediateTensors] = None,
272
num_steps: int = 1,
273
+ **kwargs,
274
) -> Optional[List[SamplerOutput]]:
275
"""
276
Execute the model on the given input.
vllm/worker/multi_step_model_runner.py
+1
@@ -544,6 +544,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
544
model_input.record_step_event(current_stream)
545
546
if get_pp_group().is_last_rank and self.is_driver_worker:
547
+ assert isinstance(output, list)
548
assert len(
549
output
550
) == 1, "MultiStepModelRunner requires single-step base_models"
vllm/worker/worker_base.py
+8
-6
@@ -11,7 +11,6 @@ from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
11
from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
13
from vllm.model_executor.layers.sampler import SamplerOutput
14
-from vllm.platforms import current_platform
14
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
15
from vllm.utils import (enable_trace_function_call_for_thread,
16
resolve_obj_by_qualname, update_environment_variables)
@@ -44,6 +43,8 @@ class WorkerBase(ABC):
43
self.prompt_adapter_config = vllm_config.prompt_adapter_config
44
self.observability_config = vllm_config.observability_config
45
self.kv_transfer_config = vllm_config.kv_transfer_config
46
+ from vllm.platforms import current_platform
47
+ self.current_platform = current_platform
48
49
@abstractmethod
50
def init_device(self) -> None:
@@ -74,17 +75,17 @@ class WorkerBase(ABC):
75
"""
76
raise NotImplementedError
77
77
- @current_platform.inference_mode()
78
def start_worker_execution_loop(self) -> None:
79
"""Execute model loop in parallel worker.
80
81
You can stop the loop by executing a driver worker with an empty output.
82
See `stop_remote_worker_execution_loop` for more details.
83
"""
84
- while True:
85
- output = self.execute_model(execute_model_req=None)
86
- if output is None:
87
- return None
84
+ with self.current_platform.inference_mode():
85
+ while True:
86
+ output = self.execute_model(execute_model_req=None)
87
+ if output is None:
88
+ return None
89
90
@abstractmethod
91
def execute_model(
@@ -352,6 +353,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
353
model_execute_time = time.perf_counter() - start_time
354
if not get_pp_group().is_last_rank:
355
# output is IntermediateTensors
356
+ assert isinstance(output, IntermediateTensors)
357
if (self.observability_config is not None
358
and self.observability_config.collect_model_execute_time):
359
output.tensors["model_execute_time"] = torch.tensor(