A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Hardware][openvino] is_openvino --> current_platform.is_openvino (#9716)
Mengqing Cao committed
Oct 26, 2024 at 18:59 UTC
5cbdccd151ef50e3fc040690248a8d86d3b93c2a
10 files changed
+69
-38
tests/kernels/test_attention_selector.py
+2
-1
@@ -30,7 +30,8 @@ def test_env(name: str, device: str, monkeypatch):
30
False)
31
assert backend.name == "ROCM_FLASH"
32
elif device == "openvino":
33
- with patch("vllm.attention.selector.is_openvino", return_value=True):
33
+ with patch("vllm.attention.selector.current_platform.is_openvino",
34
+ return_value=True):
35
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
36
False)
37
assert backend.name == "OPENVINO"
vllm/attention/selector.py
+2
-2
@@ -10,7 +10,7 @@ import vllm.envs as envs
10
from vllm.attention.backends.abstract import AttentionBackend
11
from vllm.logger import init_logger
12
from vllm.platforms import current_platform
13
-from vllm.utils import STR_BACKEND_ENV_VAR, is_hip, is_openvino
13
+from vllm.utils import STR_BACKEND_ENV_VAR, is_hip
14
15
logger = init_logger(__name__)
16
@@ -193,7 +193,7 @@ def which_attn_to_use(
193
logger.info("Cannot use %s backend on CPU.", selected_backend)
194
return _Backend.TORCH_SDPA
195
196
- if is_openvino():
196
+ if current_platform.is_openvino():
197
if selected_backend != _Backend.OPENVINO:
198
logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
199
return _Backend.OPENVINO
vllm/config.py
+2
-2
@@ -17,7 +17,7 @@ from vllm.transformers_utils.config import (ConfigFormat, get_config,
17
get_hf_image_processor_config,
18
get_hf_text_config)
19
from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20
- is_hip, is_openvino, print_warning_once)
20
+ is_hip, print_warning_once)
21
22
if TYPE_CHECKING:
23
from ray.util.placement_group import PlacementGroup
@@ -1117,7 +1117,7 @@ class DeviceConfig:
1117
self.device_type = "cuda"
1118
elif current_platform.is_neuron():
1119
self.device_type = "neuron"
1120
- elif is_openvino():
1120
+ elif current_platform.is_openvino():
1121
self.device_type = "openvino"
1122
elif current_platform.is_tpu():
1123
self.device_type = "tpu"
vllm/executor/openvino_executor.py
+7
-13
@@ -10,6 +10,7 @@ from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
from vllm.model_executor.layers.sampler import SamplerOutput
13
+from vllm.platforms import current_platform
14
from vllm.sequence import ExecuteModelRequest
15
from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
16
get_open_port, make_async)
@@ -17,14 +18,6 @@ from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
18
logger = init_logger(__name__)
19
20
20
-def is_openvino_cpu() -> bool:
21
- return "CPU" in envs.VLLM_OPENVINO_DEVICE
22
-
23
-
24
-def is_openvino_gpu() -> bool:
25
- return "GPU" in envs.VLLM_OPENVINO_DEVICE
26
-
27
-
21
class OpenVINOExecutor(ExecutorBase):
22
23
uses_ray: bool = False
@@ -32,7 +25,8 @@ class OpenVINOExecutor(ExecutorBase):
25
def _init_executor(self) -> None:
26
assert self.device_config.device_type == "openvino"
27
assert self.lora_config is None, "OpenVINO backend doesn't support LoRA"
35
- assert is_openvino_cpu() or is_openvino_gpu(), \
28
+ assert current_platform.is_openvino_cpu() or \
29
+ current_platform.is_openvino_gpu(), \
30
"OpenVINO backend supports only CPU and GPU devices"
31
32
self.ov_core = ov.Core()
@@ -163,7 +157,7 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
157
def _verify_and_get_cache_config(ov_core: ov.Core,
158
config: CacheConfig) -> CacheConfig:
159
if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8":
166
- if not is_openvino_cpu():
160
+ if not current_platform.is_openvino_cpu():
161
logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is"
162
"ignored for GPU, f16 data type will be used.")
163
config.cache_dtype = ov.Type.f16
@@ -172,7 +166,7 @@ def _verify_and_get_cache_config(ov_core: ov.Core,
166
"VLLM_OPENVINO_CPU_KV_CACHE_PRECISION env var.")
167
config.cache_dtype = ov.Type.u8
168
else:
175
- if is_openvino_cpu():
169
+ if current_platform.is_openvino_cpu():
170
ov_device = envs.VLLM_OPENVINO_DEVICE
171
inference_precision = ov_core.get_property(
172
ov_device, hints.inference_precision)
@@ -183,7 +177,7 @@ def _verify_and_get_cache_config(ov_core: ov.Core,
177
else:
178
config.cache_dtype = ov.Type.f16
179
186
- if is_openvino_cpu():
180
+ if current_platform.is_openvino_cpu():
181
if config.block_size != 32:
182
logger.info(
183
f"OpenVINO CPU optimal block size is 32, overriding currently set {config.block_size}" # noqa: G004, E501
@@ -198,7 +192,7 @@ def _verify_and_get_cache_config(ov_core: ov.Core,
192
193
kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE
194
if kv_cache_space >= 0:
201
- if kv_cache_space == 0 and is_openvino_cpu():
195
+ if kv_cache_space == 0 and current_platform.is_openvino_cpu():
196
config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
197
logger.warning(
198
"Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) "
vllm/model_executor/model_loader/openvino.py
+2
-2
@@ -12,12 +12,12 @@ from torch import nn
12
import vllm.envs as envs
13
from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
14
from vllm.config import DeviceConfig, ModelConfig
15
-from vllm.executor.openvino_executor import is_openvino_cpu
15
from vllm.logger import init_logger
16
from vllm.model_executor.layers.logits_processor import (LogitsProcessor,
17
_prune_hidden_states)
18
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
+from vllm.platforms import current_platform
21
22
logger = init_logger(__name__)
23
@@ -136,7 +136,7 @@ class OpenVINOCasualLM(nn.Module):
136
ov_device = envs.VLLM_OPENVINO_DEVICE
137
paged_attention_transformation(pt_model.model)
138
_modify_cache_parameters(pt_model.model, kv_cache_dtype,
139
- is_openvino_cpu())
139
+ current_platform.is_openvino_cpu())
140
141
ov_compiled = ov_core.compile_model(pt_model.model, ov_device)
142
self.ov_request = ov_compiled.create_infer_request()
vllm/platforms/__init__.py
+10
@@ -65,6 +65,13 @@ try:
65
except ImportError:
66
pass
67
68
+is_openvino = False
69
+try:
70
+ from importlib.metadata import version
71
+ is_openvino = "openvino" in version("vllm")
72
+except Exception:
73
+ pass
74
+
75
if is_tpu:
76
# people might install pytorch built with cuda but run on tpu
77
# so we need to check tpu first
@@ -85,6 +92,9 @@ elif is_cpu:
92
elif is_neuron:
93
from .neuron import NeuronPlatform
94
current_platform = NeuronPlatform()
95
+elif is_openvino:
96
+ from .openvino import OpenVinoPlatform
97
+ current_platform = OpenVinoPlatform()
98
else:
99
current_platform = UnspecifiedPlatform()
100
vllm/platforms/interface.py
+4
@@ -11,6 +11,7 @@ class PlatformEnum(enum.Enum):
11
XPU = enum.auto()
12
CPU = enum.auto()
13
NEURON = enum.auto()
14
+ OPENVINO = enum.auto()
15
UNSPECIFIED = enum.auto()
16
17
@@ -52,6 +53,9 @@ class Platform:
53
def is_neuron(self) -> bool:
54
return self._enum == PlatformEnum.NEURON
55
56
+ def is_openvino(self) -> bool:
57
+ return self._enum == PlatformEnum.OPENVINO
58
+
59
def is_cuda_alike(self) -> bool:
60
"""Stateless version of :func:`torch.cuda.is_available`."""
61
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
vllm/platforms/openvino.py
new
+31
@@ -0,0 +1,31 @@
1
+import torch
2
+
3
+import vllm.envs as envs
4
+from vllm.utils import print_warning_once
5
+
6
+from .interface import Platform, PlatformEnum
7
+
8
+
9
+class OpenVinoPlatform(Platform):
10
+ _enum = PlatformEnum.OPENVINO
11
+
12
+ @classmethod
13
+ def get_device_name(self, device_id: int = 0) -> str:
14
+ return "openvino"
15
+
16
+ @classmethod
17
+ def inference_mode(self):
18
+ return torch.inference_mode(mode=True)
19
+
20
+ @classmethod
21
+ def is_openvino_cpu(self) -> bool:
22
+ return "CPU" in envs.VLLM_OPENVINO_DEVICE
23
+
24
+ @classmethod
25
+ def is_openvino_gpu(self) -> bool:
26
+ return "GPU" in envs.VLLM_OPENVINO_DEVICE
27
+
28
+ @classmethod
29
+ def is_pin_memory_available(self) -> bool:
30
+ print_warning_once("Pin memory is not supported on OpenViNO.")
31
+ return False
vllm/utils.py
+1
-10
@@ -318,15 +318,6 @@ def is_hip() -> bool:
318
return torch.version.hip is not None
319
320
321
-@lru_cache(maxsize=None)
322
-def is_openvino() -> bool:
323
- from importlib.metadata import PackageNotFoundError, version
324
- try:
325
- return "openvino" in version("vllm")
326
- except PackageNotFoundError:
327
- return False
328
-
329
-
321
@lru_cache(maxsize=None)
322
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
323
"""Returns the maximum shared memory per thread block in bytes."""
@@ -757,7 +748,7 @@ def is_pin_memory_available() -> bool:
748
elif current_platform.is_neuron():
749
print_warning_once("Pin memory is not supported on Neuron.")
750
return False
760
- elif current_platform.is_cpu() or is_openvino():
751
+ elif current_platform.is_cpu() or current_platform.is_openvino():
752
return False
753
return True
754
vllm/worker/openvino_worker.py
+8
-8
@@ -13,12 +13,12 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13
from vllm.distributed import (broadcast_tensor_dict,
14
ensure_model_parallel_initialized,
15
init_distributed_environment)
16
-from vllm.executor.openvino_executor import is_openvino_cpu
16
from vllm.inputs import INPUT_REGISTRY
17
from vllm.logger import init_logger
18
from vllm.model_executor import set_random_seed
19
from vllm.model_executor.layers.sampler import SamplerOutput
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
+from vllm.platforms import current_platform
22
from vllm.sampling_params import SamplingParams
23
from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
24
from vllm.worker.openvino_model_runner import OpenVINOModelRunner
@@ -99,7 +99,7 @@ class OpenVINOCacheEngine:
99
num_blocks, self.block_size, self.num_kv_heads, self.head_size)[1:]
100
kv_cache: List[Tuple[ov.Tensor, ov.Tensor]] = []
101
102
- if is_openvino_cpu():
102
+ if current_platform.is_openvino_cpu():
103
for _ in range(self.num_layers):
104
key_blocks = ov.Tensor(self.cache_config.cache_dtype,
105
k_block_shape)
@@ -141,7 +141,7 @@ class OpenVINOCacheEngine:
141
if num_blocks == 0:
142
return swap_cache
143
144
- assert not is_openvino_cpu(), \
144
+ assert not current_platform.is_openvino_cpu(), \
145
"CPU device isn't supposed to have swap cache"
146
147
# Update key_cache shape:
@@ -285,7 +285,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
285
cache_block_size = self.get_cache_block_size_bytes()
286
kvcache_space_bytes = self.cache_config.openvino_kvcache_space_bytes
287
288
- if is_openvino_cpu():
288
+ if current_platform.is_openvino_cpu():
289
num_device_blocks = int(kvcache_space_bytes // cache_block_size)
290
num_swap_blocks = 0
291
else:
@@ -322,7 +322,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
322
num_device_blocks = num_gpu_blocks
323
num_swap_blocks = num_cpu_blocks
324
325
- if is_openvino_cpu():
325
+ if current_platform.is_openvino_cpu():
326
assert (num_swap_blocks == 0
327
), f"{type(self)} does not support swappable cache for CPU"
328
@@ -366,7 +366,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
366
assert self.kv_cache is not None
367
368
# Populate the cache to warmup the memory
369
- if is_openvino_cpu():
369
+ if current_platform.is_openvino_cpu():
370
for key_cache, value_cache in self.kv_cache:
371
key_cache.data[:] = 0
372
value_cache.data[:] = 0
@@ -414,7 +414,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
414
blocks_to_swap_in = data["blocks_to_swap_in"]
415
blocks_to_swap_out = data["blocks_to_swap_out"]
416
417
- if is_openvino_cpu():
417
+ if current_platform.is_openvino_cpu():
418
assert len(execute_model_req.blocks_to_swap_in) == 0
419
assert len(execute_model_req.blocks_to_swap_out) == 0
420
else:
@@ -466,7 +466,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
466
def profile_run(self) -> int:
467
ov_device = envs.VLLM_OPENVINO_DEVICE
468
469
- assert not is_openvino_cpu(), \
469
+ assert not current_platform.is_openvino_cpu(), \
470
"CPU device isn't supposed to use profile run."
471
472
import openvino.properties.device as device