@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5cbdccd151
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Hardware][openvino] is_openvino --> current_platform.is_openvino (#9716)

Mengqing Cao committed Oct 26, 2024 at 18:59 UTC 5cbdccd151ef50e3fc040690248a8d86d3b93c2a
10 files changed +69 -38
tests/kernels/test_attention_selector.py
+2 -1
@@ -30,7 +30,8 @@ def test_env(name: str, device: str, monkeypatch):
30 False)
31 assert backend.name == "ROCM_FLASH"
32 elif device == "openvino":
33 - with patch("vllm.attention.selector.is_openvino", return_value=True):
33 + with patch("vllm.attention.selector.current_platform.is_openvino",
34 + return_value=True):
35 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
36 False)
37 assert backend.name == "OPENVINO"
vllm/attention/selector.py
+2 -2
@@ -10,7 +10,7 @@ import vllm.envs as envs
10 from vllm.attention.backends.abstract import AttentionBackend
11 from vllm.logger import init_logger
12 from vllm.platforms import current_platform
13 -from vllm.utils import STR_BACKEND_ENV_VAR, is_hip, is_openvino
13 +from vllm.utils import STR_BACKEND_ENV_VAR, is_hip
14
15 logger = init_logger(__name__)
16
@@ -193,7 +193,7 @@ def which_attn_to_use(
193 logger.info("Cannot use %s backend on CPU.", selected_backend)
194 return _Backend.TORCH_SDPA
195
196 - if is_openvino():
196 + if current_platform.is_openvino():
197 if selected_backend != _Backend.OPENVINO:
198 logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
199 return _Backend.OPENVINO
vllm/config.py
+2 -2
@@ -17,7 +17,7 @@ from vllm.transformers_utils.config import (ConfigFormat, get_config,
17 get_hf_image_processor_config,
18 get_hf_text_config)
19 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20 - is_hip, is_openvino, print_warning_once)
20 + is_hip, print_warning_once)
21
22 if TYPE_CHECKING:
23 from ray.util.placement_group import PlacementGroup
@@ -1117,7 +1117,7 @@ class DeviceConfig:
1117 self.device_type = "cuda"
1118 elif current_platform.is_neuron():
1119 self.device_type = "neuron"
1120 - elif is_openvino():
1120 + elif current_platform.is_openvino():
1121 self.device_type = "openvino"
1122 elif current_platform.is_tpu():
1123 self.device_type = "tpu"
vllm/executor/openvino_executor.py
+7 -13
@@ -10,6 +10,7 @@ from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 from vllm.model_executor.layers.sampler import SamplerOutput
13 +from vllm.platforms import current_platform
14 from vllm.sequence import ExecuteModelRequest
15 from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
16 get_open_port, make_async)
@@ -17,14 +18,6 @@ from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
18 logger = init_logger(__name__)
19
20
20 -def is_openvino_cpu() -> bool:
21 - return "CPU" in envs.VLLM_OPENVINO_DEVICE
22 -
23 -
24 -def is_openvino_gpu() -> bool:
25 - return "GPU" in envs.VLLM_OPENVINO_DEVICE
26 -
27 -
21 class OpenVINOExecutor(ExecutorBase):
22
23 uses_ray: bool = False
@@ -32,7 +25,8 @@ class OpenVINOExecutor(ExecutorBase):
25 def _init_executor(self) -> None:
26 assert self.device_config.device_type == "openvino"
27 assert self.lora_config is None, "OpenVINO backend doesn't support LoRA"
35 - assert is_openvino_cpu() or is_openvino_gpu(), \
28 + assert current_platform.is_openvino_cpu() or \
29 + current_platform.is_openvino_gpu(), \
30 "OpenVINO backend supports only CPU and GPU devices"
31
32 self.ov_core = ov.Core()
@@ -163,7 +157,7 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
157 def _verify_and_get_cache_config(ov_core: ov.Core,
158 config: CacheConfig) -> CacheConfig:
159 if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8":
166 - if not is_openvino_cpu():
160 + if not current_platform.is_openvino_cpu():
161 logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is"
162 "ignored for GPU, f16 data type will be used.")
163 config.cache_dtype = ov.Type.f16
@@ -172,7 +166,7 @@ def _verify_and_get_cache_config(ov_core: ov.Core,
166 "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION env var.")
167 config.cache_dtype = ov.Type.u8
168 else:
175 - if is_openvino_cpu():
169 + if current_platform.is_openvino_cpu():
170 ov_device = envs.VLLM_OPENVINO_DEVICE
171 inference_precision = ov_core.get_property(
172 ov_device, hints.inference_precision)
@@ -183,7 +177,7 @@ def _verify_and_get_cache_config(ov_core: ov.Core,
177 else:
178 config.cache_dtype = ov.Type.f16
179
186 - if is_openvino_cpu():
180 + if current_platform.is_openvino_cpu():
181 if config.block_size != 32:
182 logger.info(
183 f"OpenVINO CPU optimal block size is 32, overriding currently set {config.block_size}" # noqa: G004, E501
@@ -198,7 +192,7 @@ def _verify_and_get_cache_config(ov_core: ov.Core,
192
193 kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE
194 if kv_cache_space >= 0:
201 - if kv_cache_space == 0 and is_openvino_cpu():
195 + if kv_cache_space == 0 and current_platform.is_openvino_cpu():
196 config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
197 logger.warning(
198 "Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) "
vllm/model_executor/model_loader/openvino.py
+2 -2
@@ -12,12 +12,12 @@ from torch import nn
12 import vllm.envs as envs
13 from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
14 from vllm.config import DeviceConfig, ModelConfig
15 -from vllm.executor.openvino_executor import is_openvino_cpu
15 from vllm.logger import init_logger
16 from vllm.model_executor.layers.logits_processor import (LogitsProcessor,
17 _prune_hidden_states)
18 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 +from vllm.platforms import current_platform
21
22 logger = init_logger(__name__)
23
@@ -136,7 +136,7 @@ class OpenVINOCasualLM(nn.Module):
136 ov_device = envs.VLLM_OPENVINO_DEVICE
137 paged_attention_transformation(pt_model.model)
138 _modify_cache_parameters(pt_model.model, kv_cache_dtype,
139 - is_openvino_cpu())
139 + current_platform.is_openvino_cpu())
140
141 ov_compiled = ov_core.compile_model(pt_model.model, ov_device)
142 self.ov_request = ov_compiled.create_infer_request()
vllm/platforms/__init__.py
+10
@@ -65,6 +65,13 @@ try:
65 except ImportError:
66 pass
67
68 +is_openvino = False
69 +try:
70 + from importlib.metadata import version
71 + is_openvino = "openvino" in version("vllm")
72 +except Exception:
73 + pass
74 +
75 if is_tpu:
76 # people might install pytorch built with cuda but run on tpu
77 # so we need to check tpu first
@@ -85,6 +92,9 @@ elif is_cpu:
92 elif is_neuron:
93 from .neuron import NeuronPlatform
94 current_platform = NeuronPlatform()
95 +elif is_openvino:
96 + from .openvino import OpenVinoPlatform
97 + current_platform = OpenVinoPlatform()
98 else:
99 current_platform = UnspecifiedPlatform()
100
vllm/platforms/interface.py
+4
@@ -11,6 +11,7 @@ class PlatformEnum(enum.Enum):
11 XPU = enum.auto()
12 CPU = enum.auto()
13 NEURON = enum.auto()
14 + OPENVINO = enum.auto()
15 UNSPECIFIED = enum.auto()
16
17
@@ -52,6 +53,9 @@ class Platform:
53 def is_neuron(self) -> bool:
54 return self._enum == PlatformEnum.NEURON
55
56 + def is_openvino(self) -> bool:
57 + return self._enum == PlatformEnum.OPENVINO
58 +
59 def is_cuda_alike(self) -> bool:
60 """Stateless version of :func:`torch.cuda.is_available`."""
61 return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
vllm/platforms/openvino.py new
+31
@@ -0,0 +1,31 @@
1 +import torch
2 +
3 +import vllm.envs as envs
4 +from vllm.utils import print_warning_once
5 +
6 +from .interface import Platform, PlatformEnum
7 +
8 +
9 +class OpenVinoPlatform(Platform):
10 + _enum = PlatformEnum.OPENVINO
11 +
12 + @classmethod
13 + def get_device_name(self, device_id: int = 0) -> str:
14 + return "openvino"
15 +
16 + @classmethod
17 + def inference_mode(self):
18 + return torch.inference_mode(mode=True)
19 +
20 + @classmethod
21 + def is_openvino_cpu(self) -> bool:
22 + return "CPU" in envs.VLLM_OPENVINO_DEVICE
23 +
24 + @classmethod
25 + def is_openvino_gpu(self) -> bool:
26 + return "GPU" in envs.VLLM_OPENVINO_DEVICE
27 +
28 + @classmethod
29 + def is_pin_memory_available(self) -> bool:
30 + print_warning_once("Pin memory is not supported on OpenViNO.")
31 + return False
vllm/utils.py
+1 -10
@@ -318,15 +318,6 @@ def is_hip() -> bool:
318 return torch.version.hip is not None
319
320
321 -@lru_cache(maxsize=None)
322 -def is_openvino() -> bool:
323 - from importlib.metadata import PackageNotFoundError, version
324 - try:
325 - return "openvino" in version("vllm")
326 - except PackageNotFoundError:
327 - return False
328 -
329 -
321 @lru_cache(maxsize=None)
322 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
323 """Returns the maximum shared memory per thread block in bytes."""
@@ -757,7 +748,7 @@ def is_pin_memory_available() -> bool:
748 elif current_platform.is_neuron():
749 print_warning_once("Pin memory is not supported on Neuron.")
750 return False
760 - elif current_platform.is_cpu() or is_openvino():
751 + elif current_platform.is_cpu() or current_platform.is_openvino():
752 return False
753 return True
754
vllm/worker/openvino_worker.py
+8 -8
@@ -13,12 +13,12 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13 from vllm.distributed import (broadcast_tensor_dict,
14 ensure_model_parallel_initialized,
15 init_distributed_environment)
16 -from vllm.executor.openvino_executor import is_openvino_cpu
16 from vllm.inputs import INPUT_REGISTRY
17 from vllm.logger import init_logger
18 from vllm.model_executor import set_random_seed
19 from vllm.model_executor.layers.sampler import SamplerOutput
20 from vllm.multimodal import MULTIMODAL_REGISTRY
21 +from vllm.platforms import current_platform
22 from vllm.sampling_params import SamplingParams
23 from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
24 from vllm.worker.openvino_model_runner import OpenVINOModelRunner
@@ -99,7 +99,7 @@ class OpenVINOCacheEngine:
99 num_blocks, self.block_size, self.num_kv_heads, self.head_size)[1:]
100 kv_cache: List[Tuple[ov.Tensor, ov.Tensor]] = []
101
102 - if is_openvino_cpu():
102 + if current_platform.is_openvino_cpu():
103 for _ in range(self.num_layers):
104 key_blocks = ov.Tensor(self.cache_config.cache_dtype,
105 k_block_shape)
@@ -141,7 +141,7 @@ class OpenVINOCacheEngine:
141 if num_blocks == 0:
142 return swap_cache
143
144 - assert not is_openvino_cpu(), \
144 + assert not current_platform.is_openvino_cpu(), \
145 "CPU device isn't supposed to have swap cache"
146
147 # Update key_cache shape:
@@ -285,7 +285,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
285 cache_block_size = self.get_cache_block_size_bytes()
286 kvcache_space_bytes = self.cache_config.openvino_kvcache_space_bytes
287
288 - if is_openvino_cpu():
288 + if current_platform.is_openvino_cpu():
289 num_device_blocks = int(kvcache_space_bytes // cache_block_size)
290 num_swap_blocks = 0
291 else:
@@ -322,7 +322,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
322 num_device_blocks = num_gpu_blocks
323 num_swap_blocks = num_cpu_blocks
324
325 - if is_openvino_cpu():
325 + if current_platform.is_openvino_cpu():
326 assert (num_swap_blocks == 0
327 ), f"{type(self)} does not support swappable cache for CPU"
328
@@ -366,7 +366,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
366 assert self.kv_cache is not None
367
368 # Populate the cache to warmup the memory
369 - if is_openvino_cpu():
369 + if current_platform.is_openvino_cpu():
370 for key_cache, value_cache in self.kv_cache:
371 key_cache.data[:] = 0
372 value_cache.data[:] = 0
@@ -414,7 +414,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
414 blocks_to_swap_in = data["blocks_to_swap_in"]
415 blocks_to_swap_out = data["blocks_to_swap_out"]
416
417 - if is_openvino_cpu():
417 + if current_platform.is_openvino_cpu():
418 assert len(execute_model_req.blocks_to_swap_in) == 0
419 assert len(execute_model_req.blocks_to_swap_out) == 0
420 else:
@@ -466,7 +466,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
466 def profile_run(self) -> int:
467 ov_device = envs.VLLM_OPENVINO_DEVICE
468
469 - assert not is_openvino_cpu(), \
469 + assert not current_platform.is_openvino_cpu(), \
470 "CPU device isn't supposed to use profile run."
471
472 import openvino.properties.device as device