A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Platforms] Refactor openvino code (#10573)
Signed-off-by: statelesshz <hzji210@gmail.com>
JiHuazhong committed
Nov 23, 2024 at 14:23 UTC
86a44fb8967f757b0701aaa33aeaa8a431714a27
2 files changed
+72
-78
vllm/executor/openvino_executor.py
+3
-78
@@ -1,19 +1,16 @@
1
from typing import List, Set, Tuple
2
3
import openvino as ov
4
-import openvino.properties.hint as hints
5
-import torch
4
5
import vllm.envs as envs
8
-from vllm.config import CacheConfig, ModelConfig
6
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7
from vllm.logger import init_logger
8
from vllm.lora.request import LoRARequest
9
from vllm.model_executor.layers.sampler import SamplerOutput
10
from vllm.platforms import current_platform
11
from vllm.sequence import ExecuteModelRequest
15
-from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
16
- get_open_port, make_async)
12
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13
+ make_async)
14
from vllm.worker.worker_base import WorkerWrapperBase
15
16
logger = init_logger(__name__)
@@ -30,11 +27,6 @@ class OpenVINOExecutor(ExecutorBase):
27
current_platform.is_openvino_gpu(), \
28
"OpenVINO backend supports only CPU and GPU devices"
29
33
- self.ov_core = ov.Core()
34
- self.model_config = _verify_and_get_model_config(self.model_config)
35
- self.cache_config = _verify_and_get_cache_config(
36
- self.ov_core, self.cache_config)
37
-
30
# Instantiate the worker and load the model to CPU.
31
self._init_worker()
32
@@ -45,7 +37,7 @@ class OpenVINOExecutor(ExecutorBase):
37
distributed_init_method = get_distributed_init_method(
38
get_ip(), get_open_port())
39
self.driver_worker = wrapper.init_worker(
48
- ov_core=self.ov_core,
40
+ ov_core=ov.Core(),
41
vllm_config=self.vllm_config,
42
local_rank=0,
43
rank=0,
@@ -130,70 +122,3 @@ class OpenVINOExecutorAsync(OpenVINOExecutor, ExecutorAsyncBase):
122
# OpenVINOExecutor will always be healthy as long as
123
# it's running.
124
return
133
-
134
-
135
-def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
136
- if config.dtype != torch.float32:
137
- logger.warning(
138
- f"Only float32 dtype is supported on OpenVINO, casting from {config.dtype}." # noqa: G004, E501
139
- )
140
- config.dtype = torch.float32
141
- if not config.enforce_eager:
142
- logger.warning(
143
- "CUDA graph is not supported on OpenVINO backend, fallback to the "
144
- "eager mode.")
145
- config.enforce_eager = True
146
- return config
147
-
148
-
149
-def _verify_and_get_cache_config(ov_core: ov.Core,
150
- config: CacheConfig) -> CacheConfig:
151
- if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8":
152
- if not current_platform.is_openvino_cpu():
153
- logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is"
154
- "ignored for GPU, f16 data type will be used.")
155
- config.cache_dtype = ov.Type.f16
156
- else:
157
- logger.info("KV cache type is overridden to u8 via "
158
- "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION env var.")
159
- config.cache_dtype = ov.Type.u8
160
- else:
161
- if current_platform.is_openvino_cpu():
162
- ov_device = envs.VLLM_OPENVINO_DEVICE
163
- inference_precision = ov_core.get_property(
164
- ov_device, hints.inference_precision)
165
- if inference_precision == ov.Type.bf16:
166
- config.cache_dtype = ov.Type.bf16
167
- else:
168
- config.cache_dtype = ov.Type.f16
169
- else:
170
- config.cache_dtype = ov.Type.f16
171
-
172
- if current_platform.is_openvino_cpu():
173
- if config.block_size != 32:
174
- logger.info(
175
- f"OpenVINO CPU optimal block size is 32, overriding currently set {config.block_size}" # noqa: G004, E501
176
- )
177
- config.block_size = 32
178
- else:
179
- if config.block_size != 16:
180
- logger.info(
181
- f"OpenVINO GPU optimal block size is 16, overriding currently set {config.block_size}" # noqa: G004, E501
182
- )
183
- config.block_size = 16
184
-
185
- kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE
186
- if kv_cache_space >= 0:
187
- if kv_cache_space == 0 and current_platform.is_openvino_cpu():
188
- config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
189
- logger.warning(
190
- "Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) "
191
- "for OpenVINO backend is not set, using 4 by default.")
192
- else:
193
- config.openvino_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore
194
- else:
195
- raise RuntimeError(
196
- "Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE"
197
- f" {kv_cache_space}, expect a positive integer value.")
198
-
199
- return config
vllm/platforms/openvino.py
+69
@@ -1,5 +1,7 @@
1
from typing import TYPE_CHECKING
2
3
+import openvino as ov
4
+import openvino.properties.hint as hints
5
import torch
6
7
import vllm.envs as envs
@@ -49,6 +51,8 @@ class OpenVinoPlatform(Platform):
51
52
@classmethod
53
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
54
+ from vllm.utils import GiB_bytes
55
+
56
parallel_config = vllm_config.parallel_config
57
assert (
58
parallel_config.world_size == 1
@@ -57,3 +61,68 @@ class OpenVinoPlatform(Platform):
61
if parallel_config.worker_cls == "auto":
62
parallel_config.worker_cls = \
63
"vllm.worker.openvino_worker.OpenVINOWorker"
64
+
65
+ # check and update model config
66
+ model_config = vllm_config.model_config
67
+ if model_config.dtype != torch.float32:
68
+ logger.warning(
69
+ f"Only float32 dtype is supported on OpenVINO, casting from {model_config.dtype}." # noqa: G004, E501
70
+ )
71
+ model_config.dtype = torch.float32
72
+ if not model_config.enforce_eager:
73
+ logger.warning(
74
+ "CUDA graph is not supported on OpenVINO backend, fallback to "
75
+ "the eager mode.")
76
+ model_config.enforce_eager = True
77
+
78
+ # check and update cache config
79
+ ov_core = ov.Core()
80
+ cache_config = vllm_config.cache_config
81
+ if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8":
82
+ if not OpenVinoPlatform.is_openvino_cpu():
83
+ logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is"
84
+ "ignored for GPU, f16 data type will be used.")
85
+ cache_config.cache_dtype = ov.Type.f16
86
+ else:
87
+ logger.info("KV cache type is overridden to u8 via "
88
+ "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION env var.")
89
+ cache_config.cache_dtype = ov.Type.u8
90
+ else:
91
+ if OpenVinoPlatform.is_openvino_cpu():
92
+ ov_device = envs.VLLM_OPENVINO_DEVICE
93
+ inference_precision = ov_core.get_property(
94
+ ov_device, hints.inference_precision)
95
+ if inference_precision == ov.Type.bf16:
96
+ cache_config.cache_dtype = ov.Type.bf16
97
+ else:
98
+ cache_config.cache_dtype = ov.Type.f16
99
+ else:
100
+ cache_config.cache_dtype = ov.Type.f16
101
+
102
+ if OpenVinoPlatform.is_openvino_cpu():
103
+ if cache_config.block_size != 32:
104
+ logger.info(
105
+ f"OpenVINO CPU optimal block size is 32, overriding currently set {cache_config.block_size}" # noqa: G004, E501
106
+ )
107
+ cache_config.block_size = 32
108
+ else:
109
+ if cache_config.block_size != 16:
110
+ logger.info(
111
+ f"OpenVINO GPU optimal block size is 16, overriding currently set {cache_config.block_size}" # noqa: G004, E501
112
+ )
113
+ cache_config.block_size = 16
114
+
115
+ kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE
116
+ if kv_cache_space >= 0:
117
+ if kv_cache_space == 0 and OpenVinoPlatform.is_openvino_cpu():
118
+ cache_config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
119
+ logger.warning(
120
+ "Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) "
121
+ "for OpenVINO backend is not set, using 4 by default.")
122
+ else:
123
+ cache_config.openvino_kvcache_space_bytes = ( # type: ignore
124
+ kv_cache_space * GiB_bytes)
125
+ else:
126
+ raise RuntimeError(
127
+ "Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE"
128
+ f" {kv_cache_space}, expect a positive integer value.")