@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 86a44fb896
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Platforms] Refactor openvino code (#10573)

Signed-off-by: statelesshz <hzji210@gmail.com>

JiHuazhong committed Nov 23, 2024 at 14:23 UTC 86a44fb8967f757b0701aaa33aeaa8a431714a27
2 files changed +72 -78
vllm/executor/openvino_executor.py
+3 -78
@@ -1,19 +1,16 @@
1 from typing import List, Set, Tuple
2
3 import openvino as ov
4 -import openvino.properties.hint as hints
5 -import torch
4
5 import vllm.envs as envs
8 -from vllm.config import CacheConfig, ModelConfig
6 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7 from vllm.logger import init_logger
8 from vllm.lora.request import LoRARequest
9 from vllm.model_executor.layers.sampler import SamplerOutput
10 from vllm.platforms import current_platform
11 from vllm.sequence import ExecuteModelRequest
15 -from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
16 - get_open_port, make_async)
12 +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13 + make_async)
14 from vllm.worker.worker_base import WorkerWrapperBase
15
16 logger = init_logger(__name__)
@@ -30,11 +27,6 @@ class OpenVINOExecutor(ExecutorBase):
27 current_platform.is_openvino_gpu(), \
28 "OpenVINO backend supports only CPU and GPU devices"
29
33 - self.ov_core = ov.Core()
34 - self.model_config = _verify_and_get_model_config(self.model_config)
35 - self.cache_config = _verify_and_get_cache_config(
36 - self.ov_core, self.cache_config)
37 -
30 # Instantiate the worker and load the model to CPU.
31 self._init_worker()
32
@@ -45,7 +37,7 @@ class OpenVINOExecutor(ExecutorBase):
37 distributed_init_method = get_distributed_init_method(
38 get_ip(), get_open_port())
39 self.driver_worker = wrapper.init_worker(
48 - ov_core=self.ov_core,
40 + ov_core=ov.Core(),
41 vllm_config=self.vllm_config,
42 local_rank=0,
43 rank=0,
@@ -130,70 +122,3 @@ class OpenVINOExecutorAsync(OpenVINOExecutor, ExecutorAsyncBase):
122 # OpenVINOExecutor will always be healthy as long as
123 # it's running.
124 return
133 -
134 -
135 -def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
136 - if config.dtype != torch.float32:
137 - logger.warning(
138 - f"Only float32 dtype is supported on OpenVINO, casting from {config.dtype}." # noqa: G004, E501
139 - )
140 - config.dtype = torch.float32
141 - if not config.enforce_eager:
142 - logger.warning(
143 - "CUDA graph is not supported on OpenVINO backend, fallback to the "
144 - "eager mode.")
145 - config.enforce_eager = True
146 - return config
147 -
148 -
149 -def _verify_and_get_cache_config(ov_core: ov.Core,
150 - config: CacheConfig) -> CacheConfig:
151 - if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8":
152 - if not current_platform.is_openvino_cpu():
153 - logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is"
154 - "ignored for GPU, f16 data type will be used.")
155 - config.cache_dtype = ov.Type.f16
156 - else:
157 - logger.info("KV cache type is overridden to u8 via "
158 - "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION env var.")
159 - config.cache_dtype = ov.Type.u8
160 - else:
161 - if current_platform.is_openvino_cpu():
162 - ov_device = envs.VLLM_OPENVINO_DEVICE
163 - inference_precision = ov_core.get_property(
164 - ov_device, hints.inference_precision)
165 - if inference_precision == ov.Type.bf16:
166 - config.cache_dtype = ov.Type.bf16
167 - else:
168 - config.cache_dtype = ov.Type.f16
169 - else:
170 - config.cache_dtype = ov.Type.f16
171 -
172 - if current_platform.is_openvino_cpu():
173 - if config.block_size != 32:
174 - logger.info(
175 - f"OpenVINO CPU optimal block size is 32, overriding currently set {config.block_size}" # noqa: G004, E501
176 - )
177 - config.block_size = 32
178 - else:
179 - if config.block_size != 16:
180 - logger.info(
181 - f"OpenVINO GPU optimal block size is 16, overriding currently set {config.block_size}" # noqa: G004, E501
182 - )
183 - config.block_size = 16
184 -
185 - kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE
186 - if kv_cache_space >= 0:
187 - if kv_cache_space == 0 and current_platform.is_openvino_cpu():
188 - config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
189 - logger.warning(
190 - "Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) "
191 - "for OpenVINO backend is not set, using 4 by default.")
192 - else:
193 - config.openvino_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore
194 - else:
195 - raise RuntimeError(
196 - "Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE"
197 - f" {kv_cache_space}, expect a positive integer value.")
198 -
199 - return config
vllm/platforms/openvino.py
+69
@@ -1,5 +1,7 @@
1 from typing import TYPE_CHECKING
2
3 +import openvino as ov
4 +import openvino.properties.hint as hints
5 import torch
6
7 import vllm.envs as envs
@@ -49,6 +51,8 @@ class OpenVinoPlatform(Platform):
51
52 @classmethod
53 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
54 + from vllm.utils import GiB_bytes
55 +
56 parallel_config = vllm_config.parallel_config
57 assert (
58 parallel_config.world_size == 1
@@ -57,3 +61,68 @@ class OpenVinoPlatform(Platform):
61 if parallel_config.worker_cls == "auto":
62 parallel_config.worker_cls = \
63 "vllm.worker.openvino_worker.OpenVINOWorker"
64 +
65 + # check and update model config
66 + model_config = vllm_config.model_config
67 + if model_config.dtype != torch.float32:
68 + logger.warning(
69 + f"Only float32 dtype is supported on OpenVINO, casting from {model_config.dtype}." # noqa: G004, E501
70 + )
71 + model_config.dtype = torch.float32
72 + if not model_config.enforce_eager:
73 + logger.warning(
74 + "CUDA graph is not supported on OpenVINO backend, fallback to "
75 + "the eager mode.")
76 + model_config.enforce_eager = True
77 +
78 + # check and update cache config
79 + ov_core = ov.Core()
80 + cache_config = vllm_config.cache_config
81 + if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8":
82 + if not OpenVinoPlatform.is_openvino_cpu():
83 + logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is"
84 + "ignored for GPU, f16 data type will be used.")
85 + cache_config.cache_dtype = ov.Type.f16
86 + else:
87 + logger.info("KV cache type is overridden to u8 via "
88 + "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION env var.")
89 + cache_config.cache_dtype = ov.Type.u8
90 + else:
91 + if OpenVinoPlatform.is_openvino_cpu():
92 + ov_device = envs.VLLM_OPENVINO_DEVICE
93 + inference_precision = ov_core.get_property(
94 + ov_device, hints.inference_precision)
95 + if inference_precision == ov.Type.bf16:
96 + cache_config.cache_dtype = ov.Type.bf16
97 + else:
98 + cache_config.cache_dtype = ov.Type.f16
99 + else:
100 + cache_config.cache_dtype = ov.Type.f16
101 +
102 + if OpenVinoPlatform.is_openvino_cpu():
103 + if cache_config.block_size != 32:
104 + logger.info(
105 + f"OpenVINO CPU optimal block size is 32, overriding currently set {cache_config.block_size}" # noqa: G004, E501
106 + )
107 + cache_config.block_size = 32
108 + else:
109 + if cache_config.block_size != 16:
110 + logger.info(
111 + f"OpenVINO GPU optimal block size is 16, overriding currently set {cache_config.block_size}" # noqa: G004, E501
112 + )
113 + cache_config.block_size = 16
114 +
115 + kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE
116 + if kv_cache_space >= 0:
117 + if kv_cache_space == 0 and OpenVinoPlatform.is_openvino_cpu():
118 + cache_config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
119 + logger.warning(
120 + "Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) "
121 + "for OpenVINO backend is not set, using 4 by default.")
122 + else:
123 + cache_config.openvino_kvcache_space_bytes = ( # type: ignore
124 + kv_cache_space * GiB_bytes)
125 + else:
126 + raise RuntimeError(
127 + "Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE"
128 + f" {kv_cache_space}, expect a positive integer value.")