A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix] Fix OpenVINO model runner (#12750)
Harry Mellor committed
Feb 5, 2025 at 06:42 UTC
fcf2e3d7fcc9898b7a1b26bacea22753ab76f3a6
3 files changed
+12
-12
vllm/attention/backends/openvino.py
+4
@@ -140,3 +140,7 @@ class OpenVINOAttentionMetadata:
140
# `model_executable`.
141
multi_modal_placeholder_index_maps: Optional[Dict[
142
str, MultiModalPlaceholderMap.IndexMap]]
143
+
144
+ # Enable/disable KV scales calculation. This is so that we can disable the
145
+ # calculation until after prefill and cuda graph capture.
146
+ enable_kv_scales_calculation: bool
vllm/model_executor/model_loader/openvino.py
+5
-6
@@ -13,7 +13,7 @@ from torch import nn
13
14
import vllm.envs as envs
15
from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
16
-from vllm.config import DeviceConfig, ModelConfig
16
+from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
17
from vllm.logger import init_logger
18
from vllm.model_executor.layers.logits_processor import (LogitsProcessor,
19
_prune_hidden_states)
@@ -103,7 +103,6 @@ class OpenVINOCausalLM(nn.Module):
103
self,
104
ov_core: ov.Core,
105
model_config: ModelConfig,
106
- device_config: DeviceConfig,
106
kv_cache_dtype: ov.Type,
107
) -> None:
108
super().__init__()
@@ -187,8 +186,7 @@ class OpenVINOCausalLM(nn.Module):
186
187
188
def get_model(
190
- model_config: ModelConfig,
191
- device_config: DeviceConfig,
189
+ vllm_config: VllmConfig,
190
kv_cache_dtype: ov.Type,
191
**kwargs,
192
) -> torch.nn.Module:
@@ -201,5 +199,6 @@ def get_model(
199
"be added in the future. If this is important to you, "
200
"please open an issue on github.")
201
204
- return OpenVINOCausalLM(ov_core, model_config, device_config,
205
- kv_cache_dtype)
202
+ with set_current_vllm_config(vllm_config):
203
+ return OpenVINOCausalLM(ov_core, vllm_config.model_config,
204
+ kv_cache_dtype)
vllm/worker/openvino_model_runner.py
+3
-6
@@ -54,15 +54,13 @@ class OpenVINOModelRunner(ModelRunnerBase):
54
):
55
self.ov_core = ov_core
56
ModelRunnerBase.__init__(self, vllm_config=vllm_config)
57
- cache_config = self.cache_config
58
- model_config = self.model_config
57
self.is_driver_worker = is_driver_worker
58
59
self.device = self.device_config.device
60
61
self.kv_cache_dtype = kv_cache_dtype
64
- self.sliding_window = model_config.get_sliding_window()
65
- self.block_size = cache_config.block_size
62
+ self.sliding_window = self.model_config.get_sliding_window()
63
+ self.block_size = self.cache_config.block_size
64
65
self.attn_backend = get_attn_backend(
66
self.model_config.get_head_size(),
@@ -81,8 +79,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
79
self.model: nn.Module # Set after init_Model
80
81
def load_model(self) -> None:
84
- self.model = get_model(model_config=self.model_config,
85
- device_config=self.device_config,
82
+ self.model = get_model(vllm_config=self.vllm_config,
83
kv_cache_dtype=self.kv_cache_dtype,
84
ov_core=self.ov_core)
85