@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fcf2e3d7fc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix] Fix OpenVINO model runner (#12750)

Harry Mellor committed Feb 5, 2025 at 06:42 UTC fcf2e3d7fcc9898b7a1b26bacea22753ab76f3a6
3 files changed +12 -12
vllm/attention/backends/openvino.py
+4
@@ -140,3 +140,7 @@ class OpenVINOAttentionMetadata:
140 # `model_executable`.
141 multi_modal_placeholder_index_maps: Optional[Dict[
142 str, MultiModalPlaceholderMap.IndexMap]]
143 +
144 + # Enable/disable KV scales calculation. This is so that we can disable the
145 + # calculation until after prefill and cuda graph capture.
146 + enable_kv_scales_calculation: bool
vllm/model_executor/model_loader/openvino.py
+5 -6
@@ -13,7 +13,7 @@ from torch import nn
13
14 import vllm.envs as envs
15 from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
16 -from vllm.config import DeviceConfig, ModelConfig
16 +from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
17 from vllm.logger import init_logger
18 from vllm.model_executor.layers.logits_processor import (LogitsProcessor,
19 _prune_hidden_states)
@@ -103,7 +103,6 @@ class OpenVINOCausalLM(nn.Module):
103 self,
104 ov_core: ov.Core,
105 model_config: ModelConfig,
106 - device_config: DeviceConfig,
106 kv_cache_dtype: ov.Type,
107 ) -> None:
108 super().__init__()
@@ -187,8 +186,7 @@ class OpenVINOCausalLM(nn.Module):
186
187
188 def get_model(
190 - model_config: ModelConfig,
191 - device_config: DeviceConfig,
189 + vllm_config: VllmConfig,
190 kv_cache_dtype: ov.Type,
191 **kwargs,
192 ) -> torch.nn.Module:
@@ -201,5 +199,6 @@ def get_model(
199 "be added in the future. If this is important to you, "
200 "please open an issue on github.")
201
204 - return OpenVINOCausalLM(ov_core, model_config, device_config,
205 - kv_cache_dtype)
202 + with set_current_vllm_config(vllm_config):
203 + return OpenVINOCausalLM(ov_core, vllm_config.model_config,
204 + kv_cache_dtype)
vllm/worker/openvino_model_runner.py
+3 -6
@@ -54,15 +54,13 @@ class OpenVINOModelRunner(ModelRunnerBase):
54 ):
55 self.ov_core = ov_core
56 ModelRunnerBase.__init__(self, vllm_config=vllm_config)
57 - cache_config = self.cache_config
58 - model_config = self.model_config
57 self.is_driver_worker = is_driver_worker
58
59 self.device = self.device_config.device
60
61 self.kv_cache_dtype = kv_cache_dtype
64 - self.sliding_window = model_config.get_sliding_window()
65 - self.block_size = cache_config.block_size
62 + self.sliding_window = self.model_config.get_sliding_window()
63 + self.block_size = self.cache_config.block_size
64
65 self.attn_backend = get_attn_backend(
66 self.model_config.get_head_size(),
@@ -81,8 +79,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
79 self.model: nn.Module # Set after init_Model
80
81 def load_model(self) -> None:
84 - self.model = get_model(model_config=self.model_config,
85 - device_config=self.device_config,
82 + self.model = get_model(vllm_config=self.vllm_config,
83 kv_cache_dtype=self.kv_cache_dtype,
84 ov_core=self.ov_core)
85