@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b877031d80

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 18:30 UTC

Remove openvino support in favor of external plugin (#15339)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Mar 22, 2025 at 17:06 UTC b877031d806e3d9ebc834e0191ae64de40c4ddc2

20 files changed +8 -1789

.buildkite/run-openvino-test.sh deleted

-16

@@ -1,16 +0,0 @@

1 -#!/bin/bash

2 -

3 -# This script build the OpenVINO docker image and run the offline inference inside the container.

4 -# It serves a sanity check for compilation and basic model usage.

5 -set -ex

6 -

7 -# Try building the docker image

8 -docker build -t openvino-test -f Dockerfile.openvino .

9 -

10 -# Setup cleanup

11 -remove_docker_container() { docker rm -f openvino-test || true; }

12 -trap remove_docker_container EXIT

13 -remove_docker_container

14 -

15 -# Run the image and launch offline inference

16 -docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/examples/offline_inference/basic/generate.py --model facebook/opt-125m

Dockerfile.openvino deleted

-29

@@ -1,29 +0,0 @@

1 -# The vLLM Dockerfile is used to construct vLLM image that can be directly used

2 -# to run the OpenAI compatible server.

3 -

4 -FROM ubuntu:22.04 AS dev

5 -

6 -RUN apt-get update -y && \

7 - apt-get install -y \

8 - git python3-pip \

9 - ffmpeg libsm6 libxext6 libgl1

10 -WORKDIR /workspace

11 -

12 -COPY . .

13 -ARG GIT_REPO_CHECK=0

14 -RUN --mount=type=bind,source=.git,target=.git \

15 - if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi

16 -

17 -RUN python3 -m pip install -U pip

18 -# install build requirements

19 -RUN PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" python3 -m pip install -r /workspace/requirements/build.txt

20 -# build vLLM with OpenVINO backend

21 -RUN PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" VLLM_TARGET_DEVICE="openvino" python3 -m pip install /workspace

22 -

23 -COPY examples/ /workspace/examples

24 -COPY benchmarks/ /workspace/benchmarks

25 -

26 -# install development dependencies (for testing)

27 -RUN python3 -m pip install -e tests/vllm_test_utils

28 -

29 -CMD ["/bin/bash"]

docs/source/getting_started/installation.md

-1

@@ -26,4 +26,3 @@ installation/ai_accelerator

26 - Google TPU

27 - Intel Gaudi

28 - AWS Neuron

29 - - OpenVINO

docs/source/getting_started/installation/ai_accelerator.md

-77

@@ -36,16 +36,6 @@ vLLM is a Python library that supports the following AI accelerators. Select you

37 ::::

39 -::::{tab-item} OpenVINO

40 -:sync: openvino

41 -

42 -:::{include} ai_accelerator/openvino.inc.md

43 -:start-after: "# Installation"

44 -:end-before: "## Requirements"

45 -:::

46 -

47 -::::

48 -

39 :::::

41 ## Requirements

@@ -83,16 +73,6 @@ vLLM is a Python library that supports the following AI accelerators. Select you

74 ::::

86 -::::{tab-item} OpenVINO

87 -:sync: openvino

88 -

89 -:::{include} ai_accelerator/openvino.inc.md

90 -:start-after: "## Requirements"

91 -:end-before: "## Set up using Python"

92 -:::

93 -

94 -::::

95 -

76 :::::

78 ## Configure a new environment

@@ -130,14 +110,6 @@ vLLM is a Python library that supports the following AI accelerators. Select you

110

111 ::::

112

133 -::::{tab-item} OpenVINO

134 -:sync: openvino

135 -

136 -:::{include} python_env_setup.inc.md

137 -:::

138 -

139 -::::

140 -

113 :::::

114

115 ## Set up using Python

@@ -177,16 +149,6 @@ vLLM is a Python library that supports the following AI accelerators. Select you

149

150 ::::

151

180 -::::{tab-item} OpenVINO

181 -:sync: openvino

182 -

183 -:::{include} ai_accelerator/openvino.inc.md

184 -:start-after: "### Pre-built wheels"

185 -:end-before: "### Build wheel from source"

186 -:::

187 -

188 -::::

189 -

152 :::::

153

154 ### Build wheel from source

@@ -224,16 +186,6 @@ vLLM is a Python library that supports the following AI accelerators. Select you

186

187 ::::

188

227 -::::{tab-item} OpenVINO

228 -:sync: openvino

229 -

230 -:::{include} ai_accelerator/openvino.inc.md

231 -:start-after: "### Build wheel from source"

232 -:end-before: "## Set up using Docker"

233 -:::

234 -

235 -::::

236 -

189 :::::

190

191 ## Set up using Docker

@@ -273,16 +225,6 @@ vLLM is a Python library that supports the following AI accelerators. Select you

225

226 ::::

227

276 -::::{tab-item} OpenVINO

277 -:sync: openvino

278 -

279 -:::{include} ai_accelerator/openvino.inc.md

280 -:start-after: "### Pre-built images"

281 -:end-before: "### Build image from source"

282 -:::

283 -

284 -::::

285 -

228 :::::

229

230 ### Build image from source

@@ -320,16 +262,6 @@ vLLM is a Python library that supports the following AI accelerators. Select you

262

263 ::::

264

323 -::::{tab-item} OpenVINO

324 -:sync: openvino

325 -

326 -:::{include} ai_accelerator/openvino.inc.md

327 -:start-after: "### Build image from source"

328 -:end-before: "## Extra information"

329 -:::

330 -

331 -::::

332 -

265 :::::

266

267 ## Extra information

@@ -364,13 +296,4 @@ vLLM is a Python library that supports the following AI accelerators. Select you

296

297 ::::

298

367 -::::{tab-item} OpenVINO

368 -:sync: openvino

369 -

370 -:::{include} ai_accelerator/openvino.inc.md

371 -:start-after: "## Extra information"

372 -:::

373 -

374 -::::

375 -

299 :::::

docs/source/getting_started/installation/ai_accelerator/openvino.inc.md deleted

-110

@@ -1,110 +0,0 @@

1 -# Installation

2 -

3 -vLLM powered by OpenVINO supports all LLM models from [vLLM supported models list](#supported-models) and can perform optimal model serving on all x86-64 CPUs with, at least, AVX2 support, as well as on both integrated and discrete Intel® GPUs ([the list of supported GPUs](https://docs.openvino.ai/2024/about-openvino/release-notes-openvino/system-requirements.html#gpu)).

4 -

5 -:::{attention}

6 -There are no pre-built wheels or images for this device, so you must build vLLM from source.

7 -:::

8 -

9 -## Requirements

10 -

11 -- OS: Linux

12 -- Instruction set architecture (ISA) requirement: at least AVX2.

13 -

14 -## Set up using Python

15 -

16 -### Pre-built wheels

17 -

18 -Currently, there are no pre-built OpenVINO wheels.

19 -

20 -### Build wheel from source

21 -

22 -First, install Python and ensure you have the latest pip. For example, on Ubuntu 22.04, you can run:

23 -

24 -```console

25 -sudo apt-get update -y

26 -sudo apt-get install python3

27 -pip install --upgrade pip

28 -```

29 -

30 -Second, clone vLLM and install prerequisites for the vLLM OpenVINO backend installation:

31 -

32 -```console

33 -git clone https://github.com/vllm-project/vllm.git

34 -cd vllm

35 -pip install -r requirements/build.txt --extra-index-url https://download.pytorch.org/whl/cpu

36 -```

37 -

38 -Finally, install vLLM with OpenVINO backend:

39 -

40 -```console

41 -PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" VLLM_TARGET_DEVICE=openvino python -m pip install -v .

42 -```

43 -

44 -:::{tip}

45 -To use vLLM OpenVINO backend with a GPU device, ensure your system is properly set up. Follow the instructions provided here: [https://docs.openvino.ai/2024/get-started/configurations/configurations-intel-gpu.html](https://docs.openvino.ai/2024/get-started/configurations/configurations-intel-gpu.html).

46 -:::

47 -

48 -## Set up using Docker

49 -

50 -### Pre-built images

51 -

52 -Currently, there are no pre-built OpenVINO images.

53 -

54 -### Build image from source

55 -

56 -```console

57 -docker build -f Dockerfile.openvino -t vllm-openvino-env .

58 -docker run -it --rm vllm-openvino-env

59 -```

60 -

61 -## Extra information

62 -

63 -## Supported features

64 -

65 -OpenVINO vLLM backend supports the following advanced vLLM features:

66 -

67 -- Prefix caching (`--enable-prefix-caching`)

68 -- Chunked prefill (`--enable-chunked-prefill`)

69 -

70 -## Performance tips

71 -

72 -### vLLM OpenVINO backend environment variables

73 -

74 -- `VLLM_OPENVINO_DEVICE` to specify which device utilize for the inference. If there are multiple GPUs in the system, additional indexes can be used to choose the proper one (e.g, `VLLM_OPENVINO_DEVICE=GPU.1`). If the value is not specified, CPU device is used by default.

75 -- `VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON` to enable U8 weights compression during model loading stage. By default, compression is turned off. You can also export model with different compression techniques using `optimum-cli` and pass exported folder as `<model_id>`

76 -

77 -### CPU performance tips

78 -

79 -CPU uses the following environment variables to control behavior:

80 -

81 -- `VLLM_OPENVINO_KVCACHE_SPACE` to specify the KV Cache size (e.g, `VLLM_OPENVINO_KVCACHE_SPACE=40` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.

82 -- `VLLM_OPENVINO_CPU_KV_CACHE_PRECISION=u8` to control KV cache precision. By default, FP16 / BF16 is used depending on platform.

83 -

84 -To enable better TPOT / TTFT latency, you can use vLLM's chunked prefill feature (`--enable-chunked-prefill`). Based on the experiments, the recommended batch size is `256` (`--max-num-batched-tokens`)

85 -

86 -OpenVINO best known configuration for CPU is:

87 -

88 -```console

89 -$ VLLM_OPENVINO_KVCACHE_SPACE=100 VLLM_OPENVINO_CPU_KV_CACHE_PRECISION=u8 VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON \

90 - python3 vllm/benchmarks/benchmark_throughput.py --model meta-llama/Llama-2-7b-chat-hf --dataset vllm/benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json --enable-chunked-prefill --max-num-batched-tokens 256

91 -```

92 -

93 -### GPU performance tips

94 -

95 -GPU device implements the logic for automatic detection of available GPU memory and, by default, tries to reserve as much memory as possible for the KV cache (taking into account `gpu_memory_utilization` option). However, this behavior can be overridden by explicitly specifying the desired amount of memory for the KV cache using `VLLM_OPENVINO_KVCACHE_SPACE` environment variable (e.g, `VLLM_OPENVINO_KVCACHE_SPACE=8` means 8 GB space for KV cache).

96 -

97 -Currently, the best performance using GPU can be achieved with the default vLLM execution parameters for models with quantized weights (8 and 4-bit integer data types are supported) and `preemption-mode=swap`.

98 -

99 -OpenVINO best known configuration for GPU is:

100 -

101 -```console

102 -$ VLLM_OPENVINO_DEVICE=GPU VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON \

103 - python3 vllm/benchmarks/benchmark_throughput.py --model meta-llama/Llama-2-7b-chat-hf --dataset vllm/benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json

104 -```

105 -

106 -## Limitations

107 -

108 -- LoRA serving is not supported.

109 -- Only LLM models are currently supported. LLaVa and encoder-decoder models are not currently enabled in vLLM OpenVINO integration.

110 -- Tensor and pipeline parallelism are not currently enabled in vLLM integration.

requirements/openvino.txt deleted

-8

@@ -1,8 +0,0 @@

1 -# Common dependencies

2 --r common.txt

3 -

4 -torch == 2.5.1 # should be aligned with "common" vLLM torch version

5 -openvino >= 2024.4.0 # since 2024.4.0 both CPU and GPU support Paged Attention

6 -

7 -optimum @ git+https://github.com/huggingface/optimum.git # latest optimum is used to support latest transformers version

8 -optimum-intel[nncf] @ git+https://github.com/huggingface/optimum-intel.git # latest optimum-intel is used to support latest transformers version

setup.py

+1 -9

@@ -449,10 +449,6 @@ def _is_cpu() -> bool:

449 return VLLM_TARGET_DEVICE == "cpu"

450

451

452 -def _is_openvino() -> bool:

453 - return VLLM_TARGET_DEVICE == "openvino"

454 -

455 -

452 def _is_xpu() -> bool:

453 return VLLM_TARGET_DEVICE == "xpu"

454

@@ -572,8 +568,6 @@ def get_vllm_version() -> str:

568 if gaudi_sw_version != MAIN_CUDA_VERSION:

569 gaudi_sw_version = gaudi_sw_version.replace(".", "")[:3]

570 version += f"{sep}gaudi{gaudi_sw_version}"

575 - elif _is_openvino():

576 - version += f"{sep}openvino"

571 elif _is_tpu():

572 version += f"{sep}tpu"

573 elif _is_cpu():

@@ -623,8 +617,6 @@ def get_requirements() -> list[str]:

617 requirements = _read_requirements("neuron.txt")

618 elif _is_hpu():

619 requirements = _read_requirements("hpu.txt")

626 - elif _is_openvino():

627 - requirements = _read_requirements("openvino.txt")

620 elif _is_tpu():

621 requirements = _read_requirements("tpu.txt")

622 elif _is_cpu():

@@ -634,7 +626,7 @@ def get_requirements() -> list[str]:

626 else:

627 raise ValueError(

628 "Unsupported platform, please use CUDA, ROCm, Neuron, HPU, "

637 - "OpenVINO, or CPU.")

629 + "or CPU.")

630 return requirements

631

632

tests/conftest.py

+1 -2

@@ -273,8 +273,7 @@ class HfRunner:

273 def get_default_device(self):

274 from vllm.platforms import current_platform

275

276 - return ("cpu" if current_platform.is_cpu()

277 - or current_platform.is_openvino() else "cuda")

276 + return ("cpu" if current_platform.is_cpu() else "cuda")

277

278 def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:

279 if x is None or isinstance(x, (bool, )):

tests/kernels/test_attention_selector.py

+3 -11

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

3 -from unittest.mock import Mock, patch

3 +from unittest.mock import patch

5 import pytest

6 import torch

@@ -8,7 +8,6 @@ import torch

8 from vllm.attention.selector import _cached_get_attn_backend, get_attn_backend

9 from vllm.platforms.cpu import CpuPlatform

10 from vllm.platforms.cuda import CudaPlatform

11 -from vllm.platforms.openvino import OpenVinoPlatform

11 from vllm.platforms.rocm import RocmPlatform

12 from vllm.utils import STR_BACKEND_ENV_VAR, STR_FLASH_ATTN_VAL, STR_INVALID_VAL

@@ -21,9 +20,9 @@ def clear_cache():

22 @pytest.mark.parametrize(

24 - "name", ["TORCH_SDPA", "ROCM_FLASH", "XFORMERS", "FLASHINFER", "OPENVINO"])

23 + "name", ["TORCH_SDPA", "ROCM_FLASH", "XFORMERS", "FLASHINFER"])

24 @pytest.mark.parametrize("use_v1", [True, False])

26 -@pytest.mark.parametrize("device", ["cpu", "openvino", "hip", "cuda"])

25 +@pytest.mark.parametrize("device", ["cpu", "hip", "cuda"])

26 def test_env(

27 name: str,

28 use_v1: bool,

@@ -51,13 +50,6 @@ def test_env(

50 16, False)

51 EXPECTED = "TRITON_ATTN_VLLM_V1" if use_v1 else "ROCM_FLASH"

52 assert backend.get_name() == EXPECTED

54 - elif device == "openvino":

55 - with patch("vllm.attention.selector.current_platform",

56 - OpenVinoPlatform()), patch.dict('sys.modules',

57 - {'openvino': Mock()}):

58 - backend = get_attn_backend(16, torch.float16, torch.float16,

59 - 16, False)

60 - assert backend.get_name() == "OPENVINO"

53 else:

54 if name in ["XFORMERS", "FLASHINFER"]:

55 with patch("vllm.attention.selector.current_platform",

tests/utils.py

+1 -1

@@ -786,7 +786,7 @@ def large_gpu_mark(min_gb: int) -> pytest.MarkDecorator:

786 without enough resources, or called when filtering tests to run directly.

787 """

788 try:

789 - if current_platform.is_cpu() or current_platform.is_openvino():

789 + if current_platform.is_cpu():

790 memory_gb = 0

791 else:

792 memory_gb = current_platform.get_device_total_memory() / GB_bytes

vllm/attention/backends/openvino.py deleted

-146

@@ -1,146 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -

3 -from dataclasses import dataclass

4 -from typing import Dict, List, Optional, Tuple, Type

5 -

6 -import openvino as ov

7 -import torch

8 -

9 -from vllm.attention.backends.abstract import (AttentionBackend,

10 - AttentionMetadata)

11 -from vllm.attention.backends.utils import CommonAttentionState

12 -from vllm.multimodal import MultiModalPlaceholderMap

13 -

14 -

15 -def copy_cache_block(src_tensor: ov.Tensor, dst_tensor: ov.Tensor,

16 - src_offset: int, dst_offset: int) -> None:

17 -

18 - def create_roi_tensor(

19 - tensor: ov.Tensor,

20 - block_number: int,

21 - ) -> ov.Tensor:

22 - roi_begin = ov.runtime.Coordinate([0, 0, 0, 0])

23 - roi_end = ov.runtime.Coordinate(tensor.get_shape())

24 -

25 - roi_begin[0] = block_number

26 - roi_end[0] = block_number + 1

27 -

28 - if isinstance(tensor, ov.Tensor):

29 - return ov.Tensor(tensor, roi_begin, roi_end)

30 - else:

31 - return ov.RemoteTensor(tensor, roi_begin, roi_end)

32 -

33 - src_roi_tensor = \

34 - create_roi_tensor(src_tensor, src_offset)

35 - dst_roi_tensor = \

36 - create_roi_tensor(dst_tensor, dst_offset)

37 - src_roi_tensor.copy_to(dst_roi_tensor)

38 -

39 -

40 -class OpenVINOAttentionBackend(AttentionBackend):

41 -

42 - @staticmethod

43 - def get_name() -> str:

44 - return "OPENVINO"

45 -

46 - @staticmethod

47 - def get_impl_cls():

48 - # OpenVINO implements PagedAttention as part of the Optimum

49 - # exported model

50 - raise NotImplementedError

51 -

52 - @staticmethod

53 - def make_metadata(*args, **kwargs) -> "AttentionMetadata":

54 - raise NotImplementedError

55 -

56 - @staticmethod

57 - def get_state_cls() -> Type["CommonAttentionState"]:

58 - return CommonAttentionState

59 -

60 - @staticmethod

61 - def make_openvino_metadata(*args, **kwargs) -> "OpenVINOAttentionMetadata":

62 - return OpenVINOAttentionMetadata(*args, **kwargs)

63 -

64 - @staticmethod

65 - def get_kv_cache_shape(

66 - num_blocks: int,

67 - block_size: int,

68 - num_kv_heads: int,

69 - head_size: int,

70 - ) -> Tuple[int, ...]:

71 - return (2, num_blocks, num_kv_heads, block_size, head_size)

72 -

73 - @staticmethod

74 - def swap_blocks(

75 - src_tensor: ov.Tensor,

76 - dst_tensor: ov.Tensor,

77 - src_to_dists: List[Tuple[int, int]],

78 - ) -> None:

79 - for src, dst in src_to_dists:

80 - copy_cache_block(src_tensor, dst_tensor, src, dst)

81 -

82 - @staticmethod

83 - def copy_blocks(

84 - kv_caches: List[Tuple[ov.Tensor, ov.Tensor]],

85 - src_to_dists: List[Tuple[int, int]],

86 - ) -> None:

87 - for src, dst in src_to_dists:

88 - for key_cache, value_cache in kv_caches:

89 - copy_cache_block(key_cache, key_cache, src, dst)

90 - copy_cache_block(value_cache, value_cache, src, dst)

91 -

92 -

93 -@dataclass

94 -class OpenVINOAttentionMetadata:

95 - """Metadata for OpenVINOAttentionBackend.

96 -

97 - Basic terms used below:

98 - - batch_size_in_sequences - total number of sequences to execute

99 - - prompt_lens – per sequence size number of scheduled tokens

100 - - batch_size_in_tokens = sum(prompt_lens)

101 - - max_context_len = max(context_lens)

102 - - max_num_blocks = div_up(max_context_len / BLOCK_SIZE)

103 - - num_blocks – total number of blocks in block_indices

104 - """

105 -

106 - # Describes past KV cache size for each sequence within a batch

107 - # Shape: [batch_size_in_sequences]

108 - # Type: i32

109 - past_lens: torch.Tensor

110 -

111 - # Describes start indices of input / speculative tokens from

112 - # current sequences within a batch sequence

113 - # Shape: [batch_size_in_sequences + 1]

114 - # Type: i32

115 - subsequence_begins: torch.Tensor

116 -

117 - # Describes block tables for each sequence within a batch -

118 - # indices along 0th dimension in key_cache and value_cache inputs

119 - # Shape: [num_blocks]

120 - # Type: i32

121 - block_indices: torch.Tensor

122 -

123 - # Describes block tables for each sequence within a batch -

124 - # for i-th element, it is an index in block_indices with the

125 - # first block belonging to i-th sequence

126 - # Shape: [batch_size_in_sequences + 1]

127 - # Type: i32

128 - block_indices_begins: torch.Tensor

129 -

130 - # Describes max context length

131 - # Shape: scalar

132 - # Type: i32

133 - max_context_len: torch.Tensor

134 -

135 - # The index maps that relate multi-modal embeddings to the corresponding

136 - # placeholders.

137 - #

138 - # N.B. These aren't really related to attention and don't belong on this

139 - # type -- this is just a temporary solution to make them available to

140 - # `model_executable`.

141 - multi_modal_placeholder_index_maps: Optional[Dict[

142 - str, MultiModalPlaceholderMap.IndexMap]]

143 -

144 - # Enable/disable KV scales calculation. This is so that we can disable the

145 - # calculation until after prefill and cuda graph capture.

146 - enable_kv_scales_calculation: bool

vllm/config.py

+1 -1

@@ -1801,7 +1801,7 @@ class DeviceConfig:

1801 self.device_type = device

1802

1803 # Some device types require processing inputs on CPU

1804 - if self.device_type in ["neuron", "openvino"]:

1804 + if self.device_type in ["neuron"]:

1805 self.device = torch.device("cpu")

1806 elif self.device_type in ["tpu"]:

1807 self.device = None

vllm/engine/arg_utils.py

-1

@@ -40,7 +40,6 @@ DEVICE_OPTIONS = [

40 "cuda",

41 "neuron",

42 "cpu",

43 - "openvino",

43 "tpu",

44 "xpu",

45 "hpu",

vllm/envs.py

+1 -27

@@ -40,10 +40,6 @@ if TYPE_CHECKING:

40 VLLM_CPU_KVCACHE_SPACE: int = 0

41 VLLM_CPU_OMP_THREADS_BIND: str = ""

42 VLLM_CPU_MOE_PREPACK: bool = True

43 - VLLM_OPENVINO_DEVICE: str = "CPU"

44 - VLLM_OPENVINO_KVCACHE_SPACE: int = 0

45 - VLLM_OPENVINO_CPU_KV_CACHE_PRECISION: Optional[str] = None

46 - VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False

43 VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")

44 VLLM_XLA_CHECK_RECOMPILATION: bool = False

45 VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024

@@ -131,7 +127,7 @@ environment_variables: dict[str, Callable[[], Any]] = {

127 # ================== Installation Time Env Vars ==================

128

129 # Target device of vLLM, supporting [cuda (by default),

134 - # rocm, neuron, cpu, openvino]

130 + # rocm, neuron, cpu]

131 "VLLM_TARGET_DEVICE":

132 lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda"),

133

@@ -358,28 +354,6 @@ environment_variables: dict[str, Callable[[], Any]] = {

354 "VLLM_CPU_MOE_PREPACK":

355 lambda: bool(int(os.getenv("VLLM_CPU_MOE_PREPACK", "1"))),

356

361 - # OpenVINO device selection

362 - # default is CPU

363 - "VLLM_OPENVINO_DEVICE":

364 - lambda: os.getenv("VLLM_OPENVINO_DEVICE", "CPU").upper(),

365 -

366 - # OpenVINO key-value cache space

367 - # default is 4GB

368 - "VLLM_OPENVINO_KVCACHE_SPACE":

369 - lambda: int(os.getenv("VLLM_OPENVINO_KVCACHE_SPACE", "0")),

370 -

371 - # OpenVINO KV cache precision

372 - # default is bf16 if natively supported by platform, otherwise f16

373 - # To enable KV cache compression, please, explicitly specify u8

374 - "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION":

375 - lambda: os.getenv("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION", None),

376 -

377 - # Enables weights compression during model export via HF Optimum

378 - # default is False

379 - "VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS":

380 - lambda:

381 - (os.environ.get("VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS", "0").lower() in

382 - ("on", "true", "1")),

357 # If the env var is set, then all workers will execute as separate

358 # processes from the engine, and we use the same mechanism to trigger

359 # execution on all workers.

vllm/model_executor/model_loader/openvino.py deleted

-204

@@ -1,204 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -

3 -# ruff: noqa: SIM117

4 -from pathlib import Path

5 -from typing import Optional

6 -

7 -import openvino as ov

8 -import torch

9 -from huggingface_hub import HfApi

10 -from openvino._offline_transformations import paged_attention_transformation

11 -from optimum.intel import OVModelForCausalLM

12 -from torch import nn

13 -

14 -import vllm.envs as envs

15 -from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config

16 -from vllm.forward_context import get_forward_context

17 -from vllm.logger import init_logger

18 -from vllm.model_executor.layers.logits_processor import (LogitsProcessor,

19 - _prune_hidden_states)

20 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput

21 -from vllm.model_executor.sampling_metadata import SamplingMetadata

22 -from vllm.platforms import current_platform

23 -

24 -logger = init_logger(__name__)

25 -

26 -

27 -def _flatten_inputs(inputs):

28 - """

29 - Helper function for making nested inputs flattens

30 - """

31 - flatten_inputs = []

32 - for input_data in inputs:

33 - if input_data is None:

34 - continue

35 - if isinstance(input_data, (list, tuple)):

36 - flatten_inputs.extend(_flatten_inputs(input_data))

37 - elif isinstance(input_data, dict):

38 - flatten_inputs.extend(_flatten_inputs(list(input_data.values())))

39 - else:

40 - flatten_inputs.append(input_data)

41 - return flatten_inputs

42 -

43 -

44 -def _modify_cache_parameters(model: ov.Model, kv_cache_dtype: ov.Type,

45 - is_cpu: bool):

46 - # Apply hardware dependent modifications to KV tensors

47 - for parameter in model.get_parameters():

48 - input = parameter.get_output_tensor(0)

49 - input_names = input.get_names()

50 - if len(input_names) != 1:

51 - continue

52 - input_name = next(iter(input_names))

53 - shape = parameter.get_partial_shape()

54 - # use real block size if available, just a placeholder

55 - # to provide the expected rank

56 - num_blocks = ov.Dimension()

57 - block_size = ov.Dimension()

58 - head_size = ov.Dimension()

59 - if input_name.startswith("key_cache."):

60 - cpu_shape = [num_blocks, shape[1], block_size, head_size]

61 - gpu_shape = [num_blocks, shape[1], shape[2], block_size]

62 - elif input_name.startswith("value_cache."):

63 - cpu_shape = [num_blocks, shape[1], block_size, head_size]

64 - gpu_shape = [num_blocks, shape[1], block_size, shape[2]]

65 - else:

66 - continue

67 - parameter.set_partial_shape(

68 - ov.PartialShape(cpu_shape if is_cpu else gpu_shape))

69 - parameter.set_element_type(kv_cache_dtype)

70 - model.validate_nodes_and_infer_types()

71 -

72 -

73 -def _require_model_export(model_id, revision=None, subfolder=None):

74 - model_dir = Path(model_id)

75 - if subfolder is not None:

76 - model_dir = model_dir / subfolder

77 - if model_dir.is_dir():

78 - return (not (model_dir / "openvino_model.xml").exists()

79 - or not (model_dir / "openvino_model.bin").exists())

80 -

81 - hf_api = HfApi()

82 - try:

83 - model_info = hf_api.model_info(model_id, revision=revision or "main")

84 - normalized_subfolder = (None if subfolder is None else

85 - Path(subfolder).as_posix())

86 - model_files = [

87 - file.rfilename for file in model_info.siblings

88 - if normalized_subfolder is None

89 - or file.rfilename.startswith(normalized_subfolder)

90 - ]

91 - ov_model_path = ("openvino_model.xml" if normalized_subfolder is None

92 - else f"{normalized_subfolder}/openvino_model.xml")

93 - return (ov_model_path not in model_files

94 - or ov_model_path.replace(".xml", ".bin") not in model_files)

95 - except Exception:

96 - return True

97 -

98 -

99 -class OpenVINOCausalLM(nn.Module):

100 -

101 - def __init__(

102 - self,

103 - ov_core: ov.Core,

104 - model_config: ModelConfig,

105 - kv_cache_dtype: ov.Type,

106 - ) -> None:

107 - super().__init__()

108 - self.logits_processor = LogitsProcessor(

109 - model_config.hf_config.vocab_size, logits_as_input=True)

110 - self.sampler = Sampler()

111 -

112 - export = _require_model_export(model_config.model)

113 - if export:

114 - logger.warning(

115 - f"Provided model id {model_config.model} does not " # noqa: G004

116 - "contain OpenVINO IR, the model will be converted to IR with "

117 - "default options. If you need to use specific options for "

118 - "model conversion, use optimum-cli export openvino with "

119 - "desired options.")

120 - else:

121 - logger.warning(

122 - "OpenVINO IR is available for provided model id " # noqa: G004

123 - f"{model_config.model}. This IR will be used for inference "

124 - "as-is, all possible options that may affect model conversion "

125 - "are ignored.")

126 -

127 - load_in_8bit = (envs.VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS

128 - if export else False)

129 - pt_model = OVModelForCausalLM.from_pretrained(

130 - model_config.model,

131 - export=export,

132 - compile=False,

133 - load_in_8bit=load_in_8bit,

134 - trust_remote_code=model_config.trust_remote_code,

135 - )

136 -

137 - ov_device = envs.VLLM_OPENVINO_DEVICE

138 - paged_attention_transformation(pt_model.model)

139 - _modify_cache_parameters(pt_model.model, kv_cache_dtype,

140 - current_platform.is_openvino_cpu())

141 -

142 - ov_compiled = ov_core.compile_model(pt_model.model, ov_device)

143 - self.ov_request = ov_compiled.create_infer_request()

144 -

145 - def forward(

146 - self,

147 - input_ids: torch.Tensor,

148 - positions: torch.Tensor,

149 - kv_caches: list[tuple[ov.Tensor, ov.Tensor]],

150 - ) -> torch.Tensor:

151 - flat_kv_caches = _flatten_inputs(kv_caches)

152 - attn_metadata = get_forward_context().attn_metadata

153 -

154 - inputs = [

155 - input_ids,

156 - positions,

157 - *flat_kv_caches,

158 - attn_metadata.past_lens,

159 - attn_metadata.subsequence_begins,

160 - attn_metadata.block_indices,

161 - attn_metadata.block_indices_begins,

162 - attn_metadata.max_context_len,

163 - ]

164 -

165 - self.ov_request.start_async(inputs, share_inputs=True)

166 - self.ov_request.wait()

167 -

168 - logits = torch.from_numpy(self.ov_request.get_tensor("logits").data)

169 -

170 - # TODO: remove 'view' once OpenVINO PA will drop 'seq_len' dimension

171 - return logits.view(-1, logits.shape[-1])

172 -

173 - def compute_logits(self, hidden_states: torch.Tensor,

174 - sampling_metadata: SamplingMetadata) -> torch.Tensor:

175 - hidden_states = _prune_hidden_states(hidden_states, sampling_metadata)

176 - logits = self.logits_processor(None, hidden_states, sampling_metadata)

177 - return logits

178 -

179 - def sample(

180 - self,

181 - logits: torch.Tensor,

182 - sampling_metadata: SamplingMetadata,

183 - ) -> Optional[SamplerOutput]:

184 - next_tokens = self.sampler(logits, sampling_metadata)

185 - return next_tokens

186 -

187 -

188 -def get_model(

189 - vllm_config: VllmConfig,

190 - kv_cache_dtype: ov.Type,

191 - **kwargs,

192 -) -> torch.nn.Module:

193 - lora_config = kwargs.get("lora_config")

194 - ov_core = kwargs.get("ov_core")

195 - if lora_config:

196 - raise ValueError(

197 - "OpenVINO modeling does not support LoRA, "

198 - "but LoRA is enabled. Support for this model may "

199 - "be added in the future. If this is important to you, "

200 - "please open an issue on github.")

201 -

202 - with set_current_vllm_config(vllm_config):

203 - return OpenVINOCausalLM(ov_core, vllm_config.model_config,

204 - kv_cache_dtype)

vllm/platforms/__init__.py

-17

@@ -2,7 +2,6 @@

3 import logging

4 import traceback

5 -from contextlib import suppress

5 from itertools import chain

6 from typing import TYPE_CHECKING, Optional

@@ -191,21 +190,6 @@ def neuron_platform_plugin() -> Optional[str]:

190 return "vllm.platforms.neuron.NeuronPlatform" if is_neuron else None

191

192

194 -def openvino_platform_plugin() -> Optional[str]:

195 - is_openvino = False

196 - logger.debug("Checking if OpenVINO platform is available.")

197 - with suppress(Exception):

198 - is_openvino = vllm_version_matches_substr("openvino")

199 - if is_openvino:

200 - logger.debug("Confirmed OpenVINO platform is available"

201 - " because vLLM is built with OpenVINO.")

202 - if not is_openvino:

203 - logger.debug("OpenVINO platform is not available because"

204 - " vLLM is not built with OpenVINO.")

205 -

206 - return "vllm.platforms.openvino.OpenVinoPlatform" if is_openvino else None

207 -

208 -

193 builtin_platform_plugins = {

194 'tpu': tpu_platform_plugin,

195 'cuda': cuda_platform_plugin,

@@ -214,7 +198,6 @@ builtin_platform_plugins = {

198 'xpu': xpu_platform_plugin,

199 'cpu': cpu_platform_plugin,

200 'neuron': neuron_platform_plugin,

217 - 'openvino': openvino_platform_plugin,

201 }

202

203

vllm/platforms/interface.py

-5

@@ -33,7 +33,6 @@ class _Backend(enum.Enum):

33 XFORMERS = enum.auto()

34 ROCM_FLASH = enum.auto()

35 TORCH_SDPA = enum.auto()

36 - OPENVINO = enum.auto()

36 FLASHINFER = enum.auto()

37 TRITON_MLA = enum.auto() # Supported by V1

38 FLASHMLA = enum.auto() # Supported by V1

@@ -53,7 +52,6 @@ class PlatformEnum(enum.Enum):

52 XPU = enum.auto()

53 CPU = enum.auto()

54 NEURON = enum.auto()

56 - OPENVINO = enum.auto()

55 OOT = enum.auto()

56 UNSPECIFIED = enum.auto()

@@ -136,9 +134,6 @@ class Platform:

134 def is_neuron(self) -> bool:

135 return self._enum == PlatformEnum.NEURON

136

139 - def is_openvino(self) -> bool:

140 - return self._enum == PlatformEnum.OPENVINO

141 -

137 def is_out_of_tree(self) -> bool:

138 return self._enum == PlatformEnum.OOT

139

vllm/platforms/openvino.py deleted

-152

@@ -1,152 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -

3 -from typing import TYPE_CHECKING, Optional

4 -

5 -import torch

6 -

7 -import vllm.envs as envs

8 -from vllm.logger import init_logger

9 -

10 -from .interface import Platform, PlatformEnum, _Backend

11 -

12 -if TYPE_CHECKING:

13 - from vllm.config import VllmConfig

14 -else:

15 - VllmConfig = None

16 -

17 -logger = init_logger(__name__)

18 -

19 -try:

20 - import openvino as ov

21 - import openvino.properties.hint as hints

22 -except ImportError as e:

23 - logger.warning("Failed to import OpenVINO with %r", e)

24 -

25 -

26 -class OpenVinoPlatform(Platform):

27 - _enum = PlatformEnum.OPENVINO

28 - device_name: str = "openvino"

29 - device_type: str = "openvino"

30 - dispatch_key: str = "CPU"

31 -

32 - @classmethod

33 - def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,

34 - dtype: torch.dtype, kv_cache_dtype: Optional[str],

35 - block_size: int, use_v1: bool,

36 - use_mla: bool) -> str:

37 - if selected_backend != _Backend.OPENVINO:

38 - logger.info("Cannot use %s backend on OpenVINO.", selected_backend)

39 - logger.info("Using OpenVINO Attention backend.")

40 - return "vllm.attention.backends.openvino.OpenVINOAttentionBackend"

41 -

42 - @classmethod

43 - def get_device_name(cls, device_id: int = 0) -> str:

44 - return "openvino"

45 -

46 - @classmethod

47 - def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:

48 - return False

49 -

50 - @classmethod

51 - def inference_mode(cls):

52 - return torch.inference_mode(mode=True)

53 -

54 - @classmethod

55 - def is_openvino_cpu(cls) -> bool:

56 - return "CPU" in envs.VLLM_OPENVINO_DEVICE

57 -

58 - @classmethod

59 - def is_openvino_gpu(cls) -> bool:

60 - return "GPU" in envs.VLLM_OPENVINO_DEVICE

61 -

62 - @classmethod

63 - def is_pin_memory_available(cls) -> bool:

64 - logger.warning("Pin memory is not supported on OpenViNO.")

65 - return False

66 -

67 - @classmethod

68 - def check_and_update_config(cls, vllm_config: VllmConfig) -> None:

69 - from vllm.utils import GiB_bytes

70 -

71 - parallel_config = vllm_config.parallel_config

72 - assert (parallel_config.world_size == 1

73 - ), "OpenVINO only supports single CPU socket currently."

74 -

75 - if parallel_config.worker_cls == "auto":

76 - parallel_config.worker_cls = \

77 - "vllm.worker.openvino_worker.OpenVINOWorker"

78 -

79 - # check and update model config

80 - model_config = vllm_config.model_config

81 - if model_config.dtype != torch.float32:

82 - logger.warning(

83 - f"Only float32 dtype is supported on OpenVINO, casting from {model_config.dtype}." # noqa: G004, E501

84 - )

85 - model_config.dtype = torch.float32

86 - if not model_config.enforce_eager:

87 - logger.warning(

88 - "CUDA graph is not supported on OpenVINO backend, fallback to "

89 - "the eager mode.")

90 - model_config.enforce_eager = True

91 -

92 - # check and update cache config

93 - ov_core = ov.Core()

94 - cache_config = vllm_config.cache_config

95 - if cache_config and cache_config.block_size is None:

96 - cache_config.block_size = 16

97 -

98 - if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8":

99 - if not OpenVinoPlatform.is_openvino_cpu():

100 - logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is "

101 - "ignored for GPU, f16 data type will be used.")

102 - cache_config.cache_dtype = ov.Type.f16

103 - else:

104 - logger.info("KV cache type is overridden to u8 via "

105 - "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION env var.")

106 - cache_config.cache_dtype = ov.Type.u8

107 - else:

108 - if OpenVinoPlatform.is_openvino_cpu():

109 - ov_device = envs.VLLM_OPENVINO_DEVICE

110 - inference_precision = ov_core.get_property(

111 - ov_device, hints.inference_precision)

112 - if inference_precision == ov.Type.bf16:

113 - cache_config.cache_dtype = ov.Type.bf16

114 - else:

115 - cache_config.cache_dtype = ov.Type.f16

116 - else:

117 - cache_config.cache_dtype = ov.Type.f16

118 -

119 - if OpenVinoPlatform.is_openvino_cpu():

120 - if cache_config.block_size != 32:

121 - logger.info(

122 - f"OpenVINO CPU optimal block size is 32, overriding currently set {cache_config.block_size}" # noqa: G004, E501

123 - )

124 - cache_config.block_size = 32

125 - else:

126 - if cache_config.block_size != 16:

127 - logger.info(

128 - f"OpenVINO GPU optimal block size is 16, overriding currently set {cache_config.block_size}" # noqa: G004, E501

129 - )

130 - cache_config.block_size = 16

131 -

132 - kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE

133 - if kv_cache_space >= 0:

134 - if kv_cache_space == 0 and OpenVinoPlatform.is_openvino_cpu():

135 - cache_config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore

136 - logger.warning(

137 - "Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) "

138 - "for OpenVINO backend is not set, using 4 by default.")

139 - else:

140 - cache_config.openvino_kvcache_space_bytes = ( # type: ignore

141 - kv_cache_space * GiB_bytes)

142 - else:

143 - raise RuntimeError(

144 - "Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE"

145 - f" {kv_cache_space}, expect a positive integer value.")

146 -

147 - assert vllm_config.device_config.device_type == "openvino"

148 - assert vllm_config.lora_config is None, \

149 - "OpenVINO backend doesn't support LoRA"

150 - assert cls.is_openvino_cpu() or \

151 - cls.is_openvino_gpu(), \

152 - "OpenVINO backend supports only CPU and GPU devices"

vllm/worker/openvino_model_runner.py deleted

-372

@@ -1,372 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -

3 -from collections import defaultdict

4 -from typing import Dict, List, NamedTuple, Optional, Tuple

5 -

6 -import openvino as ov

7 -import torch

8 -from torch import nn

9 -

10 -from vllm.attention import get_attn_backend

11 -from vllm.attention.backends.openvino import OpenVINOAttentionMetadata

12 -from vllm.config import VllmConfig

13 -from vllm.forward_context import set_forward_context

14 -from vllm.logger import init_logger

15 -from vllm.model_executor import SamplingMetadata

16 -from vllm.model_executor.layers.sampler import SamplerOutput

17 -from vllm.model_executor.model_loader.openvino import get_model

18 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,

19 - MultiModalKwargs, MultiModalPlaceholderMap)

20 -from vllm.sequence import SequenceGroupMetadata

21 -from vllm.worker.model_runner_base import ModelRunnerBase

22 -

23 -logger = init_logger(__name__)

24 -

25 -

26 -class ModelInput(NamedTuple):

27 - input_tokens: torch.Tensor

28 - input_positions: torch.Tensor

29 - attn_metadata: Optional[OpenVINOAttentionMetadata]

30 - seq_lens: List[int]

31 - query_lens: List[int]

32 - multi_modal_kwargs: BatchedTensorInputs

33 -

34 - @classmethod

35 - def empty(cls, device):

36 - return ModelInput(input_tokens=torch.empty(0, device=device),

37 - input_positions=torch.empty(0, device=device),

38 - attn_metadata=None,

39 - seq_lens=[],

40 - query_lens=[],

41 - multi_modal_kwargs={})

42 -

43 -

44 -class OpenVINOModelRunner(ModelRunnerBase):

45 -

46 - def __init__(

47 - self,

48 - ov_core: ov.Core,

49 - vllm_config: VllmConfig,

50 - kv_cache_dtype: Optional[str] = "auto",

51 - is_driver_worker: bool = False,

52 - *args,

53 - **kwargs,

54 - ):

55 - self.ov_core = ov_core

56 - ModelRunnerBase.__init__(self, vllm_config=vllm_config)

57 - self.is_driver_worker = is_driver_worker

58 -

59 - self.device = self.device_config.device

60 -

61 - self.kv_cache_dtype = kv_cache_dtype

62 - self.sliding_window = self.model_config.get_sliding_window()

63 - self.block_size = self.cache_config.block_size

64 -

65 - self.attn_backend = get_attn_backend(

66 - self.model_config.get_head_size(),

67 - self.model_config.dtype,

68 - self.kv_cache_dtype,

69 - self.block_size,

70 - self.model_config.is_attention_free,

71 - )

72 -

73 - # Multi-modal data support

74 - self.mm_registry = MULTIMODAL_REGISTRY

75 - self.multi_modal_input_mapper = self.mm_registry \

76 - .create_input_mapper(self.model_config)

77 -

78 - # Lazy initialization.

79 - self.model: nn.Module # Set after init_Model

80 -

81 - def load_model(self) -> None:

82 - self.model = get_model(vllm_config=self.vllm_config,

83 - kv_cache_dtype=self.kv_cache_dtype,

84 - ov_core=self.ov_core)

85 -

86 - def get_model(self) -> nn.Module:

87 - return self.model

88 -

89 - def _prepare_model_input(

90 - self,

91 - seq_group_metadata_list: List[SequenceGroupMetadata],

92 - ) -> ModelInput:

93 - """Prepare the model input based on a given sequence group.

94 -

95 - The API assumes seq_group_metadata_list is sorted by prefill -> decode.

96 -

97 - The result tensors and data structure also batches input in prefill

98 - -> decode order. For example,

99 -

100 - - input_tokens[:num_prefill_tokens] contains prefill tokens.

101 - - input_tokens[num_prefill_tokens:] contains decode tokens.

102 - """

103 - input_tokens: List[int] = []

104 - input_positions: List[int] = []

105 -

106 - seq_lens: List[int] = []

107 - past_lens: List[int] = []

108 - query_lens: List[int] = []

109 - multi_modal_kwargs_list: List[MultiModalKwargs] = []

110 - multi_modal_placeholder_maps: Dict[

111 - str,

112 - MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)

113 -

114 - subsequence_begins: List[int] = []

115 - block_indices: List[int] = []

116 - block_indices_begins: List[int] = []

117 -

118 - # initialize beginning of prefix sums

119 - subsequence_begins.append(0)

120 - block_indices_begins.append(0)

121 -

122 - if len(seq_group_metadata_list) == 0:

123 - return ModelInput.empty(self.device)

124 -

125 - for seq_group_metadata in seq_group_metadata_list:

126 - seq_ids = list(seq_group_metadata.seq_data.keys())

127 - is_prompt = seq_group_metadata.is_prompt

128 -

129 - for seq_id in seq_ids:

130 - computed_block_nums = seq_group_metadata.computed_block_nums

131 - if (self.scheduler_config is not None

132 - and self.scheduler_config.chunked_prefill_enabled

133 - and not (computed_block_nums is None

134 - or computed_block_nums == [])):

135 - raise RuntimeError(

136 - "chunked prefill cannot be used with prefix caching "

137 - "now.")

138 -

139 - seq_data = seq_group_metadata.seq_data[seq_id]

140 - if is_prompt:

141 - computed_len = seq_data.get_num_computed_tokens()

142 - else:

143 - # get_num_computed_tokens is incorrect for spec decoding.

144 - # So, we should have a special logic here.

145 - # TODO(sang): Fix it.

146 - computed_len = seq_data.get_len() - 1

147 -

148 - seq_len = min(

149 - seq_data.get_len(),

150 - computed_len + seq_group_metadata.token_chunk_size,

151 - )

152 - if is_prompt:

153 - tokens = seq_data.get_token_ids()[computed_len:seq_len]

154 - else:

155 - # Optimization. get_token_ids requires the entire copy of

156 - # tokens.

157 - tokens = [seq_data.get_last_token_id()]

158 -

159 - # Prefix cache was hit.

160 - # Prefix is not supported with sliding_window

161 - prefix_cache_hit = (computed_block_nums is not None

162 - and len(computed_block_nums) > 0

163 - and self.sliding_window is None

164 - and is_prompt)

165 -

166 - block_table = seq_group_metadata.block_tables[seq_id]

167 - # TODO(sang): Combine chunked prefill and prefix caching by

168 - # only allowing multiple of block_size chunk size.

169 - # NOTE: This only works for oooooooxxx style attention.

170 - if prefix_cache_hit:

171 - assert computed_block_nums is not None

172 - computed_len = len(computed_block_nums) * self.block_size

173 - tokens = tokens[computed_len:]

174 - elif (self.scheduler_config.chunked_prefill_enabled

175 - or not is_prompt):

176 - if seq_group_metadata.block_tables is not None:

177 - # chunked prefill or decode

178 - block_table = seq_group_metadata.block_tables[seq_id]

179 - if self.sliding_window is not None:

180 - # chunked prefill doesn't support sliding window.

181 - assert not self.scheduler_config.chunked_prefill_enabled # noqa: E501

182 - sliding_window_blocks = (self.sliding_window //

183 - self.block_size)

184 - block_table = block_table[-sliding_window_blocks:]

185 - else:

186 - # Only happens when memory profiling runs.

187 - block_table = []

188 - else:

189 - # prompt phase w/o prefix_caching, chunked_prefill

190 - pass

191 -

192 - block_indices.extend(block_table)

193 - block_indices_begins.append(block_indices_begins[-1] +

194 - len(block_table))

195 -

196 - # TODO(sang): This is a hack to make sliding window work with

197 - # paged attn. We can remove it if we make paged attn kernel

198 - # to properly handle slinding window attn.

199 - if self.sliding_window is not None and not is_prompt:

200 - seq_len = min(seq_len, self.sliding_window)

201 - computed_len = seq_len - 1

202 -

203 - seq_lens.append(seq_len)

204 -

205 - query_len = seq_len - computed_len

206 - query_lens.append(query_len)

207 -

208 - input_tokens.extend(tokens)

209 - positions_range = range(computed_len, seq_len)

210 - input_positions.extend(list(positions_range))

211 -

212 - past_lens.append(computed_len)

213 - subsequence_begins.append(subsequence_begins[-1] + query_len)

214 -

215 - if is_prompt:

216 - assert len(seq_ids) == 1

217 - else:

218 - assert (

219 - query_len == 1

220 - ), "seq_len: {}, computed_len: {}, query_len: {}".format(

221 - seq_len, computed_len, query_len)

222 -

223 - if seq_group_metadata.multi_modal_data:

224 - # NOTE: mm_data only includes the subset of multi-modal

225 - # items that intersect with the current prefill positions.

226 - mm_data, placeholder_maps = MultiModalPlaceholderMap \

227 - .from_seq_group(seq_group_metadata, positions_range)

228 -

229 - if self.mm_registry.has_processor(self.model_config):

230 - mm_kwargs = mm_data

231 - else:

232 - mm_kwargs = self.multi_modal_input_mapper(

233 - mm_data,

234 - seq_group_metadata.mm_processor_kwargs,

235 - )

236 -

237 - multi_modal_kwargs_list.append(mm_kwargs)

238 -

239 - for modality, placeholder_map in placeholder_maps.items():

240 - multi_modal_placeholder_maps[modality].extend(

241 - placeholder_map, )

242 -

243 - max_query_len = max(query_lens)

244 - assert max_query_len > 0, "query_lens: {}".format(query_lens)

245 -

246 - input_tokens = torch.tensor(input_tokens,

247 - dtype=torch.long,

248 - device=self.device) # type: ignore

249 - input_positions = torch.tensor(input_positions,

250 - dtype=torch.long,

251 - device=self.device) # type: ignore

252 -

253 - past_lens_tensor = torch.tensor(past_lens,

254 - dtype=torch.int32,

255 - device=self.device) # type: ignore

256 - subsequence_begins_tensor = torch.tensor(

257 - subsequence_begins, dtype=torch.int32,

258 - device=self.device) # type: ignore

259 - block_indices_tensor = torch.tensor(block_indices,

260 - dtype=torch.int32,

261 - device=self.device) # type: ignore

262 - block_indices_begins_tensor = torch.tensor(

263 - block_indices_begins, dtype=torch.int32,

264 - device=self.device) # type: ignore

265 -

266 - max_context_len = max(seq_lens)

267 - max_context_len_tensor = torch.tensor(

268 - max_context_len, dtype=torch.int32,

269 - device=self.device) # type: ignore

270 -

271 - placeholder_index_maps = {

272 - modality: placeholder_map.index_map()

273 - for modality, placeholder_map in

274 - multi_modal_placeholder_maps.items()

275 - }

276 -

277 - attn_metadata = self.attn_backend.make_openvino_metadata(

278 - past_lens=past_lens_tensor,

279 - subsequence_begins=subsequence_begins_tensor,

280 - block_indices=block_indices_tensor,

281 - block_indices_begins=block_indices_begins_tensor,

282 - max_context_len=max_context_len_tensor,

283 - multi_modal_placeholder_index_maps=placeholder_index_maps,

284 - enable_kv_scales_calculation=False,

285 - )

286 -

287 - multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)

288 -

289 - return ModelInput(

290 - input_tokens,

291 - input_positions,

292 - attn_metadata,

293 - seq_lens,

294 - query_lens,

295 - multi_modal_kwargs=multi_modal_kwargs,

296 - )

297 -

298 - def prepare_input_tensors(

299 - self,

300 - seq_group_metadata_list: List[SequenceGroupMetadata],

301 - ) -> Tuple[torch.Tensor, torch.Tensor, OpenVINOAttentionMetadata,

302 - SamplingMetadata, BatchedTensorInputs]:

303 - # Prepare input tensors.

304 - (

305 - input_tokens,

306 - input_positions,

307 - attn_metadata,

308 - seq_lens,

309 - query_lens,

310 - multi_modal_kwargs,

311 - ) = self._prepare_model_input(seq_group_metadata_list)

312 -

313 - sampling_metadata = SamplingMetadata.prepare(

314 - seq_group_metadata_list,

315 - seq_lens,

316 - query_lens,

317 - self.device,

318 - pin_memory=False,

319 - )

320 -

321 - return (

322 - input_tokens,

323 - input_positions,

324 - attn_metadata,

325 - sampling_metadata,

326 - multi_modal_kwargs,

327 - )

328 -

329 - @torch.inference_mode()

330 - def execute_model(

331 - self,

332 - seq_group_metadata_list: List[SequenceGroupMetadata],

333 - kv_caches: List[Tuple["ov.Tensor", "ov.Tensor"]],

334 - ) -> Optional[SamplerOutput]:

335 - (

336 - input_tokens,

337 - input_positions,

338 - attn_metadata,

339 - sampling_metadata,

340 - multi_modal_kwargs,

341 - ) = self.prepare_input_tensors(seq_group_metadata_list)

342 -

343 - model_executable = self.model

344 - execute_model_kwargs = {

345 - "input_ids":

346 - input_tokens,

347 - "positions":

348 - input_positions,

349 - "kv_caches":

350 - kv_caches,

351 - **MultiModalKwargs.as_kwargs(multi_modal_kwargs or {},

352 - device=self.device),

353 - }

354 -

355 - with set_forward_context(attn_metadata, self.vllm_config, 0):

356 - hidden_states = model_executable(**execute_model_kwargs)

357 -

358 - # Compute the logits.

359 - logits = self.model.compute_logits(hidden_states, sampling_metadata)

360 -

361 - # Sample the next token.

362 - output = self.model.sample(

363 - logits=logits,

364 - sampling_metadata=sampling_metadata,

365 - )

366 - return output

367 -

368 - def prepare_model_input(self, *args, **kwargs):

369 - raise NotImplementedError

370 -

371 - def make_model_input_from_broadcasted_tensor_dict(self, *args, **kwargs):

372 - raise NotImplementedError

vllm/worker/openvino_worker.py deleted

-600

@@ -1,600 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -"""An OpenVINO worker class."""

3 -from typing import Any, Dict, List, Optional, Tuple

4 -

5 -import openvino as ov

6 -import torch

7 -import torch.distributed

8 -import torch.nn as nn

9 -

10 -import vllm.envs as envs

11 -from vllm.attention import get_attn_backend

12 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,

13 - ParallelConfig, VllmConfig)

14 -from vllm.distributed import (broadcast_tensor_dict,

15 - ensure_model_parallel_initialized,

16 - init_distributed_environment)

17 -from vllm.inputs import INPUT_REGISTRY

18 -from vllm.logger import init_logger

19 -from vllm.model_executor import set_random_seed

20 -from vllm.model_executor.layers.sampler import SamplerOutput

21 -from vllm.multimodal import MULTIMODAL_REGISTRY

22 -from vllm.platforms import current_platform

23 -from vllm.sampling_params import SamplingParams

24 -from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata

25 -from vllm.utils import bind_kv_cache

26 -from vllm.worker.openvino_model_runner import OpenVINOModelRunner

27 -from vllm.worker.worker_base import LoRANotSupportedWorkerBase, WorkerBase

28 -

29 -logger = init_logger(__name__)

30 -

31 -

32 -class OpenVINOCacheEngine:

33 - """Manages the KV cache for OpenVINO backend.

34 -

35 - This class is responsible for initializing and managing CPU KV

36 - caches. It also provides methods for performing KV cache operations, such

37 - as copying.

38 - """

39 -

40 - def __init__(

41 - self,

42 - cache_config: CacheConfig,

43 - model_config: ModelConfig,

44 - parallel_config: ParallelConfig,

45 - device_config: DeviceConfig,

46 - ov_core: ov.Core,

47 - ov_device: str,

48 - ) -> None:

49 - assert device_config.device_type == "openvino"

50 - self.cache_config = cache_config

51 - self.model_config = model_config

52 - self.parallel_config = parallel_config

53 -

54 - self.head_size = model_config.get_head_size()

55 - if device_config.device.type == "cpu" and \

56 - cache_config.cache_dtype == ov.Type.u8:

57 - # Scale, zero point and quantized data will be stored together.

58 - # The layout for per token per head:

60 - # so, we have to extend head_size by 8, which is sizeof(float)

61 - # for scale and sizeof(float) for zeropoint

62 - self.head_size += 8

63 - self.num_layers = model_config.get_num_layers(parallel_config)

64 - self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)

65 -

66 - self.block_size = cache_config.block_size

67 - # Note: In CacheConfig, num_gpu_blocks actual is num_cpu_blocks

68 - # for OpenVINO backend with a CPU target device, because we want

69 - # to reuse KV cache management in the scheduler.

70 - self.num_device_blocks = cache_config.num_gpu_blocks

71 - self.num_swap_blocks = cache_config.num_cpu_blocks

72 -

73 - # Get attention backend.

74 - self.attn_backend = get_attn_backend(

75 - self.head_size,

76 - self.model_config.dtype,

77 - self.cache_config.cache_dtype,

78 - self.block_size,

79 - self.model_config.is_attention_free,

80 - )

81 -

82 - # Initialize the cache.

83 - self.kv_cache: List[Tuple[ov.Tensor,

84 - ov.Tensor]] = self._allocate_kv_cache(

85 - self.num_device_blocks, ov_core,

86 - ov_device)

87 -

88 - # Initialize the swap.

89 - self.swap_cache: List[Tuple[ov.Tensor,

90 - ov.Tensor]] = self._allocate_swap_cache(

91 - self.num_swap_blocks, ov_device)

92 -

93 - def _allocate_kv_cache(

94 - self,

95 - num_blocks: int,

96 - ov_core: ov.Core,

97 - ov_device: str,

98 - ) -> List[Tuple[ov.Tensor, ov.Tensor]]:

99 - """Allocates KV cache."""

100 - k_block_shape = v_block_shape = self.attn_backend.get_kv_cache_shape(

101 - num_blocks, self.block_size, self.num_kv_heads, self.head_size)[1:]

102 - kv_cache: List[Tuple[ov.Tensor, ov.Tensor]] = []

103 -

104 - if current_platform.is_openvino_cpu():

105 - for _ in range(self.num_layers):

106 - key_blocks = ov.Tensor(self.cache_config.cache_dtype,

107 - k_block_shape)

108 - value_blocks = ov.Tensor(self.cache_config.cache_dtype,

109 - v_block_shape)

110 - kv_cache.append((key_blocks, value_blocks))

111 - else:

112 - # Update key_cache shape:

113 - k_block_shape = (v_block_shape[0], v_block_shape[1],

114 - v_block_shape[3], v_block_shape[2])

115 -

116 - remote_context = ov_core.get_default_context(ov_device)

117 -

118 - for _ in range(self.num_layers):

119 - key_blocks = \

120 - remote_context.create_tensor(self.cache_config.cache_dtype,

121 - ov.Shape(k_block_shape),

122 - {})

123 -

124 - value_blocks = \

125 - remote_context.create_tensor(self.cache_config.cache_dtype,

126 - ov.Shape(v_block_shape),

127 - {})

128 -

129 - kv_cache.append((key_blocks, value_blocks))

130 -

131 - return kv_cache

132 -

133 - def _allocate_swap_cache(

134 - self,

135 - num_blocks: int,

136 - ov_device: str,

137 - ) -> List[Tuple[ov.Tensor, ov.Tensor]]:

138 - """Allocates swap cache."""

139 - k_block_shape = v_block_shape = self.attn_backend.get_kv_cache_shape(

140 - num_blocks, self.block_size, self.num_kv_heads, self.head_size)[1:]

141 - swap_cache: List[Tuple[ov.Tensor, ov.Tensor]] = []

142 -

143 - if num_blocks == 0:

144 - return swap_cache

145 -

146 - assert not current_platform.is_openvino_cpu(), \

147 - "CPU device isn't supposed to have swap cache"

148 -

149 - # Update key_cache shape:

150 - k_block_shape = (v_block_shape[0], v_block_shape[1], v_block_shape[3],

151 - v_block_shape[2])

152 -

153 - for _ in range(self.num_layers):

154 - key_blocks = ov.Tensor(self.cache_config.cache_dtype,

155 - k_block_shape)

156 - value_blocks = ov.Tensor(self.cache_config.cache_dtype,

157 - v_block_shape)

158 - swap_cache.append((key_blocks, value_blocks))

159 -

160 - return swap_cache

161 -

162 - def swap_in(self, src_to_dst: List[Tuple[int, int]]) -> None:

163 - for i in range(self.num_layers):

164 - for swap_tensor, kv_tensor in zip(self.swap_cache[i],

165 - self.kv_cache[i]):

166 - self.attn_backend.swap_blocks(swap_tensor, kv_tensor,

167 - src_to_dst)

168 -

169 - def swap_out(self, src_to_dst: List[Tuple[int, int]]) -> None:

170 - for i in range(self.num_layers):

171 - for swap_tensor, kv_tensor in zip(self.swap_cache[i],

172 - self.kv_cache[i]):

173 - self.attn_backend.swap_blocks(kv_tensor, swap_tensor,

174 - src_to_dst)

175 -

176 - def copy(self, src_to_dsts: List[Tuple[int, int]]) -> None:

177 - if (len(src_to_dsts) > 0):

178 - self.attn_backend.copy_blocks(self.kv_cache, src_to_dsts)

179 -

180 - @staticmethod

181 - def get_cache_block_size(

182 - block_size: int,

183 - cache_dtype: ov.Type,

184 - model_config: ModelConfig,

185 - parallel_config: ParallelConfig,

186 - ) -> int:

187 - head_size = model_config.get_head_size()

188 - num_kv_heads = model_config.get_num_kv_heads(parallel_config)

189 - num_layers = model_config.get_num_layers(parallel_config)

190 -

191 - if cache_dtype == ov.Type.u8:

192 - # Scale, zero point and quantized data will be stored together.

193 - # The layout for per token per head:

195 - # so, we have to extend head_size by 8, which is sizeof(float)

196 - # for scale and sizeof(float) for zeropoint

197 - head_size += 8

198 -

199 - key_cache_block = block_size * num_kv_heads * head_size

200 - value_cache_block = key_cache_block

201 - total = num_layers * (key_cache_block + value_cache_block)

202 - dtype_size = cache_dtype.size

203 - return dtype_size * total

204 -

205 -

206 -class OpenVINOWorker(LoRANotSupportedWorkerBase):

207 - """A worker class that executes the model on OpenVINO backend.

208 -

209 - Each worker is associated with a single OpenVINO device. The worker is

210 - responsible for maintaining the KV cache and executing the model on the

211 - OpenVINO backend.

212 - """

213 -

214 - def __init__(

215 - self,

216 - vllm_config: VllmConfig,

217 - local_rank: int,

218 - rank: int,

219 - distributed_init_method: str,

220 - is_driver_worker: bool = False,

221 - ) -> None:

222 - WorkerBase.__init__(self, vllm_config)

223 - self.ov_core = ov.Core()

224 - self.parallel_config.rank = rank

225 - self.local_rank = local_rank

226 - self.rank = rank

227 - self.distributed_init_method = distributed_init_method

228 - self.is_driver_worker = is_driver_worker

229 - if self.is_driver_worker:

230 - assert self.rank == 0, "The driver worker must have rank 0."

231 -

232 - if self.model_config.trust_remote_code:

233 - # note: lazy import to avoid importing torch before initializing

234 - from vllm.utils import init_cached_hf_modules

235 -

236 - init_cached_hf_modules()

237 - self.model_runner = OpenVINOModelRunner(

238 - self.ov_core,

239 - vllm_config=self.vllm_config,

240 - kv_cache_dtype=self.vllm_config.cache_config.cache_dtype,

241 - is_driver_worker=is_driver_worker,

242 - )

243 - # Uninitialized cache engine. Will be initialized by

244 - # initialize_cache.

245 - self.cache_engine: OpenVINOCacheEngine

246 - self.kv_cache: List[Tuple[ov.Tensor, ov.Tensor]]

247 -

248 - def init_device(self) -> None:

249 - self.init_distributed_environment()

250 - # Set random seed.

251 - set_random_seed(self.model_config.seed)

252 -

253 - def load_model(self):

254 - self.model_runner.load_model()

255 -

256 - def determine_num_available_blocks(self) -> Tuple[int, int]:

257 - """Determine the number of blocks available for the KV cache.

258 -

259 - This determines how many KV blocks can fit into the configured

260 - KV cache space.

261 - """

262 - # For OpenVINO backend, in case of CPU device, the block number will be

263 - # calculated based on the openvino_kvcache_space_bytes.

264 - cache_block_size = self.get_cache_block_size_bytes()

265 - kvcache_space_bytes = self.cache_config.openvino_kvcache_space_bytes

266 -

267 - if current_platform.is_openvino_cpu():

268 - num_device_blocks = int(kvcache_space_bytes // cache_block_size)

269 - num_swap_blocks = 0

270 - else:

271 - if kvcache_space_bytes > 0:

272 - logger.info("KV_CACHE size was explicitly configured via "

273 - "VLLM_OPENVINO_KVCACHE_SPACE environment "

274 - "variable, ignoring profiling run.")

275 - kv_cache_size = kvcache_space_bytes

276 - else:

277 - try:

278 - kv_cache_size = self.profile_run()

279 - except Exception as err:

280 - raise RuntimeError(

281 - "The error occurred during profile run. This might be "

282 - "due to insufficient GPU memory. Consider decreasing "

283 - "`max_model_len` to limit the maximum simultaneously "

284 - "processed tokens.") from err

285 -

286 - num_device_blocks = int(kv_cache_size // cache_block_size)

287 - num_swap_blocks = int(self.cache_config.swap_space_bytes //

288 - cache_block_size)

289 -

290 - return num_device_blocks, num_swap_blocks

291 -

292 - def initialize_cache(self, num_gpu_blocks: int,

293 - num_cpu_blocks: int) -> None:

294 - """Initialize the KV cache. Swappable CPU memory is only

295 - supported on GPU.

296 -

297 - For CPU, we use the num_gpu_blocks to

298 - determine how many non-swappable CPU blocks to allocate.

299 - """

300 -

301 - num_device_blocks = num_gpu_blocks

302 - num_swap_blocks = num_cpu_blocks

303 -

304 - if current_platform.is_openvino_cpu():

305 - assert (num_swap_blocks == 0

306 - ), f"{type(self)} does not support swappable cache for CPU"

307 -

308 - self._validate_num_blocks(num_device_blocks)

309 - self.cache_config.num_gpu_blocks = num_device_blocks

310 - self.cache_config.num_cpu_blocks = num_swap_blocks

311 -

312 - # Initialize the cache.

313 - self._init_cache_engine()

314 -

315 - def _validate_num_blocks(self, num_blocks: int) -> None:

316 - """Raise errors if the num_blocks is invalid."""

317 - if num_blocks <= 0:

318 - raise ValueError(

319 - "No available memory for the cache blocks. "

320 - "Try increasing `VLLM_OPENVINO_KVCACHE_SPACE` when "

321 - "initializing the engine.")

322 -

323 - max_seq_len = self.cache_config.block_size * num_blocks

324 - if self.model_config.max_model_len > max_seq_len:

325 - raise ValueError(

326 - f"The model's max seq len ({self.model_config.max_model_len}) "

327 - "is larger than the maximum number of tokens that can be "

328 - f"stored in KV cache ({max_seq_len}). Try increasing "

329 - "`VLLM_OPENVINO_KVCACHE_SPACE` or decreasing `max_model_len` "

330 - "when initializing the engine.")

331 -

332 - def _init_cache_engine(self) -> None:

333 - ov_device = envs.VLLM_OPENVINO_DEVICE

334 - self.cache_engine = OpenVINOCacheEngine(

335 - self.cache_config,

336 - self.model_config,

337 - self.parallel_config,

338 - self.device_config,

339 - self.ov_core,

340 - ov_device,

341 - )

342 - self.kv_cache = self.cache_engine.kv_cache

343 - bind_kv_cache(self.compilation_config.static_forward_context,

344 - [self.kv_cache])

345 - self.model_runner.block_size = self.cache_engine.block_size

346 -

347 - assert self.kv_cache is not None

348 -

349 - # Populate the cache to warmup the memory

350 - if current_platform.is_openvino_cpu():

351 - for key_cache, value_cache in self.kv_cache:

352 - key_cache.data[:] = 0

353 - value_cache.data[:] = 0

354 -

355 - def cache_swap_in(self, src_to_dst: List[Tuple[int, int]]) -> None:

356 - self.cache_engine.swap_in(src_to_dst)

357 -

358 - def cache_swap_out(self, src_to_dst: List[Tuple[int, int]]) -> None:

359 - self.cache_engine.swap_out(src_to_dst)

360 -

361 - def cache_copy(

362 - self,

363 - blocks_to_copy: List[Tuple[int, int]],

364 - ) -> None:

365 - self.cache_engine.copy(blocks_to_copy) # type: ignore

366 -

367 - def get_model(self) -> nn.Module:

368 - return self.model_runner.get_model()

369 -

370 - @torch.inference_mode()

371 - def execute_model(

372 - self,

373 - execute_model_req: Optional[ExecuteModelRequest] = None,

374 - ) -> List[SamplerOutput]:

375 - if execute_model_req is None:

376 - seq_group_metadata_list = None

377 - else:

378 - seq_group_metadata_list = execute_model_req.seq_group_metadata_list

379 -

380 - if self.is_driver_worker:

381 - assert seq_group_metadata_list is not None

382 - num_seq_groups: int = len(seq_group_metadata_list)

383 - assert execute_model_req is not None

384 - blocks_to_copy = execute_model_req.blocks_to_copy

385 - blocks_to_swap_in = execute_model_req.blocks_to_swap_in

386 - blocks_to_swap_out = execute_model_req.blocks_to_swap_out

387 - data: Dict[str, Any] = {

388 - "num_seq_groups": num_seq_groups,

389 - "blocks_to_copy": execute_model_req.blocks_to_copy,

390 - "blocks_to_swap_in": execute_model_req.blocks_to_swap_in,

391 - "blocks_to_swap_out": execute_model_req.blocks_to_swap_out,

392 - }

393 - broadcast_tensor_dict(data, src=0)

394 - else:

395 - data = broadcast_tensor_dict(src=0)

396 - num_seq_groups = data["num_seq_groups"]

397 - blocks_to_copy = data["blocks_to_copy"]

398 - blocks_to_swap_in = data["blocks_to_swap_in"]

399 - blocks_to_swap_out = data["blocks_to_swap_out"]

400 -

401 - if current_platform.is_openvino_cpu():

402 - assert len(execute_model_req.blocks_to_swap_in) == 0

403 - assert len(execute_model_req.blocks_to_swap_out) == 0

404 - else:

405 - self.cache_swap_in(blocks_to_swap_in)

406 - self.cache_swap_out(blocks_to_swap_out)

407 -

408 - self.cache_copy(blocks_to_copy)

409 -

410 - # If there is no input, we don't need to execute the model.

411 - if num_seq_groups == 0:

412 - return []

413 -

414 - output = self.model_runner.execute_model(seq_group_metadata_list,

415 - self.kv_cache)

416 -

417 - # OpenVINO worker only supports single-step execution.

418 - return [output]

419 -

420 - def init_distributed_environment(self) -> None:

421 - """Initialize the distributed environment."""

422 -

423 - parallel_config = self.parallel_config

424 - rank = self.rank

425 - distributed_init_method = self.distributed_init_method

426 - init_distributed_environment(

427 - world_size=parallel_config.world_size,

428 - rank=rank,

429 - distributed_init_method=distributed_init_method,

430 - backend="gloo",

431 - )

432 -

433 - # A small all_reduce for warmup.

434 - torch.distributed.all_reduce(torch.zeros(1).cpu())

435 -

436 - ensure_model_parallel_initialized(

437 - parallel_config.tensor_parallel_size,

438 - parallel_config.pipeline_parallel_size,

439 - )

440 -

441 - def get_cache_block_size_bytes(self) -> int:

442 - """Return the size in bytes of a single KV cache block."""

443 - return OpenVINOCacheEngine.get_cache_block_size(

444 - self.cache_config.block_size,

445 - self.cache_config.cache_dtype,

446 - self.model_config,

447 - self.parallel_config,

448 - )

449 -

450 - def profile_run(self) -> int:

451 - ov_device = envs.VLLM_OPENVINO_DEVICE

452 -

453 - assert not current_platform.is_openvino_cpu(), \

454 - "CPU device isn't supposed to use profile run."

455 -

456 - import openvino.properties.device as device

457 - import openvino.properties.intel_gpu as intel_gpu

458 -

459 - ov_core = self.ov_core

460 - cache_config = self.cache_config

461 - model_config = self.model_config

462 - parallel_config = self.parallel_config

463 - device_config = self.device_config

464 - input_registry = INPUT_REGISTRY

465 - mm_registry = MULTIMODAL_REGISTRY

466 - mm_registry.init_mm_limits_per_prompt(model_config)

467 -

468 - # Execute a forward pass with dummy inputs to profile the memory usage

469 - # of the model.

470 - def model_profile_run():

471 - top_k = model_config.get_vocab_size() - 1

472 - sampling_params = SamplingParams(top_p=0.99, top_k=top_k)

473 -

474 - max_num_batched_tokens = \

475 - self.scheduler_config.max_num_batched_tokens

476 - max_num_seqs = self.scheduler_config.max_num_seqs

477 - tmp_cache_config = CacheConfig(cache_config.block_size,

478 - cache_config.gpu_memory_utilization,

479 - cache_config.swap_space_bytes,

480 - "auto")

481 - tmp_cache_config.num_gpu_blocks = 1

482 - tmp_cache_config.num_cpu_blocks = 0

483 - tmp_cache_config.cache_dtype = cache_config.cache_dtype

484 -

485 - profiling_cache_engine = OpenVINOCacheEngine(

486 - tmp_cache_config, model_config, parallel_config, device_config,

487 - ov_core, ov_device)

488 -

489 - # Profile memory usage with max_num_sequences sequences and the

490 - # total # number of tokens equal to max_num_batched_tokens.

491 - seqs: List[SequenceGroupMetadata] = []

492 - for group_id in range(max_num_seqs):

493 - seq_len = (max_num_batched_tokens // max_num_seqs +

494 - (group_id < max_num_batched_tokens % max_num_seqs))

495 - block_size = cache_config.block_size

496 - seq_num_blocks = (seq_len + block_size - 1) // block_size

497 -

498 - dummy_data = input_registry \

499 - .dummy_data_for_profiling(model_config,

500 - seq_len,

501 - mm_registry)

502 -

503 - block_tables = [[0] * seq_num_blocks] * max_num_seqs

504 - seq = SequenceGroupMetadata(

505 - request_id=str(group_id),

506 - is_prompt=True,

507 - seq_data={group_id: dummy_data.seq_data},

508 - sampling_params=sampling_params,

509 - block_tables=block_tables,

510 - lora_request=None,

511 - multi_modal_data=dummy_data.multi_modal_data)

512 - seqs.append(seq)

513 -

514 - self.model_runner.block_size = tmp_cache_config.block_size

515 -

516 - bind_kv_cache(self.compilation_config.static_forward_context,

517 - profiling_cache_engine.kv_cache)

518 - # Run the model with the dummy inputs.

519 - self.model_runner.execute_model(seqs,

520 - profiling_cache_engine.kv_cache)

521 -

522 - # Explicitly revert bind_kv_cache and delete temporary KV cache

523 - # manager to free KV cache when real inputs will be passed to OV

524 - bind_kv_cache(self.compilation_config.static_forward_context, [[

525 - torch.tensor([])

526 - for _ in range(len(profiling_cache_engine.kv_cache))

527 - ]])

528 - del profiling_cache_engine

529 -

530 - logger.info(

531 - "Start profiling run with dummy inputs to evaluate "

532 - "memory usage for %s. It might take a while.", ov_device)

533 -

534 - model_profile_run()

535 -

536 - gpu_device_type = ov_core.get_property(ov_device, device.type)

537 - memory_statistics = \

538 - ov_core.get_property(ov_device, intel_gpu.memory_statistics)

539 - memory_utilization = cache_config.gpu_memory_utilization

540 -

541 - if gpu_device_type == device.Type.INTEGRATED and \

542 - memory_utilization >= 0.9:

543 - logger.warning(

544 - "iGPU is used with high gpu_memory_utilization=%f "

545 - "value. This may cause low performance due to "

546 - "occupying the majority of available system "

547 - "memory. Please consider decreasing "

548 - "gpu_memory_utilization or explicitly setting "

549 - "`VLLM_OPENVINO_KVCACHE_SPACE` (GB) environment "

550 - "variable.", memory_utilization)

551 -

552 - # sum up all used device memory

553 - device_memory_types = ["cl_mem", "usm_device"]

554 - used_device_mem = \

555 - sum(memory_statistics.get(key, 0) for key in device_memory_types)

556 -

557 - if gpu_device_type == device.Type.INTEGRATED:

558 - used_device_mem += memory_statistics.get("usm_host", 0)

559 -

560 - # there could be unaccounted extra memory reserved by kernels, kept

561 - # in memory pools, etc

562 - # therefore, add a threshold to account for this

563 - used_memory_threshold = 1.1

564 - used_device_mem *= used_memory_threshold

565 -

566 - total_device_memory = \

567 - ov_core.get_property(ov_device, intel_gpu.device_total_mem_size)

568 -

569 - def format_memory_size(size) -> str:

570 - units = ["B", "KB", "MB", "GB"]

571 - unit_index = 0

572 -

573 - while size > 1024 and unit_index < len(units) - 1:

574 - size /= 1024

575 - unit_index += 1

576 -

577 - return f"{size:.2f} {units[unit_index]}"

578 -

579 - total_device_memory_str = \

580 - format(format_memory_size(total_device_memory))

581 - used_device_memory_str = \

582 - format(format_memory_size(used_device_mem))

583 -

584 - logger.info(

585 - "Total %s memory: %s. "

586 - "Amount of memory required to run the model with "

587 - "max_num_batched_tokens=%d: %s.", ov_device,

588 - total_device_memory_str,

589 - self.scheduler_config.max_num_batched_tokens,

590 - used_device_memory_str)

591 -

592 - if used_device_mem >= total_device_memory:

593 - raise RuntimeError(

594 - f"The required memory size {used_device_memory_str} for model "

595 - "is higher than the total available device "

596 - "memory {total_device_memory_str}. Please consider to "

597 - "decrease `max_num_batched_tokens` or increase "

598 - "`gpu_memory_utilization`")

599 -

600 - return total_device_memory * memory_utilization - used_device_mem