A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Model][OpenVINO] Fix regressions from #8346 (#10045)
Signed-off-by: Peter Salas <peter@fixie.ai>
Peter Salas committed
Nov 5, 2024 at 20:19 UTC
ffc0f2b47add6e0f70e2b5d4b4aaac64ee97f8ad
3 files changed
+15
-5
.buildkite/run-openvino-test.sh
+1
-1
@@ -11,4 +11,4 @@ trap remove_docker_container EXIT
11
remove_docker_container
12
13
# Run the image and launch offline inference
14
-docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/vllm/examples/offline_inference.py
14
+docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/examples/offline_inference.py
vllm/attention/backends/openvino.py
+11
-1
@@ -1,5 +1,5 @@
1
from dataclasses import dataclass
2
-from typing import List, Tuple, Type
2
+from typing import Dict, List, Optional, Tuple, Type
3
4
import openvino as ov
5
import torch
@@ -7,6 +7,7 @@ import torch
7
from vllm.attention.backends.abstract import (AttentionBackend,
8
AttentionMetadata)
9
from vllm.attention.backends.utils import CommonAttentionState
10
+from vllm.multimodal import MultiModalPlaceholderMap
11
12
13
def copy_cache_block(src_tensor: ov.Tensor, dst_tensor: ov.Tensor,
@@ -128,3 +129,12 @@ class OpenVINOAttentionMetadata:
129
# Shape: scalar
130
# Type: i32
131
max_context_len: torch.Tensor
132
+
133
+ # The index maps that relate multi-modal embeddings to the corresponding
134
+ # placeholders.
135
+ #
136
+ # N.B. These aren't really related to attention and don't belong on this
137
+ # type -- this is just a temporary solution to make them available to
138
+ # `model_executable`.
139
+ multi_modal_placeholder_index_maps: Optional[Dict[
140
+ str, MultiModalPlaceholderMap.IndexMap]]
vllm/model_executor/models/molmo.py
+3
-3
@@ -21,8 +21,8 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
21
get_tensor_model_parallel_world_size,
22
split_tensor_along_last_dim,
23
tensor_model_parallel_all_gather)
24
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
25
- token_inputs)
24
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
25
+ InputContext, token_inputs)
26
from vllm.model_executor import SamplingMetadata
27
from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
28
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -915,7 +915,7 @@ def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
915
if "image_masks" in out:
916
dummy_imgdata["image_masks"] = out["image_masks"]
917
dummy_imgdata["seq_len"] = torch.tensor(seq_len, dtype=torch.long)
918
- return dummy_seqdata, {"image": dummy_imgdata}
918
+ return DummyData(dummy_seqdata, {"image": dummy_imgdata})
919
920
921
def pad_images(