@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ffc0f2b47a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Model][OpenVINO] Fix regressions from #8346 (#10045)

Signed-off-by: Peter Salas <peter@fixie.ai>

Peter Salas committed Nov 5, 2024 at 20:19 UTC ffc0f2b47add6e0f70e2b5d4b4aaac64ee97f8ad
3 files changed +15 -5
.buildkite/run-openvino-test.sh
+1 -1
@@ -11,4 +11,4 @@ trap remove_docker_container EXIT
11 remove_docker_container
12
13 # Run the image and launch offline inference
14 -docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/vllm/examples/offline_inference.py
14 +docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/examples/offline_inference.py
vllm/attention/backends/openvino.py
+11 -1
@@ -1,5 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import List, Tuple, Type
2 +from typing import Dict, List, Optional, Tuple, Type
3
4 import openvino as ov
5 import torch
@@ -7,6 +7,7 @@ import torch
7 from vllm.attention.backends.abstract import (AttentionBackend,
8 AttentionMetadata)
9 from vllm.attention.backends.utils import CommonAttentionState
10 +from vllm.multimodal import MultiModalPlaceholderMap
11
12
13 def copy_cache_block(src_tensor: ov.Tensor, dst_tensor: ov.Tensor,
@@ -128,3 +129,12 @@ class OpenVINOAttentionMetadata:
129 # Shape: scalar
130 # Type: i32
131 max_context_len: torch.Tensor
132 +
133 + # The index maps that relate multi-modal embeddings to the corresponding
134 + # placeholders.
135 + #
136 + # N.B. These aren't really related to attention and don't belong on this
137 + # type -- this is just a temporary solution to make them available to
138 + # `model_executable`.
139 + multi_modal_placeholder_index_maps: Optional[Dict[
140 + str, MultiModalPlaceholderMap.IndexMap]]
vllm/model_executor/models/molmo.py
+3 -3
@@ -21,8 +21,8 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
21 get_tensor_model_parallel_world_size,
22 split_tensor_along_last_dim,
23 tensor_model_parallel_all_gather)
24 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
25 - token_inputs)
24 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
25 + InputContext, token_inputs)
26 from vllm.model_executor import SamplingMetadata
27 from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
28 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -915,7 +915,7 @@ def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
915 if "image_masks" in out:
916 dummy_imgdata["image_masks"] = out["image_masks"]
917 dummy_imgdata["seq_len"] = torch.tensor(seq_len, dtype=torch.long)
918 - return dummy_seqdata, {"image": dummy_imgdata}
918 + return DummyData(dummy_seqdata, {"image": dummy_imgdata})
919
920
921 def pad_images(