A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix broadcasting logic for `multi_modal_kwargs` (#6836)
Cyrus Leung committed
Jul 31, 2024 at 10:38 UTC
f230cc2ca6614dd4eecf3af9f12c3ddbcf83036e
16 files changed
+254
-211
.buildkite/test-pipeline.yaml
+2
-3
@@ -56,7 +56,6 @@ steps:
56
fast_check: true
57
commands:
58
- pytest -v -s core
59
- - pytest -v -s distributed/test_parallel_state.py
59
60
- label: Distributed Comm Ops Test
61
#mirror_hardwares: [amd]
@@ -90,13 +89,13 @@ steps:
89
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
90
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
91
- TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
93
- - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
92
+ - TEST_DIST_MODEL=llava-hf/llava-v1.6-mistral-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
93
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
94
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
95
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
96
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
97
- TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
99
- - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
98
+ - TEST_DIST_MODEL=llava-hf/llava-v1.6-mistral-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
99
- pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
100
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
101
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
docs/source/dev/multimodal/multimodal_index.rst
+2
@@ -44,6 +44,8 @@ Base Classes
44
45
.. autodata:: vllm.multimodal.BatchedTensors
46
47
+.. autodata:: vllm.multimodal.BatchedTensorInputs
48
+
49
.. autoclass:: vllm.multimodal.MultiModalDataBuiltins
50
:members:
51
:show-inheritance:
tests/distributed/test_multimodal_broadcast.py
+5
-4
@@ -19,10 +19,10 @@ from vllm.utils import cuda_device_count_stateless
19
20
model = os.environ["TEST_DIST_MODEL"]
21
22
-if model.startswith("llava-hf/llava"):
22
+if model.startswith("llava-hf/llava-1.5"):
23
from ..models.test_llava import models, run_test
24
-elif model.startswith("microsoft/Phi-3-vision"):
25
- from ..models.test_phi3v import models, run_test
24
+elif model.startswith("llava-hf/llava-v1.6"):
25
+ from ..models.test_llava_next import models, run_test
26
else:
27
raise NotImplementedError(f"Unsupported model: {model}")
28
@@ -45,7 +45,8 @@ def test_models(hf_runner, vllm_runner, image_assets,
45
vllm_runner,
46
image_assets,
47
model=models[0],
48
- size_factors=[1.0],
48
+ # So that LLaVA-NeXT processor may return nested list
49
+ size_factors=[0.25, 0.5, 1.0],
50
dtype=dtype,
51
max_tokens=max_tokens,
52
num_logprobs=num_logprobs,
tests/distributed/test_parallel_state.py
deleted
-57
@@ -1,57 +0,0 @@
1
-from typing import Any, Dict
2
-
3
-import pytest
4
-import torch
5
-
6
-from vllm.distributed.parallel_state import (_split_tensor_dict,
7
- _update_nested_dict)
8
-
9
-
10
-def test_split_tensor_dict():
11
- test_dict = {
12
- "key_a": "a",
13
- "key_b": torch.arange(8, dtype=torch.float32),
14
- "key_c": {
15
- "key_1": torch.arange(5, dtype=torch.float32),
16
- "key_2": torch.tensor([], dtype=torch.float32),
17
- "key_3": 123,
18
- },
19
- "key_d": {},
20
- }
21
- metadata_list, tensor_list = _split_tensor_dict(test_dict)
22
- assert len(metadata_list) == 6
23
- assert torch.allclose(tensor_list[0], test_dict["key_b"])
24
- assert torch.allclose(tensor_list[1], test_dict["key_c"]["key_1"])
25
- assert torch.allclose(tensor_list[2], test_dict["key_c"]["key_2"])
26
-
27
-
28
-def test_split_tensor_dict_invalid_key():
29
- test_dict = {
30
- "a%b": "a",
31
- }
32
- with pytest.raises(AssertionError):
33
- _split_tensor_dict(test_dict)
34
-
35
-
36
-def test_update_nested_dict():
37
- flattened_keys_values = [("key1%key2%key3", "value1"),
38
- ("key1%key2%key4", "value2"),
39
- ("key1%key5", "value3"), ("key6%key7", "value4"),
40
- ("key8", "value5")]
41
- res: Dict[str, Any] = {}
42
-
43
- for flat_key, value in flattened_keys_values:
44
- _update_nested_dict(res, flat_key, value)
45
- assert res == {
46
- "key1": {
47
- "key2": {
48
- "key3": "value1",
49
- "key4": "value2"
50
- },
51
- "key5": "value3"
52
- },
53
- "key6": {
54
- "key7": "value4"
55
- },
56
- "key8": "value5"
57
- }
tests/models/test_llava_next.py
+64
-32
@@ -1,14 +1,12 @@
1
-from typing import List, Optional, Tuple
1
+from typing import List, Optional, Tuple, Type
2
3
import pytest
4
from transformers import AutoConfig, AutoTokenizer
5
6
-from vllm.model_executor.models.llava_next import (
7
- get_llava_next_image_feature_size)
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
8
11
-from ..conftest import IMAGE_ASSETS
9
+from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
from .utils import check_logprobs_close
11
12
pytestmark = pytest.mark.vlm
@@ -27,6 +25,8 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
25
26
IMAGE_TOKEN_ID = 32000
27
28
+models = ["llava-hf/llava-v1.6-vicuna-7b-hf"]
29
+
30
31
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
32
Optional[SampleLogprobs]],
@@ -50,34 +50,19 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
50
return hf_output_ids, hf_output_str, out_logprobs
51
52
53
-@pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-vicuna-7b-hf"])
54
-@pytest.mark.parametrize(
55
- "size_factors",
56
- [
57
- # No image
58
- [],
59
- # Single-scale
60
- [1.0],
61
- # Single-scale, batched
62
- [1.0, 1.0, 1.0],
63
- # Multi-scale
64
- [0.25, 0.5, 1.0],
65
- ],
66
-)
67
-@pytest.mark.parametrize("dtype", ["half"])
68
-@pytest.mark.parametrize("max_tokens", [128])
69
-@pytest.mark.parametrize("num_logprobs", [5])
70
-def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
71
- dtype, max_tokens, num_logprobs) -> None:
72
- """Inference result should be the same between hf and vllm.
73
-
74
- All the image fixtures for the test is under tests/images.
75
- For huggingface runner, we provide the PIL images as input.
76
- For vllm runner, we provide MultiModalDataDict objects
77
- and corresponding vision language config as input.
78
- Note, the text input is also adjusted to abide by vllm contract.
79
- The text output is sanitized to be able to compare with hf.
80
- """
53
+def run_test(
54
+ hf_runner: Type[HfRunner],
55
+ vllm_runner: Type[VllmRunner],
56
+ image_assets: _ImageAssets,
57
+ model: str,
58
+ *,
59
+ size_factors: List[float],
60
+ dtype: str,
61
+ max_tokens: int,
62
+ num_logprobs: int,
63
+ tensor_parallel_size: int,
64
+ distributed_executor_backend: Optional[str] = None,
65
+):
66
images = [asset.pil_image for asset in image_assets]
67
68
inputs_per_image = [(
@@ -89,6 +74,8 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
74
with vllm_runner(model,
75
dtype=dtype,
76
max_model_len=4096,
77
+ tensor_parallel_size=tensor_parallel_size,
78
+ distributed_executor_backend=distributed_executor_backend,
79
enforce_eager=True) as vllm_model:
80
vllm_outputs_per_image = [
81
vllm_model.generate_greedy_logprobs(prompts,
@@ -122,9 +109,54 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
109
)
110
111
112
+@pytest.mark.parametrize("model", models)
113
+@pytest.mark.parametrize(
114
+ "size_factors",
115
+ [
116
+ # No image
117
+ [],
118
+ # Single-scale
119
+ [1.0],
120
+ # Single-scale, batched
121
+ [1.0, 1.0, 1.0],
122
+ # Multi-scale
123
+ [0.25, 0.5, 1.0],
124
+ ],
125
+)
126
+@pytest.mark.parametrize("dtype", ["half"])
127
+@pytest.mark.parametrize("max_tokens", [128])
128
+@pytest.mark.parametrize("num_logprobs", [5])
129
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
130
+ dtype, max_tokens, num_logprobs) -> None:
131
+ """Inference result should be the same between hf and vllm.
132
+
133
+ All the image fixtures for the test is under tests/images.
134
+ For huggingface runner, we provide the PIL images as input.
135
+ For vllm runner, we provide MultiModalDataDict objects
136
+ and corresponding vision language config as input.
137
+ Note, the text input is also adjusted to abide by vllm contract.
138
+ The text output is sanitized to be able to compare with hf.
139
+ """
140
+ run_test(
141
+ hf_runner,
142
+ vllm_runner,
143
+ image_assets,
144
+ model,
145
+ size_factors=size_factors,
146
+ dtype=dtype,
147
+ max_tokens=max_tokens,
148
+ num_logprobs=num_logprobs,
149
+ tensor_parallel_size=1,
150
+ )
151
+
152
+
153
@pytest.mark.parametrize("height_and_width_and_result", [(1669, 2560, 2144),
154
(183, 488, 776)])
155
def test_image_feature_size(height_and_width_and_result):
156
+ # Avoid initializing CUDA too early in distributed tests
157
+ from vllm.model_executor.models.llava_next import (
158
+ get_llava_next_image_feature_size)
159
+
160
height, width, result = height_and_width_and_result
161
config = AutoConfig.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf")
162
assert get_llava_next_image_feature_size(config,
vllm/distributed/parallel_state.py
+11
-35
@@ -45,22 +45,16 @@ TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
45
46
47
def _split_tensor_dict(
48
- tensor_dict: Dict[str, Union[torch.Tensor, Any]],
49
- prefix: str = "") -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
48
+ tensor_dict: Dict[str, Union[torch.Tensor, Any]]
49
+) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
50
"""Split the tensor dictionary into two parts:
51
1. A list of (key, value) pairs. If the value is a tensor, it is replaced
52
by its metadata.
53
2. A list of tensors.
54
-
55
- If the Tensor is nested under `tensor_dict["key1"]["key2"]`, the key of its
56
- metadata will be "key1%key2".
54
"""
55
metadata_list: List[Tuple[str, Any]] = []
59
- tensor_list = []
56
+ tensor_list: List[torch.Tensor] = []
57
for key, value in tensor_dict.items():
61
- assert "%" not in key, (
62
- "Avoid having '%' in key "
63
- "as it is used as a separator for nested entries.")
58
if isinstance(value, torch.Tensor):
59
# Note: we cannot use `value.device` here,
60
# because it contains not only the device type but also the device
@@ -68,31 +62,13 @@ def _split_tensor_dict(
62
# receiving side will set the device index.
63
device = value.device.type
64
metadata_list.append(
71
- (prefix + key, TensorMetadata(device, value.dtype,
72
- value.size())))
65
+ (key, TensorMetadata(device, value.dtype, value.size())))
66
tensor_list.append(value)
74
- elif isinstance(value, dict):
75
- if len(value) == 0:
76
- metadata_list.append((prefix + key, value))
77
- inner_metadata_list, inner_tensor_list = _split_tensor_dict(
78
- value, prefix + key + "%")
79
- metadata_list.extend(inner_metadata_list)
80
- tensor_list.extend(inner_tensor_list)
67
else:
82
- metadata_list.append((prefix + key, value))
68
+ metadata_list.append((key, value))
69
return metadata_list, tensor_list
70
71
86
-def _update_nested_dict(nested_dict, flattened_key, value):
87
- key_splits = flattened_key.split("%")
88
- cur_dict = nested_dict
89
- for k in key_splits[:-1]:
90
- if k not in cur_dict:
91
- cur_dict[k] = {}
92
- cur_dict = cur_dict[k]
93
- cur_dict[key_splits[-1]] = value
94
-
95
-
72
class GroupCoordinator:
73
"""
74
PyTorch ProcessGroup wrapper for a group of processes.
@@ -566,7 +542,7 @@ class GroupCoordinator:
542
device=value.device)
543
if tensor.numel() == 0:
544
# Skip broadcasting empty tensors.
569
- _update_nested_dict(tensor_dict, key, tensor)
545
+ tensor_dict[key] = tensor
546
continue
547
if tensor.is_cpu:
548
# use metadata_group for CPU tensors
@@ -583,9 +559,9 @@ class GroupCoordinator:
559
group=group,
560
async_op=True)
561
async_handles.append(handle)
586
- _update_nested_dict(tensor_dict, key, tensor)
562
+ tensor_dict[key] = tensor
563
else:
588
- _update_nested_dict(tensor_dict, key, value)
564
+ tensor_dict[key] = value
565
for async_handle in async_handles:
566
async_handle.wait()
567
return tensor_dict
@@ -661,7 +637,7 @@ class GroupCoordinator:
637
device=value.device)
638
if tensor.numel() == 0:
639
# Skip broadcasting empty tensors.
664
- _update_nested_dict(tensor_dict, key, tensor)
640
+ tensor_dict[key] = tensor
641
continue
642
if tensor.is_cpu:
643
# use metadata_group for CPU tensors
@@ -673,9 +649,9 @@ class GroupCoordinator:
649
torch.distributed.recv(tensor,
650
src=self.ranks[src],
651
group=group)
676
- _update_nested_dict(tensor_dict, key, tensor)
652
+ tensor_dict[key] = tensor
653
else:
678
- _update_nested_dict(tensor_dict, key, value)
654
+ tensor_dict[key] = value
655
return tensor_dict
656
657
def barrier(self):
vllm/multimodal/__init__.py
+4
-2
@@ -1,5 +1,6 @@
1
-from .base import (BatchedTensors, MultiModalDataBuiltins, MultiModalDataDict,
2
- MultiModalInputs, MultiModalPlugin, NestedTensors)
1
+from .base import (BatchedTensorInputs, BatchedTensors, MultiModalDataBuiltins,
2
+ MultiModalDataDict, MultiModalInputs, MultiModalPlugin,
3
+ NestedTensors)
4
from .registry import MultiModalRegistry
5
6
MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -12,6 +13,7 @@ See also:
13
"""
14
15
__all__ = [
16
+ "BatchedTensorInputs",
17
"BatchedTensors",
18
"MultiModalDataBuiltins",
19
"MultiModalDataDict",
vllm/multimodal/base.py
+42
-20
@@ -9,10 +9,12 @@ import torch
9
import torch.types
10
from PIL import Image
11
from torch import nn
12
+from typing_extensions import TypeAlias
13
14
from vllm.config import ModelConfig
15
from vllm.inputs import InputContext
16
from vllm.logger import init_logger
17
+from vllm.utils import JSONTree, json_map_leaves
18
19
logger = init_logger(__name__)
20
@@ -22,11 +24,16 @@ Use a list instead of a tensor if the dimensions of each element do not match.
24
Currently only supports up to singly nested list of tensors.
25
"""
26
25
-BatchedTensors = Union[GenericSequence[NestedTensors], NestedTensors]
27
+BatchedTensors: TypeAlias = JSONTree[torch.Tensor]
28
"""
27
-If each input tensor in the batch has the same size, this is a single batched
28
-tensor; otherwise, this is a list of :class:`NestedTensors` with one element
29
-per item in the batch.
29
+A nested JSON structure of tensors which have been batched via
30
+:meth:`MultiModalInputs.batch`.
31
+"""
32
+
33
+BatchedTensorInputs: TypeAlias = Dict[str, JSONTree[torch.Tensor]]
34
+"""
35
+A dictionary containing nested tensors which have been batched via
36
+:meth:`MultiModalInputs.batch`.
37
"""
38
39
if sys.version_info < (3, 9):
@@ -46,14 +53,17 @@ class MultiModalInputs(_MultiModalInputsBase):
53
"""
54
55
@staticmethod
49
- def try_concat(
56
+ def _try_concat(
57
tensors: List[NestedTensors],
51
- *,
52
- device: torch.types.Device,
53
- ) -> BatchedTensors:
58
+ ) -> Union[GenericSequence[NestedTensors], NestedTensors]:
59
+ """
60
+ If each input tensor in the batch has the same shape, return a single
61
+ batched tensor; otherwise, return a list of :class:`NestedTensors` with
62
+ one element per item in the batch.
63
+ """
64
# may be list rather than tensors
65
if isinstance(tensors[0], list):
56
- return [[t.to(device=device) for t in tensor[0]]
66
+ return [[t for t in tensor[0]]
67
for tensor in cast(List[List[torch.Tensor]], tensors)]
68
69
tensors_ = cast(List[torch.Tensor], tensors)
@@ -62,18 +72,21 @@ class MultiModalInputs(_MultiModalInputsBase):
72
73
for tensor in tensors_:
74
if tensor.shape[1:] != unbatched_shape:
65
- return [
66
- tensor.squeeze(0).to(device=device) for tensor in tensors_
67
- ]
75
+ return [tensor.squeeze(0) for tensor in tensors_]
76
69
- return torch.cat(tensors_, dim=0).to(device=device)
77
+ return torch.cat(tensors_, dim=0)
78
79
@staticmethod
72
- def batch(
73
- inputs_list: List["MultiModalInputs"],
74
- device: torch.types.Device,
75
- ) -> Dict[str, BatchedTensors]:
76
- """Batch multiple inputs together into a dictionary."""
80
+ def batch(inputs_list: List["MultiModalInputs"]) -> BatchedTensorInputs:
81
+ """
82
+ Batch multiple inputs together into a dictionary.
83
+
84
+ The resulting dictionary has the same keys as the inputs.
85
+ If the corresponding value from each input is a tensor and they all
86
+ share the same shape, the output value is a single batched tensor;
87
+ otherwise, the output value is a list containing the original value
88
+ from each input.
89
+ """
90
if len(inputs_list) == 0:
91
return {}
92
@@ -90,9 +103,18 @@ class MultiModalInputs(_MultiModalInputsBase):
103
item_lists[k].append(v)
104
105
return {
93
- k: MultiModalInputs.try_concat(item_list, device=device)
106
+ k: MultiModalInputs._try_concat(item_list)
107
for k, item_list in item_lists.items()
95
- }
108
+ } # type: ignore
109
+
110
+ @staticmethod
111
+ def as_kwargs(
112
+ batched_inputs: BatchedTensorInputs,
113
+ *,
114
+ device: torch.types.Device,
115
+ ) -> BatchedTensorInputs:
116
+ return json_map_leaves(lambda x: x.to(device, non_blocking=True),
117
+ batched_inputs)
118
119
120
class MultiModalDataBuiltins(TypedDict, total=False):
vllm/spec_decode/draft_model_runner.py
+3
-1
@@ -15,6 +15,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15
ModelConfig, MultiModalConfig, ParallelConfig,
16
PromptAdapterConfig, SchedulerConfig)
17
from vllm.logger import init_logger
18
+from vllm.multimodal import MultiModalInputs
19
from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
20
SamplerOutput)
21
from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
@@ -323,7 +324,8 @@ class TP1DraftModelRunner(ModelRunner):
324
kv_caches=kv_caches,
325
attn_metadata=model_input.attn_metadata,
326
intermediate_tensors=intermediate_tensors,
326
- **multi_modal_kwargs,
327
+ **MultiModalInputs.as_kwargs(multi_modal_kwargs,
328
+ device=self.device),
329
)
330
331
# Compute the logits.
vllm/utils.py
+50
-1
@@ -17,7 +17,7 @@ from functools import lru_cache, partial, wraps
17
from platform import uname
18
from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
19
Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
20
- Union)
20
+ Union, overload)
21
22
import numpy as np
23
import numpy.typing as npt
@@ -53,6 +53,7 @@ TORCH_DTYPE_TO_NUMPY_DTYPE = {
53
P = ParamSpec('P')
54
K = TypeVar("K")
55
T = TypeVar("T")
56
+U = TypeVar("U")
57
58
59
class _Sentinel:
@@ -712,6 +713,54 @@ def merge_dicts(dict1: Dict[K, List[T]],
713
return dict(merged_dict)
714
715
716
+JSONTree = Union[Dict[str, "JSONTree[T]"], List["JSONTree[T]"],
717
+ Tuple["JSONTree[T]", ...], T]
718
+"""A nested JSON structure where the leaves need not be JSON-serializable."""
719
+
720
+
721
+@overload
722
+def json_map_leaves(
723
+ func: Callable[[T], U],
724
+ value: Dict[str, JSONTree[T]],
725
+) -> Dict[str, JSONTree[U]]:
726
+ ...
727
+
728
+
729
+@overload
730
+def json_map_leaves(
731
+ func: Callable[[T], U],
732
+ value: List[JSONTree[T]],
733
+) -> List[JSONTree[U]]:
734
+ ...
735
+
736
+
737
+@overload
738
+def json_map_leaves(
739
+ func: Callable[[T], U],
740
+ value: Tuple[JSONTree[T], ...],
741
+) -> Tuple[JSONTree[U], ...]:
742
+ ...
743
+
744
+
745
+@overload
746
+def json_map_leaves(
747
+ func: Callable[[T], U],
748
+ value: JSONTree[T],
749
+) -> JSONTree[U]:
750
+ ...
751
+
752
+
753
+def json_map_leaves(func: Callable[[T], U], value: JSONTree[T]) -> JSONTree[U]:
754
+ if isinstance(value, dict):
755
+ return {k: json_map_leaves(func, v) for k, v in value.items()}
756
+ elif isinstance(value, list):
757
+ return [json_map_leaves(func, v) for v in value]
758
+ elif isinstance(value, tuple):
759
+ return tuple(json_map_leaves(func, v) for v in value)
760
+ else:
761
+ return func(value)
762
+
763
+
764
def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
765
"""Flatten a list of lists to a single list."""
766
return [item for sublist in lists for item in sublist]
vllm/worker/cpu_model_runner.py
+15
-12
@@ -1,6 +1,5 @@
1
from dataclasses import dataclass
2
-from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
3
- Type, Union)
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
3
4
import torch
5
from torch import nn
@@ -12,7 +11,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
from vllm.logger import init_logger
12
from vllm.model_executor import SamplingMetadata
13
from vllm.model_executor.model_loader import get_model
15
-from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
14
+from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
15
MultiModalInputs)
16
from vllm.sequence import (IntermediateTensors, SamplerOutput,
17
SequenceGroupMetadata)
@@ -41,7 +40,7 @@ class CPUModelInput(ModelRunnerInputBase):
40
input_positions: Optional[torch.Tensor] = None
41
attn_metadata: Optional["AttentionMetadata"] = None
42
sampling_metadata: Optional["SamplingMetadata"] = None
44
- multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
43
+ multi_modal_kwargs: Optional[BatchedTensorInputs] = None
44
virtual_engine: Optional[int] = None
45
46
def as_broadcastable_tensor_dict(
@@ -136,7 +135,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
135
self,
136
seq_group_metadata_list: List[SequenceGroupMetadata],
137
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
139
- Mapping[str, BatchedTensors]]:
138
+ BatchedTensorInputs]:
139
assert len(seq_group_metadata_list) > 0
140
input_tokens: List[int] = []
141
input_positions: List[int] = []
@@ -214,8 +213,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
213
slot_mapping=slot_mapping,
214
)
215
217
- multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
218
- device=self.device)
216
+ multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
217
218
return (input_tokens, input_positions, attn_metadata, seq_lens,
219
multi_modal_kwargs)
@@ -361,11 +359,16 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
359
360
model_executable = self.model
361
execute_model_kwargs = {
364
- "input_ids": model_input.input_tokens,
365
- "positions": model_input.input_positions,
366
- "kv_caches": kv_caches,
367
- "attn_metadata": model_input.attn_metadata,
368
- **(model_input.multi_modal_kwargs or {}),
362
+ "input_ids":
363
+ model_input.input_tokens,
364
+ "positions":
365
+ model_input.input_positions,
366
+ "kv_caches":
367
+ kv_caches,
368
+ "attn_metadata":
369
+ model_input.attn_metadata,
370
+ **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
371
+ device=self.device),
372
}
373
374
hidden_states = model_executable(**execute_model_kwargs)
vllm/worker/embedding_model_runner.py
+11
-5
@@ -8,6 +8,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
8
PromptAdapterConfig, SchedulerConfig)
9
from vllm.logger import init_logger
10
from vllm.model_executor.pooling_metadata import PoolingMetadata
11
+from vllm.multimodal import MultiModalInputs
12
from vllm.pooling_params import PoolingParams
13
from vllm.sequence import (IntermediateTensors, PoolerOutput, SequenceData,
14
SequenceGroupMetadata)
@@ -99,11 +100,16 @@ class EmbeddingModelRunner(
100
kv_caches = [None] * num_layers
101
102
execute_model_kwargs = {
102
- "input_ids": model_input.input_tokens,
103
- "positions": model_input.input_positions,
104
- "kv_caches": kv_caches,
105
- "attn_metadata": model_input.attn_metadata,
106
- **(model_input.multi_modal_kwargs or {}),
103
+ "input_ids":
104
+ model_input.input_tokens,
105
+ "positions":
106
+ model_input.input_positions,
107
+ "kv_caches":
108
+ kv_caches,
109
+ "attn_metadata":
110
+ model_input.attn_metadata,
111
+ **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
112
+ device=self.device),
113
}
114
115
hidden_states = model_executable(**execute_model_kwargs)
vllm/worker/model_runner.py
+7
-7
@@ -4,8 +4,8 @@ import time
4
import warnings
5
import weakref
6
from dataclasses import dataclass
7
-from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
8
- Tuple, Type, TypeVar, Union)
7
+from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type,
8
+ TypeVar, Union)
9
10
import numpy as np
11
import torch
@@ -40,7 +40,7 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
40
from vllm.model_executor.models.interfaces import (supports_lora,
41
supports_vision)
42
from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
43
-from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
43
+from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
44
MultiModalInputs)
45
from vllm.prompt_adapter.layers import PromptAdapterMapping
46
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -94,7 +94,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
94
attn_metadata: Optional["AttentionMetadata"] = None
95
prompt_adapter_mapping: Optional[PromptAdapterMapping] = None
96
prompt_adapter_requests: Optional[Set[PromptAdapterRequest]] = None
97
- multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
97
+ multi_modal_kwargs: Optional[BatchedTensorInputs] = None
98
request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
99
finished_requests_ids: Optional[List[str]] = None
100
virtual_engine: int = 0
@@ -608,8 +608,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
608
data.multi_modal_inputs for data in self.inter_data_list
609
if data.multi_modal_inputs is not None
610
]
611
- multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
612
- device=self.runner.device)
611
+ multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
612
613
return self.model_input_cls(
614
input_tokens=input_tokens_tensor,
@@ -1361,7 +1360,8 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1360
kv_caches=kv_caches,
1361
attn_metadata=model_input.attn_metadata,
1362
intermediate_tensors=intermediate_tensors,
1364
- **multi_modal_kwargs,
1363
+ **MultiModalInputs.as_kwargs(multi_modal_kwargs,
1364
+ device=self.device),
1365
**seqlen_agnostic_kwargs)
1366
1367
# Compute the logits in the last pipeline stage.
vllm/worker/neuron_model_runner.py
+8
-9
@@ -1,6 +1,5 @@
1
from dataclasses import dataclass
2
-from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
3
- Union)
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
3
4
import torch
5
from torch import nn
@@ -10,7 +9,7 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
9
from vllm.logger import init_logger
10
from vllm.model_executor import SamplingMetadata
11
from vllm.model_executor.model_loader.neuron import get_neuron_model
13
-from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
12
+from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
13
MultiModalInputs)
14
from vllm.sequence import (IntermediateTensors, SamplerOutput,
15
SequenceGroupMetadata)
@@ -32,7 +31,7 @@ class ModelInputForNeuron(ModelRunnerInputBase):
31
input_positions: Optional[torch.Tensor] = None
32
input_block_ids: Optional[torch.Tensor] = None
33
sampling_metadata: Optional["SamplingMetadata"] = None
35
- multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
34
+ multi_modal_kwargs: Optional[BatchedTensorInputs] = None
35
36
def as_broadcastable_tensor_dict(
37
self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -84,8 +83,8 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
83
def _prepare_prompt(
84
self,
85
seq_group_metadata_list: List[SequenceGroupMetadata],
87
- ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[int], Mapping[
88
- str, BatchedTensors]]:
86
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[int],
87
+ BatchedTensorInputs]:
88
assert len(seq_group_metadata_list) > 0
89
input_tokens: List[List[int]] = []
90
input_positions: List[List[int]] = []
@@ -134,8 +133,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
133
dtype=torch.long,
134
device=self.device)
135
137
- multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
138
- device=self.device)
136
+ multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
137
138
return (input_tokens, input_positions, input_block_ids, seq_lens,
139
multi_modal_kwargs)
@@ -244,7 +242,8 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
242
input_ids=model_input.input_tokens,
243
positions=model_input.input_positions,
244
input_block_ids=model_input.input_block_ids,
247
- **(model_input.multi_modal_kwargs or {}),
245
+ **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
246
+ device=self.device),
247
)
248
249
# Compute the logits.
vllm/worker/openvino_model_runner.py
+15
-11
@@ -1,4 +1,4 @@
1
-from typing import List, Mapping, NamedTuple, Optional, Tuple
1
+from typing import List, NamedTuple, Optional, Tuple
2
3
import openvino as ov
4
import torch
@@ -12,7 +12,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.model_loader.openvino import get_model
15
-from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
15
+from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16
MultiModalInputs)
17
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
18
@@ -25,7 +25,7 @@ class ModelInput(NamedTuple):
25
attn_metadata: Optional[OpenVINOAttentionMetadata]
26
seq_lens: List[int]
27
query_lens: List[int]
28
- multi_modal_kwargs: Mapping[str, BatchedTensors]
28
+ multi_modal_kwargs: BatchedTensorInputs
29
30
@classmethod
31
def empty(cls, device):
@@ -265,8 +265,7 @@ class OpenVINOModelRunner:
265
max_context_len=max_context_len_tensor,
266
)
267
268
- multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
269
- device=self.device)
268
+ multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
269
270
return ModelInput(
271
input_tokens,
@@ -281,7 +280,7 @@ class OpenVINOModelRunner:
280
self,
281
seq_group_metadata_list: List[SequenceGroupMetadata],
282
) -> Tuple[torch.Tensor, torch.Tensor, OpenVINOAttentionMetadata,
284
- SamplingMetadata, Mapping[str, BatchedTensors]]:
283
+ SamplingMetadata, BatchedTensorInputs]:
284
# Prepare input tensors.
285
(
286
input_tokens,
@@ -324,11 +323,16 @@ class OpenVINOModelRunner:
323
324
model_executable = self.model
325
execute_model_kwargs = {
327
- "input_ids": input_tokens,
328
- "positions": input_positions,
329
- "kv_caches": kv_caches,
330
- "attn_metadata": attn_metadata,
331
- **(multi_modal_kwargs or {}),
326
+ "input_ids":
327
+ input_tokens,
328
+ "positions":
329
+ input_positions,
330
+ "kv_caches":
331
+ kv_caches,
332
+ "attn_metadata":
333
+ attn_metadata,
334
+ **MultiModalInputs.as_kwargs(multi_modal_kwargs or {},
335
+ device=self.device),
336
}
337
338
hidden_states = model_executable(**execute_model_kwargs)
vllm/worker/xpu_model_runner.py
+15
-12
@@ -1,6 +1,5 @@
1
from dataclasses import dataclass
2
-from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
3
- Type, Union)
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
3
4
import torch
5
import torch.nn as nn
@@ -14,7 +13,7 @@ from vllm.inputs import INPUT_REGISTRY
13
from vllm.logger import init_logger
14
from vllm.model_executor.model_loader import get_model
15
from vllm.model_executor.models.interfaces import supports_vision
17
-from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
16
+from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17
MultiModalInputs)
18
from vllm.sampling_params import SamplingParams
19
from vllm.sequence import (IntermediateTensors, SamplerOutput,
@@ -49,7 +48,7 @@ class ModelInputForXPU(ModelRunnerInputBase):
48
input_positions: Optional[torch.Tensor] = None
49
attn_metadata: Optional["AttentionMetadata"] = None
50
sampling_metadata: Optional["SamplingMetadata"] = None
52
- multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
51
+ multi_modal_kwargs: Optional[BatchedTensorInputs] = None
52
53
def as_broadcastable_tensor_dict(
54
self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -376,11 +375,16 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
375
376
model_executable = self.model
377
execute_model_kwargs = {
379
- "input_ids": model_input.input_tokens,
380
- "positions": model_input.input_positions,
381
- "kv_caches": kv_caches,
382
- "attn_metadata": model_input.attn_metadata,
383
- **(model_input.multi_modal_kwargs or {}),
378
+ "input_ids":
379
+ model_input.input_tokens,
380
+ "positions":
381
+ model_input.input_positions,
382
+ "kv_caches":
383
+ kv_caches,
384
+ "attn_metadata":
385
+ model_input.attn_metadata,
386
+ **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
387
+ device=self.device),
388
}
389
390
hidden_states = model_executable(**execute_model_kwargs)
@@ -404,7 +408,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
408
self,
409
seq_group_metadata_list: List[SequenceGroupMetadata],
410
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
407
- Mapping[str, BatchedTensors]]:
411
+ BatchedTensorInputs]:
412
assert len(seq_group_metadata_list) > 0
413
input_tokens: List[int] = []
414
input_positions: List[int] = []
@@ -496,8 +500,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
500
block_tables=torch.tensor([], device=self.device, dtype=torch.int),
501
)
502
499
- multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
500
- device=self.device)
503
+ multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
504
505
return (input_tokens, input_positions, attn_metadata, seq_lens,
506
multi_modal_kwargs)