@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f230cc2ca6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix broadcasting logic for `multi_modal_kwargs` (#6836)

Cyrus Leung committed Jul 31, 2024 at 10:38 UTC f230cc2ca6614dd4eecf3af9f12c3ddbcf83036e
16 files changed +254 -211
.buildkite/test-pipeline.yaml
+2 -3
@@ -56,7 +56,6 @@ steps:
56 fast_check: true
57 commands:
58 - pytest -v -s core
59 - - pytest -v -s distributed/test_parallel_state.py
59
60 - label: Distributed Comm Ops Test
61 #mirror_hardwares: [amd]
@@ -90,13 +89,13 @@ steps:
89 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
90 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
91 - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
93 - - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
92 + - TEST_DIST_MODEL=llava-hf/llava-v1.6-mistral-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
93 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
94 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
95 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
96 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
97 - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
99 - - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
98 + - TEST_DIST_MODEL=llava-hf/llava-v1.6-mistral-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
99 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
100 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
101 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
docs/source/dev/multimodal/multimodal_index.rst
+2
@@ -44,6 +44,8 @@ Base Classes
44
45 .. autodata:: vllm.multimodal.BatchedTensors
46
47 +.. autodata:: vllm.multimodal.BatchedTensorInputs
48 +
49 .. autoclass:: vllm.multimodal.MultiModalDataBuiltins
50 :members:
51 :show-inheritance:
tests/distributed/test_multimodal_broadcast.py
+5 -4
@@ -19,10 +19,10 @@ from vllm.utils import cuda_device_count_stateless
19
20 model = os.environ["TEST_DIST_MODEL"]
21
22 -if model.startswith("llava-hf/llava"):
22 +if model.startswith("llava-hf/llava-1.5"):
23 from ..models.test_llava import models, run_test
24 -elif model.startswith("microsoft/Phi-3-vision"):
25 - from ..models.test_phi3v import models, run_test
24 +elif model.startswith("llava-hf/llava-v1.6"):
25 + from ..models.test_llava_next import models, run_test
26 else:
27 raise NotImplementedError(f"Unsupported model: {model}")
28
@@ -45,7 +45,8 @@ def test_models(hf_runner, vllm_runner, image_assets,
45 vllm_runner,
46 image_assets,
47 model=models[0],
48 - size_factors=[1.0],
48 + # So that LLaVA-NeXT processor may return nested list
49 + size_factors=[0.25, 0.5, 1.0],
50 dtype=dtype,
51 max_tokens=max_tokens,
52 num_logprobs=num_logprobs,
tests/distributed/test_parallel_state.py deleted
-57
@@ -1,57 +0,0 @@
1 -from typing import Any, Dict
2 -
3 -import pytest
4 -import torch
5 -
6 -from vllm.distributed.parallel_state import (_split_tensor_dict,
7 - _update_nested_dict)
8 -
9 -
10 -def test_split_tensor_dict():
11 - test_dict = {
12 - "key_a": "a",
13 - "key_b": torch.arange(8, dtype=torch.float32),
14 - "key_c": {
15 - "key_1": torch.arange(5, dtype=torch.float32),
16 - "key_2": torch.tensor([], dtype=torch.float32),
17 - "key_3": 123,
18 - },
19 - "key_d": {},
20 - }
21 - metadata_list, tensor_list = _split_tensor_dict(test_dict)
22 - assert len(metadata_list) == 6
23 - assert torch.allclose(tensor_list[0], test_dict["key_b"])
24 - assert torch.allclose(tensor_list[1], test_dict["key_c"]["key_1"])
25 - assert torch.allclose(tensor_list[2], test_dict["key_c"]["key_2"])
26 -
27 -
28 -def test_split_tensor_dict_invalid_key():
29 - test_dict = {
30 - "a%b": "a",
31 - }
32 - with pytest.raises(AssertionError):
33 - _split_tensor_dict(test_dict)
34 -
35 -
36 -def test_update_nested_dict():
37 - flattened_keys_values = [("key1%key2%key3", "value1"),
38 - ("key1%key2%key4", "value2"),
39 - ("key1%key5", "value3"), ("key6%key7", "value4"),
40 - ("key8", "value5")]
41 - res: Dict[str, Any] = {}
42 -
43 - for flat_key, value in flattened_keys_values:
44 - _update_nested_dict(res, flat_key, value)
45 - assert res == {
46 - "key1": {
47 - "key2": {
48 - "key3": "value1",
49 - "key4": "value2"
50 - },
51 - "key5": "value3"
52 - },
53 - "key6": {
54 - "key7": "value4"
55 - },
56 - "key8": "value5"
57 - }
tests/models/test_llava_next.py
+64 -32
@@ -1,14 +1,12 @@
1 -from typing import List, Optional, Tuple
1 +from typing import List, Optional, Tuple, Type
2
3 import pytest
4 from transformers import AutoConfig, AutoTokenizer
5
6 -from vllm.model_executor.models.llava_next import (
7 - get_llava_next_image_feature_size)
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.sequence import SampleLogprobs
8
11 -from ..conftest import IMAGE_ASSETS
9 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10 from .utils import check_logprobs_close
11
12 pytestmark = pytest.mark.vlm
@@ -27,6 +25,8 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
25
26 IMAGE_TOKEN_ID = 32000
27
28 +models = ["llava-hf/llava-v1.6-vicuna-7b-hf"]
29 +
30
31 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
32 Optional[SampleLogprobs]],
@@ -50,34 +50,19 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
50 return hf_output_ids, hf_output_str, out_logprobs
51
52
53 -@pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-vicuna-7b-hf"])
54 -@pytest.mark.parametrize(
55 - "size_factors",
56 - [
57 - # No image
58 - [],
59 - # Single-scale
60 - [1.0],
61 - # Single-scale, batched
62 - [1.0, 1.0, 1.0],
63 - # Multi-scale
64 - [0.25, 0.5, 1.0],
65 - ],
66 -)
67 -@pytest.mark.parametrize("dtype", ["half"])
68 -@pytest.mark.parametrize("max_tokens", [128])
69 -@pytest.mark.parametrize("num_logprobs", [5])
70 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
71 - dtype, max_tokens, num_logprobs) -> None:
72 - """Inference result should be the same between hf and vllm.
73 -
74 - All the image fixtures for the test is under tests/images.
75 - For huggingface runner, we provide the PIL images as input.
76 - For vllm runner, we provide MultiModalDataDict objects
77 - and corresponding vision language config as input.
78 - Note, the text input is also adjusted to abide by vllm contract.
79 - The text output is sanitized to be able to compare with hf.
80 - """
53 +def run_test(
54 + hf_runner: Type[HfRunner],
55 + vllm_runner: Type[VllmRunner],
56 + image_assets: _ImageAssets,
57 + model: str,
58 + *,
59 + size_factors: List[float],
60 + dtype: str,
61 + max_tokens: int,
62 + num_logprobs: int,
63 + tensor_parallel_size: int,
64 + distributed_executor_backend: Optional[str] = None,
65 +):
66 images = [asset.pil_image for asset in image_assets]
67
68 inputs_per_image = [(
@@ -89,6 +74,8 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
74 with vllm_runner(model,
75 dtype=dtype,
76 max_model_len=4096,
77 + tensor_parallel_size=tensor_parallel_size,
78 + distributed_executor_backend=distributed_executor_backend,
79 enforce_eager=True) as vllm_model:
80 vllm_outputs_per_image = [
81 vllm_model.generate_greedy_logprobs(prompts,
@@ -122,9 +109,54 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
109 )
110
111
112 +@pytest.mark.parametrize("model", models)
113 +@pytest.mark.parametrize(
114 + "size_factors",
115 + [
116 + # No image
117 + [],
118 + # Single-scale
119 + [1.0],
120 + # Single-scale, batched
121 + [1.0, 1.0, 1.0],
122 + # Multi-scale
123 + [0.25, 0.5, 1.0],
124 + ],
125 +)
126 +@pytest.mark.parametrize("dtype", ["half"])
127 +@pytest.mark.parametrize("max_tokens", [128])
128 +@pytest.mark.parametrize("num_logprobs", [5])
129 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
130 + dtype, max_tokens, num_logprobs) -> None:
131 + """Inference result should be the same between hf and vllm.
132 +
133 + All the image fixtures for the test is under tests/images.
134 + For huggingface runner, we provide the PIL images as input.
135 + For vllm runner, we provide MultiModalDataDict objects
136 + and corresponding vision language config as input.
137 + Note, the text input is also adjusted to abide by vllm contract.
138 + The text output is sanitized to be able to compare with hf.
139 + """
140 + run_test(
141 + hf_runner,
142 + vllm_runner,
143 + image_assets,
144 + model,
145 + size_factors=size_factors,
146 + dtype=dtype,
147 + max_tokens=max_tokens,
148 + num_logprobs=num_logprobs,
149 + tensor_parallel_size=1,
150 + )
151 +
152 +
153 @pytest.mark.parametrize("height_and_width_and_result", [(1669, 2560, 2144),
154 (183, 488, 776)])
155 def test_image_feature_size(height_and_width_and_result):
156 + # Avoid initializing CUDA too early in distributed tests
157 + from vllm.model_executor.models.llava_next import (
158 + get_llava_next_image_feature_size)
159 +
160 height, width, result = height_and_width_and_result
161 config = AutoConfig.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf")
162 assert get_llava_next_image_feature_size(config,
vllm/distributed/parallel_state.py
+11 -35
@@ -45,22 +45,16 @@ TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
45
46
47 def _split_tensor_dict(
48 - tensor_dict: Dict[str, Union[torch.Tensor, Any]],
49 - prefix: str = "") -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
48 + tensor_dict: Dict[str, Union[torch.Tensor, Any]]
49 +) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
50 """Split the tensor dictionary into two parts:
51 1. A list of (key, value) pairs. If the value is a tensor, it is replaced
52 by its metadata.
53 2. A list of tensors.
54 -
55 - If the Tensor is nested under `tensor_dict["key1"]["key2"]`, the key of its
56 - metadata will be "key1%key2".
54 """
55 metadata_list: List[Tuple[str, Any]] = []
59 - tensor_list = []
56 + tensor_list: List[torch.Tensor] = []
57 for key, value in tensor_dict.items():
61 - assert "%" not in key, (
62 - "Avoid having '%' in key "
63 - "as it is used as a separator for nested entries.")
58 if isinstance(value, torch.Tensor):
59 # Note: we cannot use `value.device` here,
60 # because it contains not only the device type but also the device
@@ -68,31 +62,13 @@ def _split_tensor_dict(
62 # receiving side will set the device index.
63 device = value.device.type
64 metadata_list.append(
71 - (prefix + key, TensorMetadata(device, value.dtype,
72 - value.size())))
65 + (key, TensorMetadata(device, value.dtype, value.size())))
66 tensor_list.append(value)
74 - elif isinstance(value, dict):
75 - if len(value) == 0:
76 - metadata_list.append((prefix + key, value))
77 - inner_metadata_list, inner_tensor_list = _split_tensor_dict(
78 - value, prefix + key + "%")
79 - metadata_list.extend(inner_metadata_list)
80 - tensor_list.extend(inner_tensor_list)
67 else:
82 - metadata_list.append((prefix + key, value))
68 + metadata_list.append((key, value))
69 return metadata_list, tensor_list
70
71
86 -def _update_nested_dict(nested_dict, flattened_key, value):
87 - key_splits = flattened_key.split("%")
88 - cur_dict = nested_dict
89 - for k in key_splits[:-1]:
90 - if k not in cur_dict:
91 - cur_dict[k] = {}
92 - cur_dict = cur_dict[k]
93 - cur_dict[key_splits[-1]] = value
94 -
95 -
72 class GroupCoordinator:
73 """
74 PyTorch ProcessGroup wrapper for a group of processes.
@@ -566,7 +542,7 @@ class GroupCoordinator:
542 device=value.device)
543 if tensor.numel() == 0:
544 # Skip broadcasting empty tensors.
569 - _update_nested_dict(tensor_dict, key, tensor)
545 + tensor_dict[key] = tensor
546 continue
547 if tensor.is_cpu:
548 # use metadata_group for CPU tensors
@@ -583,9 +559,9 @@ class GroupCoordinator:
559 group=group,
560 async_op=True)
561 async_handles.append(handle)
586 - _update_nested_dict(tensor_dict, key, tensor)
562 + tensor_dict[key] = tensor
563 else:
588 - _update_nested_dict(tensor_dict, key, value)
564 + tensor_dict[key] = value
565 for async_handle in async_handles:
566 async_handle.wait()
567 return tensor_dict
@@ -661,7 +637,7 @@ class GroupCoordinator:
637 device=value.device)
638 if tensor.numel() == 0:
639 # Skip broadcasting empty tensors.
664 - _update_nested_dict(tensor_dict, key, tensor)
640 + tensor_dict[key] = tensor
641 continue
642 if tensor.is_cpu:
643 # use metadata_group for CPU tensors
@@ -673,9 +649,9 @@ class GroupCoordinator:
649 torch.distributed.recv(tensor,
650 src=self.ranks[src],
651 group=group)
676 - _update_nested_dict(tensor_dict, key, tensor)
652 + tensor_dict[key] = tensor
653 else:
678 - _update_nested_dict(tensor_dict, key, value)
654 + tensor_dict[key] = value
655 return tensor_dict
656
657 def barrier(self):
vllm/multimodal/__init__.py
+4 -2
@@ -1,5 +1,6 @@
1 -from .base import (BatchedTensors, MultiModalDataBuiltins, MultiModalDataDict,
2 - MultiModalInputs, MultiModalPlugin, NestedTensors)
1 +from .base import (BatchedTensorInputs, BatchedTensors, MultiModalDataBuiltins,
2 + MultiModalDataDict, MultiModalInputs, MultiModalPlugin,
3 + NestedTensors)
4 from .registry import MultiModalRegistry
5
6 MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -12,6 +13,7 @@ See also:
13 """
14
15 __all__ = [
16 + "BatchedTensorInputs",
17 "BatchedTensors",
18 "MultiModalDataBuiltins",
19 "MultiModalDataDict",
vllm/multimodal/base.py
+42 -20
@@ -9,10 +9,12 @@ import torch
9 import torch.types
10 from PIL import Image
11 from torch import nn
12 +from typing_extensions import TypeAlias
13
14 from vllm.config import ModelConfig
15 from vllm.inputs import InputContext
16 from vllm.logger import init_logger
17 +from vllm.utils import JSONTree, json_map_leaves
18
19 logger = init_logger(__name__)
20
@@ -22,11 +24,16 @@ Use a list instead of a tensor if the dimensions of each element do not match.
24 Currently only supports up to singly nested list of tensors.
25 """
26
25 -BatchedTensors = Union[GenericSequence[NestedTensors], NestedTensors]
27 +BatchedTensors: TypeAlias = JSONTree[torch.Tensor]
28 """
27 -If each input tensor in the batch has the same size, this is a single batched
28 -tensor; otherwise, this is a list of :class:`NestedTensors` with one element
29 -per item in the batch.
29 +A nested JSON structure of tensors which have been batched via
30 +:meth:`MultiModalInputs.batch`.
31 +"""
32 +
33 +BatchedTensorInputs: TypeAlias = Dict[str, JSONTree[torch.Tensor]]
34 +"""
35 +A dictionary containing nested tensors which have been batched via
36 +:meth:`MultiModalInputs.batch`.
37 """
38
39 if sys.version_info < (3, 9):
@@ -46,14 +53,17 @@ class MultiModalInputs(_MultiModalInputsBase):
53 """
54
55 @staticmethod
49 - def try_concat(
56 + def _try_concat(
57 tensors: List[NestedTensors],
51 - *,
52 - device: torch.types.Device,
53 - ) -> BatchedTensors:
58 + ) -> Union[GenericSequence[NestedTensors], NestedTensors]:
59 + """
60 + If each input tensor in the batch has the same shape, return a single
61 + batched tensor; otherwise, return a list of :class:`NestedTensors` with
62 + one element per item in the batch.
63 + """
64 # may be list rather than tensors
65 if isinstance(tensors[0], list):
56 - return [[t.to(device=device) for t in tensor[0]]
66 + return [[t for t in tensor[0]]
67 for tensor in cast(List[List[torch.Tensor]], tensors)]
68
69 tensors_ = cast(List[torch.Tensor], tensors)
@@ -62,18 +72,21 @@ class MultiModalInputs(_MultiModalInputsBase):
72
73 for tensor in tensors_:
74 if tensor.shape[1:] != unbatched_shape:
65 - return [
66 - tensor.squeeze(0).to(device=device) for tensor in tensors_
67 - ]
75 + return [tensor.squeeze(0) for tensor in tensors_]
76
69 - return torch.cat(tensors_, dim=0).to(device=device)
77 + return torch.cat(tensors_, dim=0)
78
79 @staticmethod
72 - def batch(
73 - inputs_list: List["MultiModalInputs"],
74 - device: torch.types.Device,
75 - ) -> Dict[str, BatchedTensors]:
76 - """Batch multiple inputs together into a dictionary."""
80 + def batch(inputs_list: List["MultiModalInputs"]) -> BatchedTensorInputs:
81 + """
82 + Batch multiple inputs together into a dictionary.
83 +
84 + The resulting dictionary has the same keys as the inputs.
85 + If the corresponding value from each input is a tensor and they all
86 + share the same shape, the output value is a single batched tensor;
87 + otherwise, the output value is a list containing the original value
88 + from each input.
89 + """
90 if len(inputs_list) == 0:
91 return {}
92
@@ -90,9 +103,18 @@ class MultiModalInputs(_MultiModalInputsBase):
103 item_lists[k].append(v)
104
105 return {
93 - k: MultiModalInputs.try_concat(item_list, device=device)
106 + k: MultiModalInputs._try_concat(item_list)
107 for k, item_list in item_lists.items()
95 - }
108 + } # type: ignore
109 +
110 + @staticmethod
111 + def as_kwargs(
112 + batched_inputs: BatchedTensorInputs,
113 + *,
114 + device: torch.types.Device,
115 + ) -> BatchedTensorInputs:
116 + return json_map_leaves(lambda x: x.to(device, non_blocking=True),
117 + batched_inputs)
118
119
120 class MultiModalDataBuiltins(TypedDict, total=False):
vllm/spec_decode/draft_model_runner.py
+3 -1
@@ -15,6 +15,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15 ModelConfig, MultiModalConfig, ParallelConfig,
16 PromptAdapterConfig, SchedulerConfig)
17 from vllm.logger import init_logger
18 +from vllm.multimodal import MultiModalInputs
19 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
20 SamplerOutput)
21 from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
@@ -323,7 +324,8 @@ class TP1DraftModelRunner(ModelRunner):
324 kv_caches=kv_caches,
325 attn_metadata=model_input.attn_metadata,
326 intermediate_tensors=intermediate_tensors,
326 - **multi_modal_kwargs,
327 + **MultiModalInputs.as_kwargs(multi_modal_kwargs,
328 + device=self.device),
329 )
330
331 # Compute the logits.
vllm/utils.py
+50 -1
@@ -17,7 +17,7 @@ from functools import lru_cache, partial, wraps
17 from platform import uname
18 from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
19 Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
20 - Union)
20 + Union, overload)
21
22 import numpy as np
23 import numpy.typing as npt
@@ -53,6 +53,7 @@ TORCH_DTYPE_TO_NUMPY_DTYPE = {
53 P = ParamSpec('P')
54 K = TypeVar("K")
55 T = TypeVar("T")
56 +U = TypeVar("U")
57
58
59 class _Sentinel:
@@ -712,6 +713,54 @@ def merge_dicts(dict1: Dict[K, List[T]],
713 return dict(merged_dict)
714
715
716 +JSONTree = Union[Dict[str, "JSONTree[T]"], List["JSONTree[T]"],
717 + Tuple["JSONTree[T]", ...], T]
718 +"""A nested JSON structure where the leaves need not be JSON-serializable."""
719 +
720 +
721 +@overload
722 +def json_map_leaves(
723 + func: Callable[[T], U],
724 + value: Dict[str, JSONTree[T]],
725 +) -> Dict[str, JSONTree[U]]:
726 + ...
727 +
728 +
729 +@overload
730 +def json_map_leaves(
731 + func: Callable[[T], U],
732 + value: List[JSONTree[T]],
733 +) -> List[JSONTree[U]]:
734 + ...
735 +
736 +
737 +@overload
738 +def json_map_leaves(
739 + func: Callable[[T], U],
740 + value: Tuple[JSONTree[T], ...],
741 +) -> Tuple[JSONTree[U], ...]:
742 + ...
743 +
744 +
745 +@overload
746 +def json_map_leaves(
747 + func: Callable[[T], U],
748 + value: JSONTree[T],
749 +) -> JSONTree[U]:
750 + ...
751 +
752 +
753 +def json_map_leaves(func: Callable[[T], U], value: JSONTree[T]) -> JSONTree[U]:
754 + if isinstance(value, dict):
755 + return {k: json_map_leaves(func, v) for k, v in value.items()}
756 + elif isinstance(value, list):
757 + return [json_map_leaves(func, v) for v in value]
758 + elif isinstance(value, tuple):
759 + return tuple(json_map_leaves(func, v) for v in value)
760 + else:
761 + return func(value)
762 +
763 +
764 def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
765 """Flatten a list of lists to a single list."""
766 return [item for sublist in lists for item in sublist]
vllm/worker/cpu_model_runner.py
+15 -12
@@ -1,6 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
3 - Type, Union)
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
3
4 import torch
5 from torch import nn
@@ -12,7 +11,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 from vllm.logger import init_logger
12 from vllm.model_executor import SamplingMetadata
13 from vllm.model_executor.model_loader import get_model
15 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
14 +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
15 MultiModalInputs)
16 from vllm.sequence import (IntermediateTensors, SamplerOutput,
17 SequenceGroupMetadata)
@@ -41,7 +40,7 @@ class CPUModelInput(ModelRunnerInputBase):
40 input_positions: Optional[torch.Tensor] = None
41 attn_metadata: Optional["AttentionMetadata"] = None
42 sampling_metadata: Optional["SamplingMetadata"] = None
44 - multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
43 + multi_modal_kwargs: Optional[BatchedTensorInputs] = None
44 virtual_engine: Optional[int] = None
45
46 def as_broadcastable_tensor_dict(
@@ -136,7 +135,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
135 self,
136 seq_group_metadata_list: List[SequenceGroupMetadata],
137 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
139 - Mapping[str, BatchedTensors]]:
138 + BatchedTensorInputs]:
139 assert len(seq_group_metadata_list) > 0
140 input_tokens: List[int] = []
141 input_positions: List[int] = []
@@ -214,8 +213,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
213 slot_mapping=slot_mapping,
214 )
215
217 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
218 - device=self.device)
216 + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
217
218 return (input_tokens, input_positions, attn_metadata, seq_lens,
219 multi_modal_kwargs)
@@ -361,11 +359,16 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
359
360 model_executable = self.model
361 execute_model_kwargs = {
364 - "input_ids": model_input.input_tokens,
365 - "positions": model_input.input_positions,
366 - "kv_caches": kv_caches,
367 - "attn_metadata": model_input.attn_metadata,
368 - **(model_input.multi_modal_kwargs or {}),
362 + "input_ids":
363 + model_input.input_tokens,
364 + "positions":
365 + model_input.input_positions,
366 + "kv_caches":
367 + kv_caches,
368 + "attn_metadata":
369 + model_input.attn_metadata,
370 + **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
371 + device=self.device),
372 }
373
374 hidden_states = model_executable(**execute_model_kwargs)
vllm/worker/embedding_model_runner.py
+11 -5
@@ -8,6 +8,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
8 PromptAdapterConfig, SchedulerConfig)
9 from vllm.logger import init_logger
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 +from vllm.multimodal import MultiModalInputs
12 from vllm.pooling_params import PoolingParams
13 from vllm.sequence import (IntermediateTensors, PoolerOutput, SequenceData,
14 SequenceGroupMetadata)
@@ -99,11 +100,16 @@ class EmbeddingModelRunner(
100 kv_caches = [None] * num_layers
101
102 execute_model_kwargs = {
102 - "input_ids": model_input.input_tokens,
103 - "positions": model_input.input_positions,
104 - "kv_caches": kv_caches,
105 - "attn_metadata": model_input.attn_metadata,
106 - **(model_input.multi_modal_kwargs or {}),
103 + "input_ids":
104 + model_input.input_tokens,
105 + "positions":
106 + model_input.input_positions,
107 + "kv_caches":
108 + kv_caches,
109 + "attn_metadata":
110 + model_input.attn_metadata,
111 + **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
112 + device=self.device),
113 }
114
115 hidden_states = model_executable(**execute_model_kwargs)
vllm/worker/model_runner.py
+7 -7
@@ -4,8 +4,8 @@ import time
4 import warnings
5 import weakref
6 from dataclasses import dataclass
7 -from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
8 - Tuple, Type, TypeVar, Union)
7 +from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type,
8 + TypeVar, Union)
9
10 import numpy as np
11 import torch
@@ -40,7 +40,7 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
40 from vllm.model_executor.models.interfaces import (supports_lora,
41 supports_vision)
42 from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
43 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
43 +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
44 MultiModalInputs)
45 from vllm.prompt_adapter.layers import PromptAdapterMapping
46 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -94,7 +94,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
94 attn_metadata: Optional["AttentionMetadata"] = None
95 prompt_adapter_mapping: Optional[PromptAdapterMapping] = None
96 prompt_adapter_requests: Optional[Set[PromptAdapterRequest]] = None
97 - multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
97 + multi_modal_kwargs: Optional[BatchedTensorInputs] = None
98 request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
99 finished_requests_ids: Optional[List[str]] = None
100 virtual_engine: int = 0
@@ -608,8 +608,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
608 data.multi_modal_inputs for data in self.inter_data_list
609 if data.multi_modal_inputs is not None
610 ]
611 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
612 - device=self.runner.device)
611 + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
612
613 return self.model_input_cls(
614 input_tokens=input_tokens_tensor,
@@ -1361,7 +1360,8 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1360 kv_caches=kv_caches,
1361 attn_metadata=model_input.attn_metadata,
1362 intermediate_tensors=intermediate_tensors,
1364 - **multi_modal_kwargs,
1363 + **MultiModalInputs.as_kwargs(multi_modal_kwargs,
1364 + device=self.device),
1365 **seqlen_agnostic_kwargs)
1366
1367 # Compute the logits in the last pipeline stage.
vllm/worker/neuron_model_runner.py
+8 -9
@@ -1,6 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
3 - Union)
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
3
4 import torch
5 from torch import nn
@@ -10,7 +9,7 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
9 from vllm.logger import init_logger
10 from vllm.model_executor import SamplingMetadata
11 from vllm.model_executor.model_loader.neuron import get_neuron_model
13 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
12 +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
13 MultiModalInputs)
14 from vllm.sequence import (IntermediateTensors, SamplerOutput,
15 SequenceGroupMetadata)
@@ -32,7 +31,7 @@ class ModelInputForNeuron(ModelRunnerInputBase):
31 input_positions: Optional[torch.Tensor] = None
32 input_block_ids: Optional[torch.Tensor] = None
33 sampling_metadata: Optional["SamplingMetadata"] = None
35 - multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
34 + multi_modal_kwargs: Optional[BatchedTensorInputs] = None
35
36 def as_broadcastable_tensor_dict(
37 self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -84,8 +83,8 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
83 def _prepare_prompt(
84 self,
85 seq_group_metadata_list: List[SequenceGroupMetadata],
87 - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[int], Mapping[
88 - str, BatchedTensors]]:
86 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[int],
87 + BatchedTensorInputs]:
88 assert len(seq_group_metadata_list) > 0
89 input_tokens: List[List[int]] = []
90 input_positions: List[List[int]] = []
@@ -134,8 +133,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
133 dtype=torch.long,
134 device=self.device)
135
137 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
138 - device=self.device)
136 + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
137
138 return (input_tokens, input_positions, input_block_ids, seq_lens,
139 multi_modal_kwargs)
@@ -244,7 +242,8 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
242 input_ids=model_input.input_tokens,
243 positions=model_input.input_positions,
244 input_block_ids=model_input.input_block_ids,
247 - **(model_input.multi_modal_kwargs or {}),
245 + **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
246 + device=self.device),
247 )
248
249 # Compute the logits.
vllm/worker/openvino_model_runner.py
+15 -11
@@ -1,4 +1,4 @@
1 -from typing import List, Mapping, NamedTuple, Optional, Tuple
1 +from typing import List, NamedTuple, Optional, Tuple
2
3 import openvino as ov
4 import torch
@@ -12,7 +12,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.model_loader.openvino import get_model
15 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
15 +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16 MultiModalInputs)
17 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
18
@@ -25,7 +25,7 @@ class ModelInput(NamedTuple):
25 attn_metadata: Optional[OpenVINOAttentionMetadata]
26 seq_lens: List[int]
27 query_lens: List[int]
28 - multi_modal_kwargs: Mapping[str, BatchedTensors]
28 + multi_modal_kwargs: BatchedTensorInputs
29
30 @classmethod
31 def empty(cls, device):
@@ -265,8 +265,7 @@ class OpenVINOModelRunner:
265 max_context_len=max_context_len_tensor,
266 )
267
268 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
269 - device=self.device)
268 + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
269
270 return ModelInput(
271 input_tokens,
@@ -281,7 +280,7 @@ class OpenVINOModelRunner:
280 self,
281 seq_group_metadata_list: List[SequenceGroupMetadata],
282 ) -> Tuple[torch.Tensor, torch.Tensor, OpenVINOAttentionMetadata,
284 - SamplingMetadata, Mapping[str, BatchedTensors]]:
283 + SamplingMetadata, BatchedTensorInputs]:
284 # Prepare input tensors.
285 (
286 input_tokens,
@@ -324,11 +323,16 @@ class OpenVINOModelRunner:
323
324 model_executable = self.model
325 execute_model_kwargs = {
327 - "input_ids": input_tokens,
328 - "positions": input_positions,
329 - "kv_caches": kv_caches,
330 - "attn_metadata": attn_metadata,
331 - **(multi_modal_kwargs or {}),
326 + "input_ids":
327 + input_tokens,
328 + "positions":
329 + input_positions,
330 + "kv_caches":
331 + kv_caches,
332 + "attn_metadata":
333 + attn_metadata,
334 + **MultiModalInputs.as_kwargs(multi_modal_kwargs or {},
335 + device=self.device),
336 }
337
338 hidden_states = model_executable(**execute_model_kwargs)
vllm/worker/xpu_model_runner.py
+15 -12
@@ -1,6 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
3 - Type, Union)
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
3
4 import torch
5 import torch.nn as nn
@@ -14,7 +13,7 @@ from vllm.inputs import INPUT_REGISTRY
13 from vllm.logger import init_logger
14 from vllm.model_executor.model_loader import get_model
15 from vllm.model_executor.models.interfaces import supports_vision
17 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
16 +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17 MultiModalInputs)
18 from vllm.sampling_params import SamplingParams
19 from vllm.sequence import (IntermediateTensors, SamplerOutput,
@@ -49,7 +48,7 @@ class ModelInputForXPU(ModelRunnerInputBase):
48 input_positions: Optional[torch.Tensor] = None
49 attn_metadata: Optional["AttentionMetadata"] = None
50 sampling_metadata: Optional["SamplingMetadata"] = None
52 - multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
51 + multi_modal_kwargs: Optional[BatchedTensorInputs] = None
52
53 def as_broadcastable_tensor_dict(
54 self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -376,11 +375,16 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
375
376 model_executable = self.model
377 execute_model_kwargs = {
379 - "input_ids": model_input.input_tokens,
380 - "positions": model_input.input_positions,
381 - "kv_caches": kv_caches,
382 - "attn_metadata": model_input.attn_metadata,
383 - **(model_input.multi_modal_kwargs or {}),
378 + "input_ids":
379 + model_input.input_tokens,
380 + "positions":
381 + model_input.input_positions,
382 + "kv_caches":
383 + kv_caches,
384 + "attn_metadata":
385 + model_input.attn_metadata,
386 + **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
387 + device=self.device),
388 }
389
390 hidden_states = model_executable(**execute_model_kwargs)
@@ -404,7 +408,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
408 self,
409 seq_group_metadata_list: List[SequenceGroupMetadata],
410 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
407 - Mapping[str, BatchedTensors]]:
411 + BatchedTensorInputs]:
412 assert len(seq_group_metadata_list) > 0
413 input_tokens: List[int] = []
414 input_positions: List[int] = []
@@ -496,8 +500,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
500 block_tables=torch.tensor([], device=self.device, dtype=torch.int),
501 )
502
499 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
500 - device=self.device)
503 + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
504
505 return (input_tokens, input_positions, attn_metadata, seq_lens,
506 multi_modal_kwargs)