@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96912550c8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc] Rename `MultiModalInputsV2 -> MultiModalInputs` (#12244)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 21, 2025 at 15:31 UTC 96912550c8399af2632f3f6830f7c3fa9e10a75a
12 files changed +31 -31
docs/source/api/multimodal/inputs.md
+1 -1
@@ -43,7 +43,7 @@
43 ```
44
45 ```{eval-rst}
46 -.. autoclass:: vllm.multimodal.inputs.MultiModalInputsV2
46 +.. autoclass:: vllm.multimodal.inputs.MultiModalInputs
47 :members:
48 :show-inheritance:
49 ```
vllm/inputs/data.py
+6 -6
@@ -9,7 +9,7 @@ from typing_extensions import NotRequired, TypedDict, TypeVar, assert_never
9 if TYPE_CHECKING:
10 from vllm.multimodal import (MultiModalDataDict, MultiModalKwargs,
11 MultiModalPlaceholderDict)
12 - from vllm.multimodal.inputs import MultiModalInputsV2
12 + from vllm.multimodal.inputs import MultiModalInputs
13
14
15 class TextPrompt(TypedDict):
@@ -207,7 +207,7 @@ def token_inputs(
207 return inputs
208
209
210 -DecoderOnlyInputs = Union[TokenInputs, "MultiModalInputsV2"]
210 +DecoderOnlyInputs = Union[TokenInputs, "MultiModalInputs"]
211 """
212 The inputs in :class:`~vllm.LLMEngine` before they are
213 passed to the model executor.
@@ -222,14 +222,14 @@ class EncoderDecoderInputs(TypedDict):
222
223 This specifies the required data for encoder-decoder models.
224 """
225 - encoder: Union[TokenInputs, "MultiModalInputsV2"]
225 + encoder: Union[TokenInputs, "MultiModalInputs"]
226 """The inputs for the encoder portion."""
227
228 - decoder: Union[TokenInputs, "MultiModalInputsV2"]
228 + decoder: Union[TokenInputs, "MultiModalInputs"]
229 """The inputs for the decoder portion."""
230
231
232 -SingletonInputs = Union[TokenInputs, "MultiModalInputsV2"]
232 +SingletonInputs = Union[TokenInputs, "MultiModalInputs"]
233 """
234 A processed :class:`SingletonPrompt` which can be passed to
235 :class:`vllm.sequence.Sequence`.
@@ -311,7 +311,7 @@ class SingletonInputsAdapter:
311 return inputs.get("multi_modal_hashes", [])
312
313 if inputs["type"] == "multimodal":
314 - # only the case when we use MultiModalInputsV2
314 + # only the case when we use MultiModalInputs
315 return inputs.get("mm_hashes", []) # type: ignore[return-value]
316
317 assert_never(inputs) # type: ignore[arg-type]
vllm/inputs/preprocess.py
+3 -3
@@ -7,7 +7,7 @@ from vllm.config import ModelConfig
7 from vllm.logger import init_logger
8 from vllm.lora.request import LoRARequest
9 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
10 -from vllm.multimodal.inputs import MultiModalDataDict, MultiModalInputsV2
10 +from vllm.multimodal.inputs import MultiModalDataDict, MultiModalInputs
11 from vllm.prompt_adapter.request import PromptAdapterRequest
12 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
13
@@ -247,7 +247,7 @@ class InputPreprocessor:
247 mm_data: MultiModalDataDict,
248 mm_processor_kwargs: Optional[Mapping[str, object]],
249 lora_request: Optional[LoRARequest],
250 - ) -> MultiModalInputsV2:
250 + ) -> MultiModalInputs:
251 """
252 Apply the model's multi-modal processor to a multi-modal prompt,
253 returning the corresponding token IDs and metadata.
@@ -271,7 +271,7 @@ class InputPreprocessor:
271 mm_data: MultiModalDataDict,
272 mm_processor_kwargs: Optional[Mapping[str, object]],
273 lora_request: Optional[LoRARequest],
274 - ) -> MultiModalInputsV2:
274 + ) -> MultiModalInputs:
275 """Async version of :meth:`_process_multimodal`."""
276 tokenizer_group = self.get_tokenizer_group()
277 tokenizer = await tokenizer_group.get_lora_tokenizer_async(lora_request
vllm/model_executor/models/blip2.py
+2 -2
@@ -15,7 +15,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
15 from vllm.model_executor.sampling_metadata import SamplingMetadata
16 from vllm.multimodal import MULTIMODAL_REGISTRY
17 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18 - MultiModalInputsV2, MultiModalKwargs,
18 + MultiModalInputs, MultiModalKwargs,
19 NestedTensors, PlaceholderRange)
20 from vllm.multimodal.parse import MultiModalDataItems
21 from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -490,7 +490,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
490 prompt: Union[str, list[int]],
491 mm_data: MultiModalDataDict,
492 hf_processor_mm_kwargs: Mapping[str, object],
493 - ) -> MultiModalInputsV2:
493 + ) -> MultiModalInputs:
494 result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
495
496 # Only <image> tokens should be considered as placeholders,
vllm/model_executor/models/chameleon.py
+2 -2
@@ -29,7 +29,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
29 from vllm.model_executor.utils import set_weight_attrs
30 from vllm.multimodal import MULTIMODAL_REGISTRY
31 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
32 - MultiModalInputsV2, MultiModalKwargs,
32 + MultiModalInputs, MultiModalKwargs,
33 NestedTensors, PlaceholderRange)
34 from vllm.multimodal.parse import MultiModalDataItems
35 from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -159,7 +159,7 @@ class ChameleonMultiModalProcessor(
159 prompt: Union[str, list[int]],
160 mm_data: MultiModalDataDict,
161 hf_processor_mm_kwargs: Mapping[str, object],
162 - ) -> MultiModalInputsV2:
162 + ) -> MultiModalInputs:
163 result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
164
165 # Only <image> tokens should be considered as placeholders,
vllm/model_executor/models/fuyu.py
+2 -2
@@ -31,7 +31,7 @@ from vllm.model_executor.models.persimmon import PersimmonForCausalLM
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34 - MultiModalInputsV2, MultiModalKwargs,
34 + MultiModalInputs, MultiModalKwargs,
35 NestedTensors, PlaceholderRange)
36 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
37 MultiModalDataItems)
@@ -232,7 +232,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
232 prompt: Union[str, list[int]],
233 mm_data: MultiModalDataDict,
234 hf_processor_mm_kwargs: Mapping[str, object],
235 - ) -> MultiModalInputsV2:
235 + ) -> MultiModalInputs:
236 result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
237
238 # Only |SPEAKER| (image) tokens should be considered as placeholders,
vllm/model_executor/models/llava.py
+3 -3
@@ -24,7 +24,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 from vllm.multimodal import MULTIMODAL_REGISTRY
26 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
27 - MultiModalInputsV2, MultiModalKwargs,
27 + MultiModalInputs, MultiModalKwargs,
28 NestedTensors)
29 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
30 ImageSize, MultiModalDataItems)
@@ -746,7 +746,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
746 prompt: Union[str, list[int]],
747 mm_data: MultiModalDataDict,
748 hf_processor_mm_kwargs: Mapping[str, object],
749 - ) -> MultiModalInputsV2:
749 + ) -> MultiModalInputs:
750 hf_config = self.info.get_hf_config()
751 image_token_id = hf_config.image_token_index
752
@@ -805,7 +805,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
805 for modality, placeholders in mm_placeholders.items()
806 }
807
808 - return MultiModalInputsV2(
808 + return MultiModalInputs(
809 type="multimodal",
810 prompt=prompt,
811 prompt_token_ids=prompt_ids,
vllm/model_executor/models/phi3v.py
+2 -2
@@ -31,7 +31,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34 - MultiModalInputsV2, MultiModalKwargs,
34 + MultiModalInputs, MultiModalKwargs,
35 NestedTensors, PlaceholderRange)
36 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
37 ImageSize, MultiModalDataItems)
@@ -484,7 +484,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
484 prompt: Union[str, list[int]],
485 mm_data: MultiModalDataDict,
486 hf_processor_mm_kwargs: Mapping[str, object],
487 - ) -> MultiModalInputsV2:
487 + ) -> MultiModalInputs:
488 result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
489
490 # Only <|image|> tokens should be considered as placeholders,
vllm/model_executor/models/qwen2_audio.py
+2 -2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.multimodal import MULTIMODAL_REGISTRY
39 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
40 - MultiModalInputsV2, MultiModalKwargs,
40 + MultiModalInputs, MultiModalKwargs,
41 NestedTensors, PlaceholderRange)
42 from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
43 MultiModalDataParser)
@@ -245,7 +245,7 @@ class Qwen2AudioMultiModalProcessor(
245 prompt: Union[str, list[int]],
246 mm_data: MultiModalDataDict,
247 hf_processor_mm_kwargs: Mapping[str, object],
248 - ) -> MultiModalInputsV2:
248 + ) -> MultiModalInputs:
249 result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
250
251 # Only <|AUDIO|> tokens should be considered as placeholders,
vllm/multimodal/inputs.py
+1 -1
@@ -491,7 +491,7 @@ A dictionary containing placeholder ranges for each modality.
491 """
492
493
494 -class MultiModalInputsV2(TypedDict):
494 +class MultiModalInputs(TypedDict):
495 """
496 Represents the outputs of
497 :class:`vllm.multimodal.processing.BaseMultiModalProcessor`,
vllm/multimodal/processing.py
+5 -5
@@ -18,8 +18,8 @@ from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
18
19 from .hasher import MultiModalHasher
20 from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
21 - MultiModalInputsV2, MultiModalKwargs,
22 - MultiModalKwargsItem, PlaceholderRange)
21 + MultiModalInputs, MultiModalKwargs, MultiModalKwargsItem,
22 + PlaceholderRange)
23 from .parse import MultiModalDataItems, MultiModalDataParser
24
25 if TYPE_CHECKING:
@@ -609,7 +609,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
609 prompt: str,
610 mm_data: MultiModalDataDict,
611 hf_processor_mm_kwargs: Mapping[str, object],
612 - ) -> MultiModalInputsV2:
612 + ) -> MultiModalInputs:
613 return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
614
615 def _get_data_parser(self) -> MultiModalDataParser:
@@ -1067,7 +1067,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1067 prompt: Union[str, list[int]],
1068 mm_data: MultiModalDataDict,
1069 hf_processor_mm_kwargs: Mapping[str, object],
1070 - ) -> MultiModalInputsV2:
1070 + ) -> MultiModalInputs:
1071 """
1072 Process multi-modal inputs to be used in vLLM.
1073
@@ -1169,7 +1169,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1169 for modality, placeholders in mm_placeholders.items()
1170 }
1171
1172 - return MultiModalInputsV2(
1172 + return MultiModalInputs(
1173 type="multimodal",
1174 prompt=prompt,
1175 prompt_token_ids=prompt_ids,
vllm/multimodal/profiling.py
+2 -2
@@ -11,7 +11,7 @@ import vllm.envs as envs
11 from vllm.inputs import DummyData
12 from vllm.logger import init_logger
13
14 -from .inputs import MultiModalDataDict, MultiModalInputsV2
14 +from .inputs import MultiModalDataDict, MultiModalInputs
15 from .processing import BaseMultiModalProcessor, BaseProcessingInfo
16
17 logger = init_logger(__name__)
@@ -131,7 +131,7 @@ class MultiModalProfiler(Generic[_I]):
131 self,
132 seq_len: int,
133 mm_counts: Mapping[str, int],
134 - ) -> MultiModalInputsV2:
134 + ) -> MultiModalInputs:
135 factory = self.dummy_inputs
136 processor_inputs = factory.get_dummy_processor_inputs(
137 seq_len, mm_counts)