@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3bc94cab69
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] VLM - Run the mm_mapper preprocessor in the frontend process (#10640)

Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Michael Goin <michael@neuralmagic.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Alexander Matveev committed Dec 3, 2024 at 05:33 UTC 3bc94cab695387eb16be90b6368029f56ce5dbc7
7 files changed +47 -25
tests/v1/engine/test_engine_core.py
+1 -2
@@ -27,9 +27,8 @@ def make_request() -> EngineCoreRequest:
27 request_id=uuid.uuid4(),
28 prompt=PROMPT,
29 prompt_token_ids=PROMPT_TOKENS,
30 - mm_data=None,
30 + mm_inputs=None,
31 mm_placeholders=None,
32 - mm_processor_kwargs=None,
32 sampling_params=SamplingParams(),
33 eos_token_id=None,
34 arrival_time=time.time(),
tests/v1/engine/test_engine_core_client.py
+1 -2
@@ -29,9 +29,8 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:
29 request_id=str(uuid.uuid4()),
30 prompt=PROMPT,
31 prompt_token_ids=PROMPT_TOKENS,
32 - mm_data=None,
32 + mm_inputs=None,
33 mm_placeholders=None,
34 - mm_processor_kwargs=None,
34 sampling_params=params,
35 eos_token_id=None,
36 arrival_time=time.time(),
vllm/inputs/data.py
+23 -1
@@ -7,7 +7,8 @@ import torch
7 from typing_extensions import NotRequired, TypedDict, TypeVar, assert_never
8
9 if TYPE_CHECKING:
10 - from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
10 + from vllm.multimodal import (MultiModalDataDict, MultiModalKwargs,
11 + MultiModalPlaceholderDict)
12 from vllm.multimodal.inputs import MultiModalInputsV2
13
14
@@ -150,6 +151,12 @@ class TokenInputs(TypedDict):
151 if the model supports it.
152 """
153
154 + multi_modal_inputs: NotRequired["MultiModalKwargs"]
155 + """
156 + Optional multi-modal inputs to pass to the model,
157 + if the model supports it.
158 + """
159 +
160 multi_modal_placeholders: NotRequired["MultiModalPlaceholderDict"]
161 """
162 Placeholder ranges for the multi-modal data.
@@ -169,6 +176,7 @@ def token_inputs(
176 token_type_ids: Optional[List[int]] = None,
177 prompt: Optional[str] = None,
178 multi_modal_data: Optional["MultiModalDataDict"] = None,
179 + multi_modal_inputs: Optional["MultiModalKwargs"] = None,
180 multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None,
181 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
182 ) -> TokenInputs:
@@ -181,6 +189,8 @@ def token_inputs(
189 inputs["token_type_ids"] = token_type_ids
190 if multi_modal_data is not None:
191 inputs["multi_modal_data"] = multi_modal_data
192 + if multi_modal_inputs is not None:
193 + inputs["multi_modal_inputs"] = multi_modal_inputs
194 if multi_modal_placeholders is not None:
195 inputs["multi_modal_placeholders"] = multi_modal_placeholders
196 if mm_processor_kwargs is not None:
@@ -273,6 +283,18 @@ class SingletonInputsAdapter:
283
284 assert_never(inputs)
285
286 + @cached_property
287 + def multi_modal_inputs(self) -> Union[Dict, "MultiModalKwargs"]:
288 + inputs = self.inputs
289 +
290 + if inputs["type"] == "token":
291 + return inputs.get("multi_modal_inputs", {})
292 +
293 + if inputs["type"] == "multimodal":
294 + return inputs.get("mm_kwargs", {})
295 +
296 + assert_never(inputs)
297 +
298 @cached_property
299 def multi_modal_placeholders(self) -> "MultiModalPlaceholderDict":
300 inputs = self.inputs
vllm/v1/engine/__init__.py
+3 -4
@@ -1,11 +1,11 @@
1 import enum
2 from dataclasses import dataclass
3 -from typing import Any, Dict, List, Optional, Union
3 +from typing import List, Optional, Union
4
5 import msgspec
6
7 from vllm.lora.request import LoRARequest
8 -from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
8 +from vllm.multimodal import MultiModalKwargs, MultiModalPlaceholderDict
9 from vllm.sampling_params import RequestOutputKind, SamplingParams
10
11
@@ -35,9 +35,8 @@ class EngineCoreRequest:
35 # always be tokenized?
36 prompt: Optional[str]
37 prompt_token_ids: List[int]
38 - mm_data: Optional[MultiModalDataDict]
38 + mm_inputs: Optional[List[MultiModalKwargs]]
39 mm_placeholders: Optional[MultiModalPlaceholderDict]
40 - mm_processor_kwargs: Optional[Dict[str, Any]]
40 sampling_params: SamplingParams
41 eos_token_id: Optional[int]
42 arrival_time: float
vllm/v1/engine/core.py
-7
@@ -84,14 +84,7 @@ class EngineCore:
84
85 def add_request(self, request: EngineCoreRequest):
86 """Add request to the scheduler."""
87 -
87 req = Request.from_engine_core_request(request)
89 - # FIXME(woosuk): The input mapping (e.g., PIL images to tensors) may
90 - # take 10-50 ms, which can cause a spike in the latency. We should
91 - # consider moving this to a separate thread.
92 - if req.mm_data:
93 - req.mm_inputs = self.mm_input_mapper.process_inputs(
94 - req.mm_data, req.mm_processor_kwargs)
88 self.scheduler.add_request(req)
89
90 def abort_requests(self, request_ids: List[str]):
vllm/v1/engine/processor.py
+11 -2
@@ -14,6 +14,7 @@ from vllm.sampling_params import SamplingParams
14 from vllm.transformers_utils.config import try_get_generation_config
15 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16 from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
17 +from vllm.v1.engine.mm_input_mapper import MMInputMapper
18
19
20 class Processor:
@@ -39,6 +40,9 @@ class Processor:
40 self.input_processor = input_registry.create_input_processor(
41 model_config)
42
43 + # Multi-modal (huggingface) input mapper
44 + self.mm_input_mapper = MMInputMapper(model_config)
45 +
46 # TODO: run in an ThreadpoolExecutor or BackgroundProcess.
47 # This ideally should releases the GIL, so we should not block the
48 # asyncio loop while this is running.
@@ -96,6 +100,12 @@ class Processor:
100 sampling_params.update_from_generation_config(
101 self.generation_config_fields, eos_token_id)
102
103 + # Preprocess multi-modal data
104 + mm_inputs = self.mm_input_mapper.process_inputs(
105 + decoder_inputs.multi_modal_data,
106 + decoder_inputs.mm_processor_kwargs) if len(
107 + decoder_inputs.multi_modal_data) > 0 else None
108 +
109 # Make Request for Detokenizer.
110 detokenizer_request = DetokenizerRequest(
111 request_id,
@@ -113,9 +123,8 @@ class Processor:
123 request_id,
124 decoder_inputs.prompt,
125 decoder_inputs.prompt_token_ids,
116 - decoder_inputs.multi_modal_data,
126 + mm_inputs,
127 decoder_inputs.multi_modal_placeholders,
118 - decoder_inputs.mm_processor_kwargs,
128 sampling_params,
129 eos_token_id,
130 arrival_time,
vllm/v1/request.py
+8 -7
@@ -45,9 +45,6 @@ class Request:
45 self._all_token_ids: List[int] = self.prompt_token_ids.copy()
46 self.num_computed_tokens = 0
47
48 - # Raw multimodal data before the mm input mapper (e.g., PIL images).
49 - self.mm_data = self.inputs.multi_modal_data
50 - self.mm_processor_kwargs = self.inputs.mm_processor_kwargs
48 mm_positions = self.inputs.multi_modal_placeholders
49 if mm_positions:
50 # FIXME(woosuk): Support other modalities.
@@ -55,7 +52,10 @@ class Request:
52 else:
53 self.mm_positions = []
54 # Output of the mm input mapper (e.g., image tensors).
58 - self.mm_inputs: List[MultiModalKwargs] = []
55 + if self.inputs.multi_modal_inputs:
56 + self.mm_inputs = self.inputs.multi_modal_inputs
57 + else:
58 + self.mm_inputs: List[MultiModalKwargs] = []
59
60 @classmethod
61 def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
@@ -64,9 +64,10 @@ class Request:
64 inputs=token_inputs(
65 prompt_token_ids=request.prompt_token_ids,
66 prompt=request.prompt,
67 - multi_modal_data=request.mm_data,
67 + multi_modal_data=None,
68 + multi_modal_inputs=request.mm_inputs,
69 multi_modal_placeholders=request.mm_placeholders,
69 - mm_processor_kwargs=request.mm_processor_kwargs,
70 + mm_processor_kwargs=None,
71 ),
72 sampling_params=request.sampling_params,
73 eos_token_id=request.eos_token_id,
@@ -110,7 +111,7 @@ class Request:
111 return RequestStatus.get_finished_reason(self.status)
112
113 def has_encoder_inputs(self) -> bool:
113 - return len(self.mm_data) > 0
114 + return len(self.mm_inputs) > 0
115
116 @property
117 def num_encoder_inputs(self) -> int: