A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] VLM - Run the mm_mapper preprocessor in the frontend process (#10640)
Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Michael Goin <michael@neuralmagic.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Alexander Matveev committed
Dec 3, 2024 at 05:33 UTC
3bc94cab695387eb16be90b6368029f56ce5dbc7
7 files changed
+47
-25
tests/v1/engine/test_engine_core.py
+1
-2
@@ -27,9 +27,8 @@ def make_request() -> EngineCoreRequest:
27
request_id=uuid.uuid4(),
28
prompt=PROMPT,
29
prompt_token_ids=PROMPT_TOKENS,
30
- mm_data=None,
30
+ mm_inputs=None,
31
mm_placeholders=None,
32
- mm_processor_kwargs=None,
32
sampling_params=SamplingParams(),
33
eos_token_id=None,
34
arrival_time=time.time(),
tests/v1/engine/test_engine_core_client.py
+1
-2
@@ -29,9 +29,8 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:
29
request_id=str(uuid.uuid4()),
30
prompt=PROMPT,
31
prompt_token_ids=PROMPT_TOKENS,
32
- mm_data=None,
32
+ mm_inputs=None,
33
mm_placeholders=None,
34
- mm_processor_kwargs=None,
34
sampling_params=params,
35
eos_token_id=None,
36
arrival_time=time.time(),
vllm/inputs/data.py
+23
-1
@@ -7,7 +7,8 @@ import torch
7
from typing_extensions import NotRequired, TypedDict, TypeVar, assert_never
8
9
if TYPE_CHECKING:
10
- from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
10
+ from vllm.multimodal import (MultiModalDataDict, MultiModalKwargs,
11
+ MultiModalPlaceholderDict)
12
from vllm.multimodal.inputs import MultiModalInputsV2
13
14
@@ -150,6 +151,12 @@ class TokenInputs(TypedDict):
151
if the model supports it.
152
"""
153
154
+ multi_modal_inputs: NotRequired["MultiModalKwargs"]
155
+ """
156
+ Optional multi-modal inputs to pass to the model,
157
+ if the model supports it.
158
+ """
159
+
160
multi_modal_placeholders: NotRequired["MultiModalPlaceholderDict"]
161
"""
162
Placeholder ranges for the multi-modal data.
@@ -169,6 +176,7 @@ def token_inputs(
176
token_type_ids: Optional[List[int]] = None,
177
prompt: Optional[str] = None,
178
multi_modal_data: Optional["MultiModalDataDict"] = None,
179
+ multi_modal_inputs: Optional["MultiModalKwargs"] = None,
180
multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None,
181
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
182
) -> TokenInputs:
@@ -181,6 +189,8 @@ def token_inputs(
189
inputs["token_type_ids"] = token_type_ids
190
if multi_modal_data is not None:
191
inputs["multi_modal_data"] = multi_modal_data
192
+ if multi_modal_inputs is not None:
193
+ inputs["multi_modal_inputs"] = multi_modal_inputs
194
if multi_modal_placeholders is not None:
195
inputs["multi_modal_placeholders"] = multi_modal_placeholders
196
if mm_processor_kwargs is not None:
@@ -273,6 +283,18 @@ class SingletonInputsAdapter:
283
284
assert_never(inputs)
285
286
+ @cached_property
287
+ def multi_modal_inputs(self) -> Union[Dict, "MultiModalKwargs"]:
288
+ inputs = self.inputs
289
+
290
+ if inputs["type"] == "token":
291
+ return inputs.get("multi_modal_inputs", {})
292
+
293
+ if inputs["type"] == "multimodal":
294
+ return inputs.get("mm_kwargs", {})
295
+
296
+ assert_never(inputs)
297
+
298
@cached_property
299
def multi_modal_placeholders(self) -> "MultiModalPlaceholderDict":
300
inputs = self.inputs
vllm/v1/engine/__init__.py
+3
-4
@@ -1,11 +1,11 @@
1
import enum
2
from dataclasses import dataclass
3
-from typing import Any, Dict, List, Optional, Union
3
+from typing import List, Optional, Union
4
5
import msgspec
6
7
from vllm.lora.request import LoRARequest
8
-from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
8
+from vllm.multimodal import MultiModalKwargs, MultiModalPlaceholderDict
9
from vllm.sampling_params import RequestOutputKind, SamplingParams
10
11
@@ -35,9 +35,8 @@ class EngineCoreRequest:
35
# always be tokenized?
36
prompt: Optional[str]
37
prompt_token_ids: List[int]
38
- mm_data: Optional[MultiModalDataDict]
38
+ mm_inputs: Optional[List[MultiModalKwargs]]
39
mm_placeholders: Optional[MultiModalPlaceholderDict]
40
- mm_processor_kwargs: Optional[Dict[str, Any]]
40
sampling_params: SamplingParams
41
eos_token_id: Optional[int]
42
arrival_time: float
vllm/v1/engine/core.py
-7
@@ -84,14 +84,7 @@ class EngineCore:
84
85
def add_request(self, request: EngineCoreRequest):
86
"""Add request to the scheduler."""
87
-
87
req = Request.from_engine_core_request(request)
89
- # FIXME(woosuk): The input mapping (e.g., PIL images to tensors) may
90
- # take 10-50 ms, which can cause a spike in the latency. We should
91
- # consider moving this to a separate thread.
92
- if req.mm_data:
93
- req.mm_inputs = self.mm_input_mapper.process_inputs(
94
- req.mm_data, req.mm_processor_kwargs)
88
self.scheduler.add_request(req)
89
90
def abort_requests(self, request_ids: List[str]):
vllm/v1/engine/processor.py
+11
-2
@@ -14,6 +14,7 @@ from vllm.sampling_params import SamplingParams
14
from vllm.transformers_utils.config import try_get_generation_config
15
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16
from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
17
+from vllm.v1.engine.mm_input_mapper import MMInputMapper
18
19
20
class Processor:
@@ -39,6 +40,9 @@ class Processor:
40
self.input_processor = input_registry.create_input_processor(
41
model_config)
42
43
+ # Multi-modal (huggingface) input mapper
44
+ self.mm_input_mapper = MMInputMapper(model_config)
45
+
46
# TODO: run in an ThreadpoolExecutor or BackgroundProcess.
47
# This ideally should releases the GIL, so we should not block the
48
# asyncio loop while this is running.
@@ -96,6 +100,12 @@ class Processor:
100
sampling_params.update_from_generation_config(
101
self.generation_config_fields, eos_token_id)
102
103
+ # Preprocess multi-modal data
104
+ mm_inputs = self.mm_input_mapper.process_inputs(
105
+ decoder_inputs.multi_modal_data,
106
+ decoder_inputs.mm_processor_kwargs) if len(
107
+ decoder_inputs.multi_modal_data) > 0 else None
108
+
109
# Make Request for Detokenizer.
110
detokenizer_request = DetokenizerRequest(
111
request_id,
@@ -113,9 +123,8 @@ class Processor:
123
request_id,
124
decoder_inputs.prompt,
125
decoder_inputs.prompt_token_ids,
116
- decoder_inputs.multi_modal_data,
126
+ mm_inputs,
127
decoder_inputs.multi_modal_placeholders,
118
- decoder_inputs.mm_processor_kwargs,
128
sampling_params,
129
eos_token_id,
130
arrival_time,
vllm/v1/request.py
+8
-7
@@ -45,9 +45,6 @@ class Request:
45
self._all_token_ids: List[int] = self.prompt_token_ids.copy()
46
self.num_computed_tokens = 0
47
48
- # Raw multimodal data before the mm input mapper (e.g., PIL images).
49
- self.mm_data = self.inputs.multi_modal_data
50
- self.mm_processor_kwargs = self.inputs.mm_processor_kwargs
48
mm_positions = self.inputs.multi_modal_placeholders
49
if mm_positions:
50
# FIXME(woosuk): Support other modalities.
@@ -55,7 +52,10 @@ class Request:
52
else:
53
self.mm_positions = []
54
# Output of the mm input mapper (e.g., image tensors).
58
- self.mm_inputs: List[MultiModalKwargs] = []
55
+ if self.inputs.multi_modal_inputs:
56
+ self.mm_inputs = self.inputs.multi_modal_inputs
57
+ else:
58
+ self.mm_inputs: List[MultiModalKwargs] = []
59
60
@classmethod
61
def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
@@ -64,9 +64,10 @@ class Request:
64
inputs=token_inputs(
65
prompt_token_ids=request.prompt_token_ids,
66
prompt=request.prompt,
67
- multi_modal_data=request.mm_data,
67
+ multi_modal_data=None,
68
+ multi_modal_inputs=request.mm_inputs,
69
multi_modal_placeholders=request.mm_placeholders,
69
- mm_processor_kwargs=request.mm_processor_kwargs,
70
+ mm_processor_kwargs=None,
71
),
72
sampling_params=request.sampling_params,
73
eos_token_id=request.eos_token_id,
@@ -110,7 +111,7 @@ class Request:
111
return RequestStatus.get_finished_reason(self.status)
112
113
def has_encoder_inputs(self) -> bool:
113
- return len(self.mm_data) > 0
114
+ return len(self.mm_inputs) > 0
115
116
@property
117
def num_encoder_inputs(self) -> int: