A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1] Clarify input processing and multimodal feature caching logic (#13211)
Roger Wang committed
Feb 13, 2025 at 03:43 UTC
fdcf64d3c6ad5ee2339a669214b43e4b735d0895
4 files changed
+45
-27
vllm/v1/engine/core.py
+8
-8
@@ -20,7 +20,7 @@ from vllm.v1.core.kv_cache_utils import get_kv_cache_configs
20
from vllm.v1.core.scheduler import Scheduler
21
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
22
EngineCoreRequestType)
23
-from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
23
+from vllm.v1.engine.mm_input_cache import MMInputCacheServer
24
from vllm.v1.executor.abstract import Executor
25
from vllm.v1.request import Request, RequestStatus
26
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
@@ -65,7 +65,7 @@ class EngineCore:
65
log_stats=self.log_stats,
66
)
67
68
- self.mm_input_mapper_server = MMInputMapperServer(
68
+ self.mm_input_cache_server = MMInputCacheServer(
69
vllm_config.model_config)
70
71
def _initialize_kv_caches(self,
@@ -102,13 +102,13 @@ class EngineCore:
102
"""Add request to the scheduler."""
103
104
if request.mm_hashes is not None:
105
- # Here, if hash exists for an image, then it will be fetched
106
- # from the cache, else it will be added to the cache.
107
- # Note that the cache here is mirrored with the client side of the
108
- # MM mapper, so anything that has a hash must have a HIT cache
109
- # entry here as well.
105
+ # Here, if hash exists for a multimodal input, then it will be
106
+ # fetched from the cache, else it will be added to the cache.
107
+ # Note that the cache here is mirrored with the client cache, so
108
+ # anything that has a hash must have a HIT cache entry here
109
+ # as well.
110
assert request.mm_inputs is not None
111
- request.mm_inputs = self.mm_input_mapper_server.process_inputs(
111
+ request.mm_inputs = self.mm_input_cache_server.get_and_update(
112
request.mm_inputs, request.mm_hashes)
113
114
req = Request.from_engine_core_request(request)
vllm/v1/engine/mm_input_cache.py
renamed
+19
-10
@@ -10,12 +10,18 @@ from vllm.utils import LRUCache
10
11
logger = init_logger(__name__)
12
13
-# The idea of MM preprocessor caching is based on having a client and a server,
14
-# where the client executes in the frontend process (=P0) and the server in the
15
-# core process (=P1).
13
+# The idea of multimodal preprocessing caching is based on having a client and
14
+# a server, where the client executes in the frontend process (=P0) and the
15
+# server in the core process (=P1).
16
#
17
-# -- Client: Executes the MM mapper and performs caching of the results.
18
-# -- Server: Performs caching of the results
17
+# -- Client:
18
+# - Apply legacy input_mapper (if one exists) to generate MultiModalKwargs.
19
+# - Perform caching of the generated MultiModalKwargs.
20
+# - This client can be deprecated once all mutimodal models migrate to use
21
+# merged preprocessor with built-in caching functionality.
22
+#
23
+# -- Server:
24
+# - Perform caching of the received MultiModalKwargs.
25
#
26
# The caching for both client and server is mirrored/similar, and this allows us
27
# to avoid the serialization of "mm_inputs" (like pixel values) between
@@ -27,7 +33,9 @@ logger = init_logger(__name__)
33
MM_CACHE_SIZE = 256
34
35
30
-class MMInputMapperClient:
36
+# TODO(ywang96): Deprecate this class once all multimodal models migrate to use
37
+# merged preprocessor with built-in caching functionality.
38
+class MMInputCacheClient:
39
40
def __init__(
41
self,
@@ -54,7 +62,8 @@ class MMInputMapperClient:
62
logger.debug("MMInputMapper: cache_hit_ratio = %.2f ",
63
self.mm_cache_hits / self.mm_cache_total)
64
57
- # TODO: Support modalities beyond image.
65
+ # NOTE: process_inputs only supports image inputs since all multimodal
66
+ # models with other modalities have migrated to use merged preprocessor.
67
def process_inputs(
68
self,
69
mm_data: MultiModalDataDict,
@@ -95,7 +104,7 @@ class MMInputMapperClient:
104
# Reuse precomputed input (for merged preprocessor)
105
mm_input = precomputed_mm_inputs[input_id]
106
else:
98
- # Apply MM mapper
107
+ # Apply legacy input_mapper
108
mm_input = self.multi_modal_input_mapper(
109
{"image": [image_inputs[input_id]]},
110
mm_processor_kwargs=mm_processor_kwargs,
@@ -114,13 +123,13 @@ class MMInputMapperClient:
123
return ret_inputs
124
125
117
-class MMInputMapperServer:
126
+class MMInputCacheServer:
127
128
def __init__(self, model_config):
129
self.use_cache = not model_config.disable_mm_preprocessor_cache
130
self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
131
123
- def process_inputs(
132
+ def get_and_update(
133
self,
134
mm_inputs: List[Optional[MultiModalKwargs]],
135
mm_hashes: List[str],
vllm/v1/engine/processor.py
+14
-6
@@ -17,7 +17,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
17
from vllm.sampling_params import SamplingParams
18
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
19
from vllm.v1.engine import EngineCoreRequest
20
-from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
20
+from vllm.v1.engine.mm_input_cache import MMInputCacheClient
21
22
23
class Processor:
@@ -46,7 +46,7 @@ class Processor:
46
model_config)
47
48
# Multi-modal (huggingface) input mapper
49
- self.mm_input_mapper_client = MMInputMapperClient(model_config)
49
+ self.mm_input_cache_client = MMInputCacheClient(model_config)
50
51
# Multi-modal hasher (for images)
52
self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
@@ -106,16 +106,24 @@ class Processor:
106
assert priority == 0, "vLLM V1 does not support priority at the moment."
107
assert trace_headers is None, "vLLM V1 does not support tracing yet."
108
109
- # Process inputs.
109
+ # Process inputs, which includes:
110
+ # 1. Tokenize text prompt, with LoRA request if one exists.
111
+ # 2. For multimodal models with a merged preprocessor, preprocess
112
+ # multimodal data and expand prompt token ids accordingly.
113
+ # 3. Apply prompt adapter to prompt token ids if one exists.
114
preprocessed_inputs = self.input_preprocessor.preprocess(
115
prompt,
116
request_id=request_id,
117
lora_request=lora_request,
118
prompt_adapter_request=prompt_adapter_request,
119
)
120
+ eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
121
+
122
+ # Process prompt and prompt token ids.
123
+ # Only applicable to multimodal models with legacy input processor.
124
processed_inputs = self.input_processor(preprocessed_inputs)
125
+
126
self._validate_model_inputs(processed_inputs)
118
- eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
127
128
if is_encoder_decoder_inputs(processed_inputs):
129
decoder_inputs = SingletonInputsAdapter(
@@ -200,8 +208,8 @@ class Processor:
208
key=lambda mm_input: modality_order_dict[list(
209
mm_input.modalities)[0]])
210
203
- # Apply mm input cache update (and input mapper if necessary).
204
- sorted_mm_inputs = self.mm_input_mapper_client.process_inputs(
211
+ # Apply mm input cache update and legacy input mapper if one exists.
212
+ sorted_mm_inputs = self.mm_input_cache_client.process_inputs(
213
mm_data=decoder_mm_data,
214
mm_hashes=sorted_mm_hashes,
215
mm_processor_kwargs=decoder_inputs.mm_processor_kwargs,
vllm/v1/worker/gpu_model_runner.py
+4
-3
@@ -27,7 +27,7 @@ from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
27
from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
28
FlashAttentionMetadata)
29
from vllm.v1.core.encoder_cache_manager import compute_encoder_budget
30
-from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
30
+from vllm.v1.engine.mm_input_cache import MMInputCacheClient
31
from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
32
KVCacheSpec)
33
from vllm.v1.outputs import LogprobsTensors, ModelRunnerOutput
@@ -95,9 +95,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
95
self.mm_registry = MULTIMODAL_REGISTRY
96
self.uses_mrope = model_config.uses_mrope
97
98
- # NOTE: Initialized input mapper is only used for processing dummy
98
+ # NOTE: Initialized client is only used for processing dummy
99
# multimodal data into multimodal kwargs for GPU memory profiling.
100
- self.mm_input_mapper_profiling = MMInputMapperClient(self.model_config)
100
+ # Only applicable to multimodal models with legacy input mapper.
101
+ self.mm_input_mapper_profiling = MMInputCacheClient(self.model_config)
102
self.mm_input_mapper_profiling.use_cache = False
103
104
encoder_compute_budget, encoder_cache_size = compute_encoder_budget(