@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fdcf64d3c6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1] Clarify input processing and multimodal feature caching logic (#13211)

Roger Wang committed Feb 13, 2025 at 03:43 UTC fdcf64d3c6ad5ee2339a669214b43e4b735d0895
4 files changed +45 -27
vllm/v1/engine/core.py
+8 -8
@@ -20,7 +20,7 @@ from vllm.v1.core.kv_cache_utils import get_kv_cache_configs
20 from vllm.v1.core.scheduler import Scheduler
21 from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
22 EngineCoreRequestType)
23 -from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
23 +from vllm.v1.engine.mm_input_cache import MMInputCacheServer
24 from vllm.v1.executor.abstract import Executor
25 from vllm.v1.request import Request, RequestStatus
26 from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
@@ -65,7 +65,7 @@ class EngineCore:
65 log_stats=self.log_stats,
66 )
67
68 - self.mm_input_mapper_server = MMInputMapperServer(
68 + self.mm_input_cache_server = MMInputCacheServer(
69 vllm_config.model_config)
70
71 def _initialize_kv_caches(self,
@@ -102,13 +102,13 @@ class EngineCore:
102 """Add request to the scheduler."""
103
104 if request.mm_hashes is not None:
105 - # Here, if hash exists for an image, then it will be fetched
106 - # from the cache, else it will be added to the cache.
107 - # Note that the cache here is mirrored with the client side of the
108 - # MM mapper, so anything that has a hash must have a HIT cache
109 - # entry here as well.
105 + # Here, if hash exists for a multimodal input, then it will be
106 + # fetched from the cache, else it will be added to the cache.
107 + # Note that the cache here is mirrored with the client cache, so
108 + # anything that has a hash must have a HIT cache entry here
109 + # as well.
110 assert request.mm_inputs is not None
111 - request.mm_inputs = self.mm_input_mapper_server.process_inputs(
111 + request.mm_inputs = self.mm_input_cache_server.get_and_update(
112 request.mm_inputs, request.mm_hashes)
113
114 req = Request.from_engine_core_request(request)
vllm/v1/engine/mm_input_cache.py renamed
+19 -10
@@ -10,12 +10,18 @@ from vllm.utils import LRUCache
10
11 logger = init_logger(__name__)
12
13 -# The idea of MM preprocessor caching is based on having a client and a server,
14 -# where the client executes in the frontend process (=P0) and the server in the
15 -# core process (=P1).
13 +# The idea of multimodal preprocessing caching is based on having a client and
14 +# a server, where the client executes in the frontend process (=P0) and the
15 +# server in the core process (=P1).
16 #
17 -# -- Client: Executes the MM mapper and performs caching of the results.
18 -# -- Server: Performs caching of the results
17 +# -- Client:
18 +# - Apply legacy input_mapper (if one exists) to generate MultiModalKwargs.
19 +# - Perform caching of the generated MultiModalKwargs.
20 +# - This client can be deprecated once all mutimodal models migrate to use
21 +# merged preprocessor with built-in caching functionality.
22 +#
23 +# -- Server:
24 +# - Perform caching of the received MultiModalKwargs.
25 #
26 # The caching for both client and server is mirrored/similar, and this allows us
27 # to avoid the serialization of "mm_inputs" (like pixel values) between
@@ -27,7 +33,9 @@ logger = init_logger(__name__)
33 MM_CACHE_SIZE = 256
34
35
30 -class MMInputMapperClient:
36 +# TODO(ywang96): Deprecate this class once all multimodal models migrate to use
37 +# merged preprocessor with built-in caching functionality.
38 +class MMInputCacheClient:
39
40 def __init__(
41 self,
@@ -54,7 +62,8 @@ class MMInputMapperClient:
62 logger.debug("MMInputMapper: cache_hit_ratio = %.2f ",
63 self.mm_cache_hits / self.mm_cache_total)
64
57 - # TODO: Support modalities beyond image.
65 + # NOTE: process_inputs only supports image inputs since all multimodal
66 + # models with other modalities have migrated to use merged preprocessor.
67 def process_inputs(
68 self,
69 mm_data: MultiModalDataDict,
@@ -95,7 +104,7 @@ class MMInputMapperClient:
104 # Reuse precomputed input (for merged preprocessor)
105 mm_input = precomputed_mm_inputs[input_id]
106 else:
98 - # Apply MM mapper
107 + # Apply legacy input_mapper
108 mm_input = self.multi_modal_input_mapper(
109 {"image": [image_inputs[input_id]]},
110 mm_processor_kwargs=mm_processor_kwargs,
@@ -114,13 +123,13 @@ class MMInputMapperClient:
123 return ret_inputs
124
125
117 -class MMInputMapperServer:
126 +class MMInputCacheServer:
127
128 def __init__(self, model_config):
129 self.use_cache = not model_config.disable_mm_preprocessor_cache
130 self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
131
123 - def process_inputs(
132 + def get_and_update(
133 self,
134 mm_inputs: List[Optional[MultiModalKwargs]],
135 mm_hashes: List[str],
vllm/v1/engine/processor.py
+14 -6
@@ -17,7 +17,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
17 from vllm.sampling_params import SamplingParams
18 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
19 from vllm.v1.engine import EngineCoreRequest
20 -from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
20 +from vllm.v1.engine.mm_input_cache import MMInputCacheClient
21
22
23 class Processor:
@@ -46,7 +46,7 @@ class Processor:
46 model_config)
47
48 # Multi-modal (huggingface) input mapper
49 - self.mm_input_mapper_client = MMInputMapperClient(model_config)
49 + self.mm_input_cache_client = MMInputCacheClient(model_config)
50
51 # Multi-modal hasher (for images)
52 self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
@@ -106,16 +106,24 @@ class Processor:
106 assert priority == 0, "vLLM V1 does not support priority at the moment."
107 assert trace_headers is None, "vLLM V1 does not support tracing yet."
108
109 - # Process inputs.
109 + # Process inputs, which includes:
110 + # 1. Tokenize text prompt, with LoRA request if one exists.
111 + # 2. For multimodal models with a merged preprocessor, preprocess
112 + # multimodal data and expand prompt token ids accordingly.
113 + # 3. Apply prompt adapter to prompt token ids if one exists.
114 preprocessed_inputs = self.input_preprocessor.preprocess(
115 prompt,
116 request_id=request_id,
117 lora_request=lora_request,
118 prompt_adapter_request=prompt_adapter_request,
119 )
120 + eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
121 +
122 + # Process prompt and prompt token ids.
123 + # Only applicable to multimodal models with legacy input processor.
124 processed_inputs = self.input_processor(preprocessed_inputs)
125 +
126 self._validate_model_inputs(processed_inputs)
118 - eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
127
128 if is_encoder_decoder_inputs(processed_inputs):
129 decoder_inputs = SingletonInputsAdapter(
@@ -200,8 +208,8 @@ class Processor:
208 key=lambda mm_input: modality_order_dict[list(
209 mm_input.modalities)[0]])
210
203 - # Apply mm input cache update (and input mapper if necessary).
204 - sorted_mm_inputs = self.mm_input_mapper_client.process_inputs(
211 + # Apply mm input cache update and legacy input mapper if one exists.
212 + sorted_mm_inputs = self.mm_input_cache_client.process_inputs(
213 mm_data=decoder_mm_data,
214 mm_hashes=sorted_mm_hashes,
215 mm_processor_kwargs=decoder_inputs.mm_processor_kwargs,
vllm/v1/worker/gpu_model_runner.py
+4 -3
@@ -27,7 +27,7 @@ from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
27 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
28 FlashAttentionMetadata)
29 from vllm.v1.core.encoder_cache_manager import compute_encoder_budget
30 -from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
30 +from vllm.v1.engine.mm_input_cache import MMInputCacheClient
31 from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
32 KVCacheSpec)
33 from vllm.v1.outputs import LogprobsTensors, ModelRunnerOutput
@@ -95,9 +95,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
95 self.mm_registry = MULTIMODAL_REGISTRY
96 self.uses_mrope = model_config.uses_mrope
97
98 - # NOTE: Initialized input mapper is only used for processing dummy
98 + # NOTE: Initialized client is only used for processing dummy
99 # multimodal data into multimodal kwargs for GPU memory profiling.
100 - self.mm_input_mapper_profiling = MMInputMapperClient(self.model_config)
100 + # Only applicable to multimodal models with legacy input mapper.
101 + self.mm_input_mapper_profiling = MMInputCacheClient(self.model_config)
102 self.mm_input_mapper_profiling.use_cache = False
103
104 encoder_compute_budget, encoder_cache_size = compute_encoder_budget(