A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] VLM preprocessor hashing (#11020)
Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: Alexander Matveev <alexm@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Alexander Matveev committed
Dec 11, 2024 at 19:55 UTC
4e116833686f3e0c0a223b05b5859ad76843a017
11 files changed
+332
-48
examples/offline_inference_vision_language.py
+109
-17
@@ -5,6 +5,8 @@ the correct prompt format on vision language models for text generation.
5
For most models, the prompt format should follow corresponding examples
6
on HuggingFace model repository.
7
"""
8
+import random
9
+
10
from transformers import AutoTokenizer
11
12
from vllm import LLM, SamplingParams
@@ -23,7 +25,9 @@ def run_llava(question: str, modality: str):
25
26
prompt = f"USER: <image>\n{question}\nASSISTANT:"
27
26
- llm = LLM(model="llava-hf/llava-1.5-7b-hf", max_model_len=4096)
28
+ llm = LLM(model="llava-hf/llava-1.5-7b-hf",
29
+ max_model_len=4096,
30
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
31
stop_token_ids = None
32
return llm, prompt, stop_token_ids
33
@@ -33,7 +37,9 @@ def run_llava_next(question: str, modality: str):
37
assert modality == "image"
38
39
prompt = f"[INST] <image>\n{question} [/INST]"
36
- llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=8192)
40
+ llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf",
41
+ max_model_len=8192,
42
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
43
stop_token_ids = None
44
return llm, prompt, stop_token_ids
45
@@ -44,7 +50,9 @@ def run_llava_next_video(question: str, modality: str):
50
assert modality == "video"
51
52
prompt = f"USER: <video>\n{question} ASSISTANT:"
47
- llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf", max_model_len=8192)
53
+ llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf",
54
+ max_model_len=8192,
55
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
56
stop_token_ids = None
57
return llm, prompt, stop_token_ids
58
@@ -61,7 +69,8 @@ def run_llava_onevision(question: str, modality: str):
69
<|im_start|>assistant\n"
70
71
llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
64
- max_model_len=16384)
72
+ max_model_len=16384,
73
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
74
stop_token_ids = None
75
return llm, prompt, stop_token_ids
76
@@ -71,7 +80,10 @@ def run_fuyu(question: str, modality: str):
80
assert modality == "image"
81
82
prompt = f"{question}\n"
74
- llm = LLM(model="adept/fuyu-8b", max_model_len=2048, max_num_seqs=2)
83
+ llm = LLM(model="adept/fuyu-8b",
84
+ max_model_len=2048,
85
+ max_num_seqs=2,
86
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
87
stop_token_ids = None
88
return llm, prompt, stop_token_ids
89
@@ -107,6 +119,7 @@ def run_phi3v(question: str, modality: str):
119
max_num_seqs=2,
120
# Note - mm_processor_kwargs can also be passed to generate/chat calls
121
mm_processor_kwargs={"num_crops": 16},
122
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
123
)
124
stop_token_ids = None
125
return llm, prompt, stop_token_ids
@@ -118,7 +131,8 @@ def run_paligemma(question: str, modality: str):
131
132
# PaliGemma has special prompt format for VQA
133
prompt = "caption en"
121
- llm = LLM(model="google/paligemma-3b-mix-224")
134
+ llm = LLM(model="google/paligemma-3b-mix-224",
135
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
136
stop_token_ids = None
137
return llm, prompt, stop_token_ids
138
@@ -128,7 +142,9 @@ def run_chameleon(question: str, modality: str):
142
assert modality == "image"
143
144
prompt = f"{question}<image>"
131
- llm = LLM(model="facebook/chameleon-7b", max_model_len=4096)
145
+ llm = LLM(model="facebook/chameleon-7b",
146
+ max_model_len=4096,
147
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
148
stop_token_ids = None
149
return llm, prompt, stop_token_ids
150
@@ -154,6 +170,7 @@ def run_minicpmv(question: str, modality: str):
170
max_model_len=4096,
171
max_num_seqs=2,
172
trust_remote_code=True,
173
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
174
)
175
# NOTE The stop_token_ids are different for various versions of MiniCPM-V
176
# 2.0
@@ -186,6 +203,7 @@ def run_h2ovl(question: str, modality: str):
203
model=model_name,
204
trust_remote_code=True,
205
max_model_len=8192,
206
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
207
)
208
209
tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -211,6 +229,7 @@ def run_internvl(question: str, modality: str):
229
model=model_name,
230
trust_remote_code=True,
231
max_model_len=4096,
232
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
233
)
234
235
tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -241,6 +260,7 @@ def run_nvlm_d(question: str, modality: str):
260
trust_remote_code=True,
261
max_model_len=4096,
262
tensor_parallel_size=4,
263
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
264
)
265
266
tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -260,7 +280,8 @@ def run_blip2(question: str, modality: str):
280
# BLIP-2 prompt format is inaccurate on HuggingFace model repository.
281
# See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
282
prompt = f"Question: {question} Answer:"
263
- llm = LLM(model="Salesforce/blip2-opt-2.7b")
283
+ llm = LLM(model="Salesforce/blip2-opt-2.7b",
284
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
285
stop_token_ids = None
286
return llm, prompt, stop_token_ids
287
@@ -274,6 +295,7 @@ def run_qwen_vl(question: str, modality: str):
295
trust_remote_code=True,
296
max_model_len=1024,
297
max_num_seqs=2,
298
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
299
)
300
301
prompt = f"{question}Picture 1: <img></img>\n"
@@ -296,6 +318,7 @@ def run_qwen2_vl(question: str, modality: str):
318
"min_pixels": 28 * 28,
319
"max_pixels": 1280 * 28 * 28,
320
},
321
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
322
)
323
324
prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
@@ -315,6 +338,7 @@ def run_pixtral_hf(question: str, modality: str):
338
llm = LLM(
339
model=model_name,
340
max_model_len=8192,
341
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
342
)
343
344
prompt = f"<s>[INST]{question}\n[IMG][/INST]"
@@ -338,6 +362,7 @@ def run_mllama(question: str, modality: str):
362
max_model_len=4096,
363
max_num_seqs=16,
364
enforce_eager=True,
365
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
366
)
367
368
prompt = f"<|image|><|begin_of_text|>{question}"
@@ -355,6 +380,7 @@ def run_molmo(question, modality):
380
model=model_name,
381
trust_remote_code=True,
382
dtype="bfloat16",
383
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
384
)
385
386
prompt = question
@@ -371,7 +397,8 @@ def run_glm4v(question: str, modality: str):
397
max_model_len=2048,
398
max_num_seqs=2,
399
trust_remote_code=True,
374
- enforce_eager=True)
400
+ enforce_eager=True,
401
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
402
prompt = question
403
stop_token_ids = [151329, 151336, 151338]
404
return llm, prompt, stop_token_ids
@@ -394,6 +421,7 @@ def run_idefics3(question: str, modality: str):
421
"longest_edge": 3 * 364
422
},
423
},
424
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
425
)
426
prompt = (
427
f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
@@ -410,7 +438,8 @@ def run_aria(question: str, modality: str):
438
llm = LLM(model=model_name,
439
tokenizer_mode="slow",
440
trust_remote_code=True,
413
- dtype="bfloat16")
441
+ dtype="bfloat16",
442
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
443
444
prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
445
"<|im_end|>\n<|im_start|>assistant\n")
@@ -430,6 +459,7 @@ def run_mantis(question: str, modality: str):
459
model="TIGER-Lab/Mantis-8B-siglip-llama3",
460
max_model_len=4096,
461
hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
462
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
463
)
464
stop_token_ids = [128009]
465
return llm, prompt, stop_token_ids
@@ -494,6 +524,35 @@ def get_multi_modal_input(args):
524
raise ValueError(msg)
525
526
527
+def apply_image_repeat(image_repeat_prob, num_prompts, data, prompt, modality):
528
+ """Repeats images with provided probability of "image_repeat_prob".
529
+ Used to simulate hit/miss for the MM preprocessor cache.
530
+ """
531
+ assert (image_repeat_prob <= 1.0 and image_repeat_prob >= 0)
532
+ no_yes = [0, 1]
533
+ probs = [1.0 - image_repeat_prob, image_repeat_prob]
534
+
535
+ inputs = []
536
+ cur_image = data
537
+ for i in range(num_prompts):
538
+ if image_repeat_prob is not None:
539
+ res = random.choices(no_yes, probs)[0]
540
+ if res == 0:
541
+ # No repeat => Modify one pixel
542
+ cur_image = cur_image.copy()
543
+ new_val = (i // 256 // 256, i // 256, i % 256)
544
+ cur_image.putpixel((0, 0), new_val)
545
+
546
+ inputs.append({
547
+ "prompt": prompt,
548
+ "multi_modal_data": {
549
+ modality: cur_image
550
+ }
551
+ })
552
+
553
+ return inputs
554
+
555
+
556
def main(args):
557
model = args.model_type
558
if model not in model_example_map:
@@ -524,14 +583,29 @@ def main(args):
583
584
else:
585
# Batch inference
527
- inputs = [{
528
- "prompt": prompt,
529
- "multi_modal_data": {
530
- modality: data
531
- },
532
- } for _ in range(args.num_prompts)]
586
+ if args.image_repeat_prob is not None:
587
+ # Repeat images with specified probability of "image_repeat_prob"
588
+ inputs = apply_image_repeat(args.image_repeat_prob,
589
+ args.num_prompts, data, prompt,
590
+ modality)
591
+ else:
592
+ # Use the same image for all prompts
593
+ inputs = [{
594
+ "prompt": prompt,
595
+ "multi_modal_data": {
596
+ modality: data
597
+ },
598
+ } for _ in range(args.num_prompts)]
599
+
600
+ if args.time_generate:
601
+ import time
602
+ start_time = time.time()
603
+ outputs = llm.generate(inputs, sampling_params=sampling_params)
604
+ elapsed_time = time.time() - start_time
605
+ print("-- generate time = {}".format(elapsed_time))
606
534
- outputs = llm.generate(inputs, sampling_params=sampling_params)
607
+ else:
608
+ outputs = llm.generate(inputs, sampling_params=sampling_params)
609
610
for o in outputs:
611
generated_text = o.outputs[0].text
@@ -561,5 +635,23 @@ if __name__ == "__main__":
635
type=int,
636
default=16,
637
help='Number of frames to extract from the video.')
638
+
639
+ parser.add_argument(
640
+ '--image-repeat-prob',
641
+ type=float,
642
+ default=None,
643
+ help='Simulates the hit-ratio for multi-modal preprocessor cache'
644
+ ' (if enabled)')
645
+
646
+ parser.add_argument(
647
+ '--mm-cache-preprocessor',
648
+ action='store_true',
649
+ help='If True, enable caching of multi-modal preprocessor/mapper.')
650
+
651
+ parser.add_argument(
652
+ '--time-generate',
653
+ action='store_true',
654
+ help='If True, then print the total generate() call time')
655
+
656
args = parser.parse_args()
657
main(args)
requirements-common.txt
+1
@@ -3,6 +3,7 @@ sentencepiece # Required for LLaMA tokenizer.
3
numpy < 2.0.0
4
requests >= 2.26.0
5
tqdm
6
+blake3
7
py-cpuinfo
8
transformers >= 4.45.2 # Required for Llama 3.2 and Qwen2-VL.
9
tokenizers >= 0.19.1 # Required for Llama 3.
tests/v1/engine/test_engine_core.py
+1
@@ -28,6 +28,7 @@ def make_request() -> EngineCoreRequest:
28
prompt=PROMPT,
29
prompt_token_ids=PROMPT_TOKENS,
30
mm_inputs=None,
31
+ mm_hashes=None,
32
mm_placeholders=None,
33
sampling_params=SamplingParams(),
34
eos_token_id=None,
tests/v1/engine/test_engine_core_client.py
+1
@@ -30,6 +30,7 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:
30
prompt=PROMPT,
31
prompt_token_ids=PROMPT_TOKENS,
32
mm_inputs=None,
33
+ mm_hashes=None,
34
mm_placeholders=None,
35
sampling_params=params,
36
eos_token_id=None,
vllm/config.py
+8
-2
@@ -147,6 +147,9 @@ class ModelConfig:
147
HuggingFace config.
148
mm_processor_kwargs: Arguments to be forwarded to the model's processor
149
for multi-modal data, e.g., image processor.
150
+ mm_cache_preprocessor: If true, then enables caching of the multi-modal
151
+ preprocessor/mapper. Otherwise, the mapper executes each time, and
152
+ for better performance consider enabling frontend process.
153
override_neuron_config: Initialize non default neuron config or
154
override default neuron config that are specific to Neuron devices,
155
this argument will be used to configure the neuron config that
@@ -185,6 +188,7 @@ class ModelConfig:
188
config_format: ConfigFormat = ConfigFormat.AUTO,
189
hf_overrides: Optional[HfOverrides] = None,
190
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
191
+ mm_cache_preprocessor: bool = False,
192
override_neuron_config: Optional[Dict[str, Any]] = None,
193
override_pooler_config: Optional["PoolerConfig"] = None) -> None:
194
self.model = model
@@ -251,6 +255,7 @@ class ModelConfig:
255
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
256
self.use_async_output_proc = use_async_output_proc
257
self.mm_processor_kwargs = mm_processor_kwargs
258
+ self.mm_cache_preprocessor = mm_cache_preprocessor
259
260
# Set enforce_eager to False if the value is unset.
261
if self.enforce_eager is None:
@@ -2686,9 +2691,10 @@ class VllmConfig:
2691
f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "
2692
f"chunked_prefill_enabled={self.scheduler_config.chunked_prefill_enabled}, " # noqa
2693
f"use_async_output_proc={self.model_config.use_async_output_proc}, "
2694
+ f"mm_cache_preprocessor={self.model_config.mm_cache_preprocessor!r}, " # noqa
2695
f"mm_processor_kwargs={self.model_config.mm_processor_kwargs}, "
2690
- f"pooler_config={self.model_config.pooler_config!r},"
2691
- f" compilation_config={self.compilation_config!r}")
2696
+ f"pooler_config={self.model_config.pooler_config!r}, "
2697
+ f"compilation_config={self.compilation_config!r}")
2698
2699
2700
_current_vllm_config: Optional[VllmConfig] = None
vllm/engine/arg_utils.py
+8
@@ -143,6 +143,7 @@ class EngineArgs:
143
tokenizer_pool_extra_config: Optional[Dict[str, Any]] = None
144
limit_mm_per_prompt: Optional[Mapping[str, int]] = None
145
mm_processor_kwargs: Optional[Dict[str, Any]] = None
146
+ mm_cache_preprocessor: bool = False
147
enable_lora: bool = False
148
enable_lora_bias: bool = False
149
max_loras: int = 1
@@ -593,6 +594,12 @@ class EngineArgs:
594
type=json.loads,
595
help=('Overrides for the multimodal input mapping/processing, '
596
'e.g., image processor. For example: {"num_crops": 4}.'))
597
+ parser.add_argument(
598
+ '--mm-cache-preprocessor',
599
+ action='store_true',
600
+ help='If true, then enables caching of the multi-modal '
601
+ 'preprocessor/mapper. Otherwise, the mapper executes each time'
602
+ ', and for better performance consider enabling frontend process.')
603
604
# LoRA related configs
605
parser.add_argument('--enable-lora',
@@ -965,6 +972,7 @@ class EngineArgs:
972
use_async_output_proc=not self.disable_async_output_proc,
973
config_format=self.config_format,
974
mm_processor_kwargs=self.mm_processor_kwargs,
975
+ mm_cache_preprocessor=self.mm_cache_preprocessor,
976
override_neuron_config=self.override_neuron_config,
977
override_pooler_config=self.override_pooler_config,
978
)
vllm/v1/engine/__init__.py
+2
-1
@@ -35,7 +35,8 @@ class EngineCoreRequest:
35
# always be tokenized?
36
prompt: Optional[str]
37
prompt_token_ids: List[int]
38
- mm_inputs: Optional[List[MultiModalKwargs]]
38
+ mm_inputs: Optional[List[Optional[MultiModalKwargs]]]
39
+ mm_hashes: Optional[List[Optional[str]]]
40
mm_placeholders: Optional[MultiModalPlaceholderDict]
41
sampling_params: SamplingParams
42
eos_token_id: Optional[int]
vllm/v1/engine/core.py
+14
-4
@@ -18,7 +18,7 @@ from vllm.v1.core.scheduler import Scheduler
18
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
19
EngineCoreProfile, EngineCoreRequest,
20
EngineCoreRequestType)
21
-from vllm.v1.engine.mm_input_mapper import MMInputMapper
21
+from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
22
from vllm.v1.executor.abstract import Executor
23
from vllm.v1.request import Request, RequestStatus
24
from vllm.v1.serial_utils import PickleEncoder
@@ -55,9 +55,6 @@ class EngineCore:
55
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
56
vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks
57
58
- # Set up multimodal input mapper (e.g., convert PIL images to tensors).
59
- self.mm_input_mapper = MMInputMapper(vllm_config.model_config)
60
-
58
# Setup scheduler.
59
self.scheduler = Scheduler(vllm_config.scheduler_config,
60
vllm_config.cache_config,
@@ -65,6 +62,8 @@ class EngineCore:
62
63
self._last_logging_time = time.time()
64
65
+ self.mm_input_mapper_server = MMInputMapperServer()
66
+
67
def _initialize_kv_caches(self,
68
cache_config: CacheConfig) -> Tuple[int, int]:
69
start = time.time()
@@ -88,7 +87,18 @@ class EngineCore:
87
88
def add_request(self, request: EngineCoreRequest):
89
"""Add request to the scheduler."""
90
+
91
+ if request.mm_hashes is not None:
92
+ # Here, if hash exists for an image, then it will be fetched
93
+ # from the cache, else it will be added to the cache.
94
+ # Note that the cache here is mirrored with the client side of the
95
+ # MM mapper, so anything that has a hash must have a HIT cache
96
+ # entry here as well.
97
+ request.mm_inputs = self.mm_input_mapper_server.process_inputs(
98
+ request.mm_inputs, request.mm_hashes)
99
+
100
req = Request.from_engine_core_request(request)
101
+
102
self.scheduler.add_request(req)
103
104
def abort_requests(self, request_ids: List[str]):
vllm/v1/engine/mm_input_mapper.py
+144
-12
@@ -1,11 +1,35 @@
1
from typing import Any, Dict, List, Optional
2
3
+import PIL
4
+from blake3 import blake3
5
+
6
from vllm.config import ModelConfig
7
+from vllm.inputs import PromptType
8
+from vllm.logger import init_logger
9
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
10
MultiModalKwargs, MultiModalRegistry)
11
+from vllm.v1.utils import LRUDictCache
12
+
13
+logger = init_logger(__name__)
14
+
15
+# The idea of MM preprocessor caching is based on having a client and a server,
16
+# where the client executes in the frontend process (=P0) and the server in the
17
+# core process (=P1).
18
+#
19
+# -- Client: Executes the MM mapper and performs caching of the results.
20
+# -- Server: Performs caching of the results
21
+#
22
+# The caching for both client and server is mirrored/similar, and this allows us
23
+# to avoid the serialization of "mm_inputs" (like pixel values) between
24
+# client (=P0) and server (=P1) processes.
25
26
+# Both Client and Server must use the same cache size
27
+# (to perform mirrored caching)
28
+# TODO: Tune the MM cache size
29
+MM_CACHE_SIZE = 256
30
8
-class MMInputMapper:
31
+
32
+class MMInputMapperClient:
33
34
def __init__(
35
self,
@@ -18,23 +42,131 @@ class MMInputMapper:
42
model_config)
43
self.mm_registry.init_mm_limits_per_prompt(model_config)
44
45
+ self.mm_cache = LRUDictCache(MM_CACHE_SIZE)
46
+
47
+ # DEBUG: Set to None to disable
48
+ self.mm_debug_cache_hit_ratio_steps = None
49
+ self.mm_cache_hits = 0
50
+ self.mm_cache_total = 0
51
+
52
+ def cache_hit_ratio(self, steps) -> float:
53
+ if self.mm_cache_total > 0 and self.mm_cache_total % steps == 0:
54
+ logger.debug("MMInputMapper: cache_hit_ratio = %.2f ",
55
+ self.mm_cache_hits / self.mm_cache_total)
56
+
57
def process_inputs(
58
self,
59
mm_data: MultiModalDataDict,
60
+ mm_hashes: Optional[List[str]],
61
mm_processor_kwargs: Optional[Dict[str, Any]],
62
+ precomputed_mm_inputs: Optional[List[MultiModalKwargs]],
63
+ ) -> List[MultiModalKwargs]:
64
+ if precomputed_mm_inputs is None:
65
+ image_inputs = mm_data["image"]
66
+ if not isinstance(image_inputs, list):
67
+ image_inputs = [image_inputs]
68
+ num_inputs = len(image_inputs)
69
+ else:
70
+ num_inputs = len(precomputed_mm_inputs)
71
+
72
+ # Check if hash is enabled
73
+ use_hash = mm_hashes is not None
74
+ if use_hash:
75
+ assert num_inputs == len(
76
+ mm_hashes), "num_inputs = {} len(mm_hashes) = {}".format(
77
+ num_inputs, len(mm_hashes))
78
+
79
+ # Process each image input separately, so that later we can schedule
80
+ # them in a fine-grained manner.
81
+ # Apply caching (if enabled) and reuse precomputed inputs (if provided)
82
+ ret_hashes = [] if use_hash else None
83
+ ret_inputs: List[MultiModalKwargs] = []
84
+ for input_id in range(num_inputs):
85
+ if self.mm_debug_cache_hit_ratio_steps is not None:
86
+ self.cache_hit_ratio(self.mm_debug_cache_hit_ratio_steps)
87
+
88
+ mm_hash = None
89
+ mm_input = None
90
+ if use_hash:
91
+ mm_hash = mm_hashes[input_id]
92
+ mm_input = self.mm_cache.get(mm_hash)
93
+
94
+ self.mm_cache_total += 1
95
+ if mm_input is None:
96
+ if precomputed_mm_inputs is not None:
97
+ # Reuse precomputed input (for merged preprocessor)
98
+ mm_input = precomputed_mm_inputs[input_id]
99
+ else:
100
+ # Apply MM mapper
101
+ mm_input = self.multi_modal_input_mapper(
102
+ {"image": [image_inputs[input_id]]},
103
+ mm_processor_kwargs=mm_processor_kwargs,
104
+ )
105
+
106
+ if use_hash:
107
+ # Add to cache
108
+ self.mm_cache.put(mm_hash, mm_input)
109
+ else:
110
+ self.mm_cache_hits += 1
111
+ mm_input = None # Avoids sending mm_input to Server
112
+
113
+ if use_hash:
114
+ ret_hashes.append(mm_hash)
115
+ ret_inputs.append(mm_input)
116
+
117
+ return ret_inputs, ret_hashes
118
+
119
+
120
+class MMInputMapperServer:
121
+
122
+ def __init__(self, ):
123
+ self.mm_cache = LRUDictCache(MM_CACHE_SIZE)
124
+
125
+ def process_inputs(
126
+ self,
127
+ mm_inputs: List[Optional[MultiModalKwargs]],
128
+ mm_hashes: List[Optional[str]],
129
) -> List[MultiModalKwargs]:
130
+ assert len(mm_inputs) == len(mm_hashes)
131
+
132
+ full_mm_inputs = []
133
+ for mm_input, mm_hash in zip(mm_inputs, mm_hashes):
134
+ if mm_input is None:
135
+ mm_input = self.mm_cache.get(mm_hash)
136
+ assert mm_input is not None
137
+ else:
138
+ self.mm_cache.put(mm_hash, mm_input)
139
+
140
+ full_mm_inputs.append(mm_input)
141
+
142
+ return full_mm_inputs
143
+
144
+
145
+class MMHasher:
146
+
147
+ def __init__(self):
148
+ pass
149
+
150
+ def hash(self, prompt: PromptType) -> Optional[List[str]]:
151
+ if "multi_modal_data" not in prompt:
152
+ return None
153
+
154
+ mm_data = prompt["multi_modal_data"]
155
image_inputs = mm_data["image"]
156
if not isinstance(image_inputs, list):
157
image_inputs = [image_inputs]
158
+ assert len(image_inputs) > 0
159
30
- # Process each image input separately so that later we can schedule
31
- # them in a fine-grained manner.
32
- mm_inputs: List[MultiModalKwargs] = []
33
- num_images = len(image_inputs)
34
- for i in range(num_images):
35
- mm_input = self.multi_modal_input_mapper(
36
- {"image": image_inputs[i]},
37
- mm_processor_kwargs=mm_processor_kwargs,
38
- )
39
- mm_inputs.append(mm_input)
40
- return mm_inputs
160
+ ret = []
161
+ for image in image_inputs:
162
+ assert isinstance(image, PIL.Image.Image)
163
+
164
+ # Convert image to bytes
165
+ bytes = image.tobytes()
166
+
167
+ # Hash image bytes
168
+ hasher = blake3()
169
+ hasher.update(bytes)
170
+ ret.append(hasher.hexdigest())
171
+
172
+ return ret
vllm/v1/engine/processor.py
+23
-12
@@ -15,7 +15,7 @@ from vllm.sampling_params import SamplingParams
15
from vllm.transformers_utils.config import try_get_generation_config
16
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
17
from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
18
-from vllm.v1.engine.mm_input_mapper import MMInputMapper
18
+from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
19
20
21
class Processor:
@@ -42,7 +42,11 @@ class Processor:
42
model_config)
43
44
# Multi-modal (huggingface) input mapper
45
- self.mm_input_mapper = MMInputMapper(model_config)
45
+ self.mm_input_mapper_client = MMInputMapperClient(model_config)
46
+
47
+ # Multi-modal hasher (for images)
48
+ self.mm_hasher = MMHasher(
49
+ ) if model_config.mm_cache_preprocessor else None
50
51
# TODO: run in an ThreadpoolExecutor or BackgroundProcess.
52
# This ideally should releases the GIL, so we should not block the
@@ -71,6 +75,11 @@ class Processor:
75
assert priority == 0, "vLLM V1 does not support priority at the moment."
76
assert trace_headers is None, "vLLM V1 does not support tracing yet."
77
78
+ # Compute MM hashes (if enabled)
79
+ mm_hashes = None
80
+ if self.mm_hasher is not None:
81
+ mm_hashes = self.mm_hasher.hash(prompt)
82
+
83
# Process inputs.
84
preprocessed_inputs = self.input_preprocessor.preprocess(
85
prompt,
@@ -101,16 +110,17 @@ class Processor:
110
sampling_params.update_from_generation_config(
111
self.generation_config_fields, eos_token_id)
112
104
- # Preprocess multi-modal data
105
- if len(decoder_inputs.multi_modal_data) == 0:
106
- mm_inputs = None
107
- elif isinstance(decoder_inputs.multi_modal_data, MultiModalKwargs):
108
- mm_inputs = [decoder_inputs.multi_modal_data]
109
- else:
110
- mm_inputs = self.mm_input_mapper.process_inputs(
111
- decoder_inputs.multi_modal_data,
112
- decoder_inputs.mm_processor_kwargs,
113
- )
113
+ # For merged preprocessor, mm_data is already mm_inputs
114
+ precomputed_mm_inputs = None
115
+ if isinstance(decoder_inputs.multi_modal_data, MultiModalKwargs):
116
+ precomputed_mm_inputs = [decoder_inputs.multi_modal_data]
117
+
118
+ # Apply MM mapper
119
+ mm_inputs = None
120
+ if len(decoder_inputs.multi_modal_data) > 0:
121
+ mm_inputs, mm_hashes = self.mm_input_mapper_client.process_inputs(
122
+ decoder_inputs.multi_modal_data, mm_hashes,
123
+ decoder_inputs.mm_processor_kwargs, precomputed_mm_inputs)
124
125
# Make Request for Detokenizer.
126
detokenizer_request = DetokenizerRequest(
@@ -130,6 +140,7 @@ class Processor:
140
decoder_inputs.prompt,
141
decoder_inputs.prompt_token_ids,
142
mm_inputs,
143
+ mm_hashes,
144
decoder_inputs.multi_modal_placeholders,
145
sampling_params,
146
eos_token_id,
vllm/v1/utils.py
+21
@@ -1,3 +1,4 @@
1
+from collections import OrderedDict
2
from contextlib import contextmanager
3
from typing import Any, Generic, Iterator, List, TypeVar, overload
4
@@ -93,3 +94,23 @@ def make_zmq_socket(path: str, type: Any) -> Iterator[zmq.Socket]:
94
95
finally:
96
ctx.destroy(linger=0)
97
+
98
+
99
+class LRUDictCache:
100
+
101
+ def __init__(self, size: int):
102
+ self.cache = OrderedDict()
103
+ self.size = size
104
+
105
+ def get(self, key, default=None):
106
+ if key not in self.cache:
107
+ return default
108
+
109
+ self.cache.move_to_end(key)
110
+ return self.cache[key]
111
+
112
+ def put(self, key, value):
113
+ self.cache[key] = value
114
+ self.cache.move_to_end(key)
115
+ if len(self.cache) > self.size:
116
+ self.cache.popitem(last=False)