@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4e11683368
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] VLM preprocessor hashing (#11020)

Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: Alexander Matveev <alexm@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Alexander Matveev committed Dec 11, 2024 at 19:55 UTC 4e116833686f3e0c0a223b05b5859ad76843a017
11 files changed +332 -48
examples/offline_inference_vision_language.py
+109 -17
@@ -5,6 +5,8 @@ the correct prompt format on vision language models for text generation.
5 For most models, the prompt format should follow corresponding examples
6 on HuggingFace model repository.
7 """
8 +import random
9 +
10 from transformers import AutoTokenizer
11
12 from vllm import LLM, SamplingParams
@@ -23,7 +25,9 @@ def run_llava(question: str, modality: str):
25
26 prompt = f"USER: <image>\n{question}\nASSISTANT:"
27
26 - llm = LLM(model="llava-hf/llava-1.5-7b-hf", max_model_len=4096)
28 + llm = LLM(model="llava-hf/llava-1.5-7b-hf",
29 + max_model_len=4096,
30 + mm_cache_preprocessor=args.mm_cache_preprocessor)
31 stop_token_ids = None
32 return llm, prompt, stop_token_ids
33
@@ -33,7 +37,9 @@ def run_llava_next(question: str, modality: str):
37 assert modality == "image"
38
39 prompt = f"[INST] <image>\n{question} [/INST]"
36 - llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=8192)
40 + llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf",
41 + max_model_len=8192,
42 + mm_cache_preprocessor=args.mm_cache_preprocessor)
43 stop_token_ids = None
44 return llm, prompt, stop_token_ids
45
@@ -44,7 +50,9 @@ def run_llava_next_video(question: str, modality: str):
50 assert modality == "video"
51
52 prompt = f"USER: <video>\n{question} ASSISTANT:"
47 - llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf", max_model_len=8192)
53 + llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf",
54 + max_model_len=8192,
55 + mm_cache_preprocessor=args.mm_cache_preprocessor)
56 stop_token_ids = None
57 return llm, prompt, stop_token_ids
58
@@ -61,7 +69,8 @@ def run_llava_onevision(question: str, modality: str):
69 <|im_start|>assistant\n"
70
71 llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
64 - max_model_len=16384)
72 + max_model_len=16384,
73 + mm_cache_preprocessor=args.mm_cache_preprocessor)
74 stop_token_ids = None
75 return llm, prompt, stop_token_ids
76
@@ -71,7 +80,10 @@ def run_fuyu(question: str, modality: str):
80 assert modality == "image"
81
82 prompt = f"{question}\n"
74 - llm = LLM(model="adept/fuyu-8b", max_model_len=2048, max_num_seqs=2)
83 + llm = LLM(model="adept/fuyu-8b",
84 + max_model_len=2048,
85 + max_num_seqs=2,
86 + mm_cache_preprocessor=args.mm_cache_preprocessor)
87 stop_token_ids = None
88 return llm, prompt, stop_token_ids
89
@@ -107,6 +119,7 @@ def run_phi3v(question: str, modality: str):
119 max_num_seqs=2,
120 # Note - mm_processor_kwargs can also be passed to generate/chat calls
121 mm_processor_kwargs={"num_crops": 16},
122 + mm_cache_preprocessor=args.mm_cache_preprocessor,
123 )
124 stop_token_ids = None
125 return llm, prompt, stop_token_ids
@@ -118,7 +131,8 @@ def run_paligemma(question: str, modality: str):
131
132 # PaliGemma has special prompt format for VQA
133 prompt = "caption en"
121 - llm = LLM(model="google/paligemma-3b-mix-224")
134 + llm = LLM(model="google/paligemma-3b-mix-224",
135 + mm_cache_preprocessor=args.mm_cache_preprocessor)
136 stop_token_ids = None
137 return llm, prompt, stop_token_ids
138
@@ -128,7 +142,9 @@ def run_chameleon(question: str, modality: str):
142 assert modality == "image"
143
144 prompt = f"{question}<image>"
131 - llm = LLM(model="facebook/chameleon-7b", max_model_len=4096)
145 + llm = LLM(model="facebook/chameleon-7b",
146 + max_model_len=4096,
147 + mm_cache_preprocessor=args.mm_cache_preprocessor)
148 stop_token_ids = None
149 return llm, prompt, stop_token_ids
150
@@ -154,6 +170,7 @@ def run_minicpmv(question: str, modality: str):
170 max_model_len=4096,
171 max_num_seqs=2,
172 trust_remote_code=True,
173 + mm_cache_preprocessor=args.mm_cache_preprocessor,
174 )
175 # NOTE The stop_token_ids are different for various versions of MiniCPM-V
176 # 2.0
@@ -186,6 +203,7 @@ def run_h2ovl(question: str, modality: str):
203 model=model_name,
204 trust_remote_code=True,
205 max_model_len=8192,
206 + mm_cache_preprocessor=args.mm_cache_preprocessor,
207 )
208
209 tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -211,6 +229,7 @@ def run_internvl(question: str, modality: str):
229 model=model_name,
230 trust_remote_code=True,
231 max_model_len=4096,
232 + mm_cache_preprocessor=args.mm_cache_preprocessor,
233 )
234
235 tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -241,6 +260,7 @@ def run_nvlm_d(question: str, modality: str):
260 trust_remote_code=True,
261 max_model_len=4096,
262 tensor_parallel_size=4,
263 + mm_cache_preprocessor=args.mm_cache_preprocessor,
264 )
265
266 tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -260,7 +280,8 @@ def run_blip2(question: str, modality: str):
280 # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
281 # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
282 prompt = f"Question: {question} Answer:"
263 - llm = LLM(model="Salesforce/blip2-opt-2.7b")
283 + llm = LLM(model="Salesforce/blip2-opt-2.7b",
284 + mm_cache_preprocessor=args.mm_cache_preprocessor)
285 stop_token_ids = None
286 return llm, prompt, stop_token_ids
287
@@ -274,6 +295,7 @@ def run_qwen_vl(question: str, modality: str):
295 trust_remote_code=True,
296 max_model_len=1024,
297 max_num_seqs=2,
298 + mm_cache_preprocessor=args.mm_cache_preprocessor,
299 )
300
301 prompt = f"{question}Picture 1: <img></img>\n"
@@ -296,6 +318,7 @@ def run_qwen2_vl(question: str, modality: str):
318 "min_pixels": 28 * 28,
319 "max_pixels": 1280 * 28 * 28,
320 },
321 + mm_cache_preprocessor=args.mm_cache_preprocessor,
322 )
323
324 prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
@@ -315,6 +338,7 @@ def run_pixtral_hf(question: str, modality: str):
338 llm = LLM(
339 model=model_name,
340 max_model_len=8192,
341 + mm_cache_preprocessor=args.mm_cache_preprocessor,
342 )
343
344 prompt = f"<s>[INST]{question}\n[IMG][/INST]"
@@ -338,6 +362,7 @@ def run_mllama(question: str, modality: str):
362 max_model_len=4096,
363 max_num_seqs=16,
364 enforce_eager=True,
365 + mm_cache_preprocessor=args.mm_cache_preprocessor,
366 )
367
368 prompt = f"<|image|><|begin_of_text|>{question}"
@@ -355,6 +380,7 @@ def run_molmo(question, modality):
380 model=model_name,
381 trust_remote_code=True,
382 dtype="bfloat16",
383 + mm_cache_preprocessor=args.mm_cache_preprocessor,
384 )
385
386 prompt = question
@@ -371,7 +397,8 @@ def run_glm4v(question: str, modality: str):
397 max_model_len=2048,
398 max_num_seqs=2,
399 trust_remote_code=True,
374 - enforce_eager=True)
400 + enforce_eager=True,
401 + mm_cache_preprocessor=args.mm_cache_preprocessor)
402 prompt = question
403 stop_token_ids = [151329, 151336, 151338]
404 return llm, prompt, stop_token_ids
@@ -394,6 +421,7 @@ def run_idefics3(question: str, modality: str):
421 "longest_edge": 3 * 364
422 },
423 },
424 + mm_cache_preprocessor=args.mm_cache_preprocessor,
425 )
426 prompt = (
427 f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
@@ -410,7 +438,8 @@ def run_aria(question: str, modality: str):
438 llm = LLM(model=model_name,
439 tokenizer_mode="slow",
440 trust_remote_code=True,
413 - dtype="bfloat16")
441 + dtype="bfloat16",
442 + mm_cache_preprocessor=args.mm_cache_preprocessor)
443
444 prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
445 "<|im_end|>\n<|im_start|>assistant\n")
@@ -430,6 +459,7 @@ def run_mantis(question: str, modality: str):
459 model="TIGER-Lab/Mantis-8B-siglip-llama3",
460 max_model_len=4096,
461 hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
462 + mm_cache_preprocessor=args.mm_cache_preprocessor,
463 )
464 stop_token_ids = [128009]
465 return llm, prompt, stop_token_ids
@@ -494,6 +524,35 @@ def get_multi_modal_input(args):
524 raise ValueError(msg)
525
526
527 +def apply_image_repeat(image_repeat_prob, num_prompts, data, prompt, modality):
528 + """Repeats images with provided probability of "image_repeat_prob".
529 + Used to simulate hit/miss for the MM preprocessor cache.
530 + """
531 + assert (image_repeat_prob <= 1.0 and image_repeat_prob >= 0)
532 + no_yes = [0, 1]
533 + probs = [1.0 - image_repeat_prob, image_repeat_prob]
534 +
535 + inputs = []
536 + cur_image = data
537 + for i in range(num_prompts):
538 + if image_repeat_prob is not None:
539 + res = random.choices(no_yes, probs)[0]
540 + if res == 0:
541 + # No repeat => Modify one pixel
542 + cur_image = cur_image.copy()
543 + new_val = (i // 256 // 256, i // 256, i % 256)
544 + cur_image.putpixel((0, 0), new_val)
545 +
546 + inputs.append({
547 + "prompt": prompt,
548 + "multi_modal_data": {
549 + modality: cur_image
550 + }
551 + })
552 +
553 + return inputs
554 +
555 +
556 def main(args):
557 model = args.model_type
558 if model not in model_example_map:
@@ -524,14 +583,29 @@ def main(args):
583
584 else:
585 # Batch inference
527 - inputs = [{
528 - "prompt": prompt,
529 - "multi_modal_data": {
530 - modality: data
531 - },
532 - } for _ in range(args.num_prompts)]
586 + if args.image_repeat_prob is not None:
587 + # Repeat images with specified probability of "image_repeat_prob"
588 + inputs = apply_image_repeat(args.image_repeat_prob,
589 + args.num_prompts, data, prompt,
590 + modality)
591 + else:
592 + # Use the same image for all prompts
593 + inputs = [{
594 + "prompt": prompt,
595 + "multi_modal_data": {
596 + modality: data
597 + },
598 + } for _ in range(args.num_prompts)]
599 +
600 + if args.time_generate:
601 + import time
602 + start_time = time.time()
603 + outputs = llm.generate(inputs, sampling_params=sampling_params)
604 + elapsed_time = time.time() - start_time
605 + print("-- generate time = {}".format(elapsed_time))
606
534 - outputs = llm.generate(inputs, sampling_params=sampling_params)
607 + else:
608 + outputs = llm.generate(inputs, sampling_params=sampling_params)
609
610 for o in outputs:
611 generated_text = o.outputs[0].text
@@ -561,5 +635,23 @@ if __name__ == "__main__":
635 type=int,
636 default=16,
637 help='Number of frames to extract from the video.')
638 +
639 + parser.add_argument(
640 + '--image-repeat-prob',
641 + type=float,
642 + default=None,
643 + help='Simulates the hit-ratio for multi-modal preprocessor cache'
644 + ' (if enabled)')
645 +
646 + parser.add_argument(
647 + '--mm-cache-preprocessor',
648 + action='store_true',
649 + help='If True, enable caching of multi-modal preprocessor/mapper.')
650 +
651 + parser.add_argument(
652 + '--time-generate',
653 + action='store_true',
654 + help='If True, then print the total generate() call time')
655 +
656 args = parser.parse_args()
657 main(args)
requirements-common.txt
+1
@@ -3,6 +3,7 @@ sentencepiece # Required for LLaMA tokenizer.
3 numpy < 2.0.0
4 requests >= 2.26.0
5 tqdm
6 +blake3
7 py-cpuinfo
8 transformers >= 4.45.2 # Required for Llama 3.2 and Qwen2-VL.
9 tokenizers >= 0.19.1 # Required for Llama 3.
tests/v1/engine/test_engine_core.py
+1
@@ -28,6 +28,7 @@ def make_request() -> EngineCoreRequest:
28 prompt=PROMPT,
29 prompt_token_ids=PROMPT_TOKENS,
30 mm_inputs=None,
31 + mm_hashes=None,
32 mm_placeholders=None,
33 sampling_params=SamplingParams(),
34 eos_token_id=None,
tests/v1/engine/test_engine_core_client.py
+1
@@ -30,6 +30,7 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:
30 prompt=PROMPT,
31 prompt_token_ids=PROMPT_TOKENS,
32 mm_inputs=None,
33 + mm_hashes=None,
34 mm_placeholders=None,
35 sampling_params=params,
36 eos_token_id=None,
vllm/config.py
+8 -2
@@ -147,6 +147,9 @@ class ModelConfig:
147 HuggingFace config.
148 mm_processor_kwargs: Arguments to be forwarded to the model's processor
149 for multi-modal data, e.g., image processor.
150 + mm_cache_preprocessor: If true, then enables caching of the multi-modal
151 + preprocessor/mapper. Otherwise, the mapper executes each time, and
152 + for better performance consider enabling frontend process.
153 override_neuron_config: Initialize non default neuron config or
154 override default neuron config that are specific to Neuron devices,
155 this argument will be used to configure the neuron config that
@@ -185,6 +188,7 @@ class ModelConfig:
188 config_format: ConfigFormat = ConfigFormat.AUTO,
189 hf_overrides: Optional[HfOverrides] = None,
190 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
191 + mm_cache_preprocessor: bool = False,
192 override_neuron_config: Optional[Dict[str, Any]] = None,
193 override_pooler_config: Optional["PoolerConfig"] = None) -> None:
194 self.model = model
@@ -251,6 +255,7 @@ class ModelConfig:
255 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
256 self.use_async_output_proc = use_async_output_proc
257 self.mm_processor_kwargs = mm_processor_kwargs
258 + self.mm_cache_preprocessor = mm_cache_preprocessor
259
260 # Set enforce_eager to False if the value is unset.
261 if self.enforce_eager is None:
@@ -2686,9 +2691,10 @@ class VllmConfig:
2691 f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "
2692 f"chunked_prefill_enabled={self.scheduler_config.chunked_prefill_enabled}, " # noqa
2693 f"use_async_output_proc={self.model_config.use_async_output_proc}, "
2694 + f"mm_cache_preprocessor={self.model_config.mm_cache_preprocessor!r}, " # noqa
2695 f"mm_processor_kwargs={self.model_config.mm_processor_kwargs}, "
2690 - f"pooler_config={self.model_config.pooler_config!r},"
2691 - f" compilation_config={self.compilation_config!r}")
2696 + f"pooler_config={self.model_config.pooler_config!r}, "
2697 + f"compilation_config={self.compilation_config!r}")
2698
2699
2700 _current_vllm_config: Optional[VllmConfig] = None
vllm/engine/arg_utils.py
+8
@@ -143,6 +143,7 @@ class EngineArgs:
143 tokenizer_pool_extra_config: Optional[Dict[str, Any]] = None
144 limit_mm_per_prompt: Optional[Mapping[str, int]] = None
145 mm_processor_kwargs: Optional[Dict[str, Any]] = None
146 + mm_cache_preprocessor: bool = False
147 enable_lora: bool = False
148 enable_lora_bias: bool = False
149 max_loras: int = 1
@@ -593,6 +594,12 @@ class EngineArgs:
594 type=json.loads,
595 help=('Overrides for the multimodal input mapping/processing, '
596 'e.g., image processor. For example: {"num_crops": 4}.'))
597 + parser.add_argument(
598 + '--mm-cache-preprocessor',
599 + action='store_true',
600 + help='If true, then enables caching of the multi-modal '
601 + 'preprocessor/mapper. Otherwise, the mapper executes each time'
602 + ', and for better performance consider enabling frontend process.')
603
604 # LoRA related configs
605 parser.add_argument('--enable-lora',
@@ -965,6 +972,7 @@ class EngineArgs:
972 use_async_output_proc=not self.disable_async_output_proc,
973 config_format=self.config_format,
974 mm_processor_kwargs=self.mm_processor_kwargs,
975 + mm_cache_preprocessor=self.mm_cache_preprocessor,
976 override_neuron_config=self.override_neuron_config,
977 override_pooler_config=self.override_pooler_config,
978 )
vllm/v1/engine/__init__.py
+2 -1
@@ -35,7 +35,8 @@ class EngineCoreRequest:
35 # always be tokenized?
36 prompt: Optional[str]
37 prompt_token_ids: List[int]
38 - mm_inputs: Optional[List[MultiModalKwargs]]
38 + mm_inputs: Optional[List[Optional[MultiModalKwargs]]]
39 + mm_hashes: Optional[List[Optional[str]]]
40 mm_placeholders: Optional[MultiModalPlaceholderDict]
41 sampling_params: SamplingParams
42 eos_token_id: Optional[int]
vllm/v1/engine/core.py
+14 -4
@@ -18,7 +18,7 @@ from vllm.v1.core.scheduler import Scheduler
18 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
19 EngineCoreProfile, EngineCoreRequest,
20 EngineCoreRequestType)
21 -from vllm.v1.engine.mm_input_mapper import MMInputMapper
21 +from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
22 from vllm.v1.executor.abstract import Executor
23 from vllm.v1.request import Request, RequestStatus
24 from vllm.v1.serial_utils import PickleEncoder
@@ -55,9 +55,6 @@ class EngineCore:
55 vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
56 vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks
57
58 - # Set up multimodal input mapper (e.g., convert PIL images to tensors).
59 - self.mm_input_mapper = MMInputMapper(vllm_config.model_config)
60 -
58 # Setup scheduler.
59 self.scheduler = Scheduler(vllm_config.scheduler_config,
60 vllm_config.cache_config,
@@ -65,6 +62,8 @@ class EngineCore:
62
63 self._last_logging_time = time.time()
64
65 + self.mm_input_mapper_server = MMInputMapperServer()
66 +
67 def _initialize_kv_caches(self,
68 cache_config: CacheConfig) -> Tuple[int, int]:
69 start = time.time()
@@ -88,7 +87,18 @@ class EngineCore:
87
88 def add_request(self, request: EngineCoreRequest):
89 """Add request to the scheduler."""
90 +
91 + if request.mm_hashes is not None:
92 + # Here, if hash exists for an image, then it will be fetched
93 + # from the cache, else it will be added to the cache.
94 + # Note that the cache here is mirrored with the client side of the
95 + # MM mapper, so anything that has a hash must have a HIT cache
96 + # entry here as well.
97 + request.mm_inputs = self.mm_input_mapper_server.process_inputs(
98 + request.mm_inputs, request.mm_hashes)
99 +
100 req = Request.from_engine_core_request(request)
101 +
102 self.scheduler.add_request(req)
103
104 def abort_requests(self, request_ids: List[str]):
vllm/v1/engine/mm_input_mapper.py
+144 -12
@@ -1,11 +1,35 @@
1 from typing import Any, Dict, List, Optional
2
3 +import PIL
4 +from blake3 import blake3
5 +
6 from vllm.config import ModelConfig
7 +from vllm.inputs import PromptType
8 +from vllm.logger import init_logger
9 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
10 MultiModalKwargs, MultiModalRegistry)
11 +from vllm.v1.utils import LRUDictCache
12 +
13 +logger = init_logger(__name__)
14 +
15 +# The idea of MM preprocessor caching is based on having a client and a server,
16 +# where the client executes in the frontend process (=P0) and the server in the
17 +# core process (=P1).
18 +#
19 +# -- Client: Executes the MM mapper and performs caching of the results.
20 +# -- Server: Performs caching of the results
21 +#
22 +# The caching for both client and server is mirrored/similar, and this allows us
23 +# to avoid the serialization of "mm_inputs" (like pixel values) between
24 +# client (=P0) and server (=P1) processes.
25
26 +# Both Client and Server must use the same cache size
27 +# (to perform mirrored caching)
28 +# TODO: Tune the MM cache size
29 +MM_CACHE_SIZE = 256
30
8 -class MMInputMapper:
31 +
32 +class MMInputMapperClient:
33
34 def __init__(
35 self,
@@ -18,23 +42,131 @@ class MMInputMapper:
42 model_config)
43 self.mm_registry.init_mm_limits_per_prompt(model_config)
44
45 + self.mm_cache = LRUDictCache(MM_CACHE_SIZE)
46 +
47 + # DEBUG: Set to None to disable
48 + self.mm_debug_cache_hit_ratio_steps = None
49 + self.mm_cache_hits = 0
50 + self.mm_cache_total = 0
51 +
52 + def cache_hit_ratio(self, steps) -> float:
53 + if self.mm_cache_total > 0 and self.mm_cache_total % steps == 0:
54 + logger.debug("MMInputMapper: cache_hit_ratio = %.2f ",
55 + self.mm_cache_hits / self.mm_cache_total)
56 +
57 def process_inputs(
58 self,
59 mm_data: MultiModalDataDict,
60 + mm_hashes: Optional[List[str]],
61 mm_processor_kwargs: Optional[Dict[str, Any]],
62 + precomputed_mm_inputs: Optional[List[MultiModalKwargs]],
63 + ) -> List[MultiModalKwargs]:
64 + if precomputed_mm_inputs is None:
65 + image_inputs = mm_data["image"]
66 + if not isinstance(image_inputs, list):
67 + image_inputs = [image_inputs]
68 + num_inputs = len(image_inputs)
69 + else:
70 + num_inputs = len(precomputed_mm_inputs)
71 +
72 + # Check if hash is enabled
73 + use_hash = mm_hashes is not None
74 + if use_hash:
75 + assert num_inputs == len(
76 + mm_hashes), "num_inputs = {} len(mm_hashes) = {}".format(
77 + num_inputs, len(mm_hashes))
78 +
79 + # Process each image input separately, so that later we can schedule
80 + # them in a fine-grained manner.
81 + # Apply caching (if enabled) and reuse precomputed inputs (if provided)
82 + ret_hashes = [] if use_hash else None
83 + ret_inputs: List[MultiModalKwargs] = []
84 + for input_id in range(num_inputs):
85 + if self.mm_debug_cache_hit_ratio_steps is not None:
86 + self.cache_hit_ratio(self.mm_debug_cache_hit_ratio_steps)
87 +
88 + mm_hash = None
89 + mm_input = None
90 + if use_hash:
91 + mm_hash = mm_hashes[input_id]
92 + mm_input = self.mm_cache.get(mm_hash)
93 +
94 + self.mm_cache_total += 1
95 + if mm_input is None:
96 + if precomputed_mm_inputs is not None:
97 + # Reuse precomputed input (for merged preprocessor)
98 + mm_input = precomputed_mm_inputs[input_id]
99 + else:
100 + # Apply MM mapper
101 + mm_input = self.multi_modal_input_mapper(
102 + {"image": [image_inputs[input_id]]},
103 + mm_processor_kwargs=mm_processor_kwargs,
104 + )
105 +
106 + if use_hash:
107 + # Add to cache
108 + self.mm_cache.put(mm_hash, mm_input)
109 + else:
110 + self.mm_cache_hits += 1
111 + mm_input = None # Avoids sending mm_input to Server
112 +
113 + if use_hash:
114 + ret_hashes.append(mm_hash)
115 + ret_inputs.append(mm_input)
116 +
117 + return ret_inputs, ret_hashes
118 +
119 +
120 +class MMInputMapperServer:
121 +
122 + def __init__(self, ):
123 + self.mm_cache = LRUDictCache(MM_CACHE_SIZE)
124 +
125 + def process_inputs(
126 + self,
127 + mm_inputs: List[Optional[MultiModalKwargs]],
128 + mm_hashes: List[Optional[str]],
129 ) -> List[MultiModalKwargs]:
130 + assert len(mm_inputs) == len(mm_hashes)
131 +
132 + full_mm_inputs = []
133 + for mm_input, mm_hash in zip(mm_inputs, mm_hashes):
134 + if mm_input is None:
135 + mm_input = self.mm_cache.get(mm_hash)
136 + assert mm_input is not None
137 + else:
138 + self.mm_cache.put(mm_hash, mm_input)
139 +
140 + full_mm_inputs.append(mm_input)
141 +
142 + return full_mm_inputs
143 +
144 +
145 +class MMHasher:
146 +
147 + def __init__(self):
148 + pass
149 +
150 + def hash(self, prompt: PromptType) -> Optional[List[str]]:
151 + if "multi_modal_data" not in prompt:
152 + return None
153 +
154 + mm_data = prompt["multi_modal_data"]
155 image_inputs = mm_data["image"]
156 if not isinstance(image_inputs, list):
157 image_inputs = [image_inputs]
158 + assert len(image_inputs) > 0
159
30 - # Process each image input separately so that later we can schedule
31 - # them in a fine-grained manner.
32 - mm_inputs: List[MultiModalKwargs] = []
33 - num_images = len(image_inputs)
34 - for i in range(num_images):
35 - mm_input = self.multi_modal_input_mapper(
36 - {"image": image_inputs[i]},
37 - mm_processor_kwargs=mm_processor_kwargs,
38 - )
39 - mm_inputs.append(mm_input)
40 - return mm_inputs
160 + ret = []
161 + for image in image_inputs:
162 + assert isinstance(image, PIL.Image.Image)
163 +
164 + # Convert image to bytes
165 + bytes = image.tobytes()
166 +
167 + # Hash image bytes
168 + hasher = blake3()
169 + hasher.update(bytes)
170 + ret.append(hasher.hexdigest())
171 +
172 + return ret
vllm/v1/engine/processor.py
+23 -12
@@ -15,7 +15,7 @@ from vllm.sampling_params import SamplingParams
15 from vllm.transformers_utils.config import try_get_generation_config
16 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
17 from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
18 -from vllm.v1.engine.mm_input_mapper import MMInputMapper
18 +from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
19
20
21 class Processor:
@@ -42,7 +42,11 @@ class Processor:
42 model_config)
43
44 # Multi-modal (huggingface) input mapper
45 - self.mm_input_mapper = MMInputMapper(model_config)
45 + self.mm_input_mapper_client = MMInputMapperClient(model_config)
46 +
47 + # Multi-modal hasher (for images)
48 + self.mm_hasher = MMHasher(
49 + ) if model_config.mm_cache_preprocessor else None
50
51 # TODO: run in an ThreadpoolExecutor or BackgroundProcess.
52 # This ideally should releases the GIL, so we should not block the
@@ -71,6 +75,11 @@ class Processor:
75 assert priority == 0, "vLLM V1 does not support priority at the moment."
76 assert trace_headers is None, "vLLM V1 does not support tracing yet."
77
78 + # Compute MM hashes (if enabled)
79 + mm_hashes = None
80 + if self.mm_hasher is not None:
81 + mm_hashes = self.mm_hasher.hash(prompt)
82 +
83 # Process inputs.
84 preprocessed_inputs = self.input_preprocessor.preprocess(
85 prompt,
@@ -101,16 +110,17 @@ class Processor:
110 sampling_params.update_from_generation_config(
111 self.generation_config_fields, eos_token_id)
112
104 - # Preprocess multi-modal data
105 - if len(decoder_inputs.multi_modal_data) == 0:
106 - mm_inputs = None
107 - elif isinstance(decoder_inputs.multi_modal_data, MultiModalKwargs):
108 - mm_inputs = [decoder_inputs.multi_modal_data]
109 - else:
110 - mm_inputs = self.mm_input_mapper.process_inputs(
111 - decoder_inputs.multi_modal_data,
112 - decoder_inputs.mm_processor_kwargs,
113 - )
113 + # For merged preprocessor, mm_data is already mm_inputs
114 + precomputed_mm_inputs = None
115 + if isinstance(decoder_inputs.multi_modal_data, MultiModalKwargs):
116 + precomputed_mm_inputs = [decoder_inputs.multi_modal_data]
117 +
118 + # Apply MM mapper
119 + mm_inputs = None
120 + if len(decoder_inputs.multi_modal_data) > 0:
121 + mm_inputs, mm_hashes = self.mm_input_mapper_client.process_inputs(
122 + decoder_inputs.multi_modal_data, mm_hashes,
123 + decoder_inputs.mm_processor_kwargs, precomputed_mm_inputs)
124
125 # Make Request for Detokenizer.
126 detokenizer_request = DetokenizerRequest(
@@ -130,6 +140,7 @@ class Processor:
140 decoder_inputs.prompt,
141 decoder_inputs.prompt_token_ids,
142 mm_inputs,
143 + mm_hashes,
144 decoder_inputs.multi_modal_placeholders,
145 sampling_params,
146 eos_token_id,
vllm/v1/utils.py
+21
@@ -1,3 +1,4 @@
1 +from collections import OrderedDict
2 from contextlib import contextmanager
3 from typing import Any, Generic, Iterator, List, TypeVar, overload
4
@@ -93,3 +94,23 @@ def make_zmq_socket(path: str, type: Any) -> Iterator[zmq.Socket]:
94
95 finally:
96 ctx.destroy(linger=0)
97 +
98 +
99 +class LRUDictCache:
100 +
101 + def __init__(self, size: int):
102 + self.cache = OrderedDict()
103 + self.size = size
104 +
105 + def get(self, key, default=None):
106 + if key not in self.cache:
107 + return default
108 +
109 + self.cache.move_to_end(key)
110 + return self.cache[key]
111 +
112 + def put(self, key, value):
113 + self.cache[key] = value
114 + self.cache.move_to_end(key)
115 + if len(self.cache) > self.size:
116 + self.cache.popitem(last=False)