A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] VLM - enable processor cache by default (#11305)
Signed-off-by: Alexander Matveev <alexm@neuralmagic.com>
Alexander Matveev committed
Dec 18, 2024 at 18:54 UTC
fdea8ec16775e1645620b5ff46b799d60df4624c
7 files changed
+72
-48
examples/offline_inference_vision_language.py
+25
-25
@@ -28,7 +28,7 @@ def run_aria(question: str, modality: str):
28
tokenizer_mode="slow",
29
trust_remote_code=True,
30
dtype="bfloat16",
31
- mm_cache_preprocessor=args.mm_cache_preprocessor)
31
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
32
33
prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
34
"<|im_end|>\n<|im_start|>assistant\n")
@@ -45,7 +45,7 @@ def run_blip2(question: str, modality: str):
45
# See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
46
prompt = f"Question: {question} Answer:"
47
llm = LLM(model="Salesforce/blip2-opt-2.7b",
48
- mm_cache_preprocessor=args.mm_cache_preprocessor)
48
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
49
stop_token_ids = None
50
return llm, prompt, stop_token_ids
51
@@ -57,7 +57,7 @@ def run_chameleon(question: str, modality: str):
57
prompt = f"{question}<image>"
58
llm = LLM(model="facebook/chameleon-7b",
59
max_model_len=4096,
60
- mm_cache_preprocessor=args.mm_cache_preprocessor)
60
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
61
stop_token_ids = None
62
return llm, prompt, stop_token_ids
63
@@ -70,7 +70,7 @@ def run_fuyu(question: str, modality: str):
70
llm = LLM(model="adept/fuyu-8b",
71
max_model_len=2048,
72
max_num_seqs=2,
73
- mm_cache_preprocessor=args.mm_cache_preprocessor)
73
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
74
stop_token_ids = None
75
return llm, prompt, stop_token_ids
76
@@ -85,7 +85,7 @@ def run_glm4v(question: str, modality: str):
85
max_num_seqs=2,
86
trust_remote_code=True,
87
enforce_eager=True,
88
- mm_cache_preprocessor=args.mm_cache_preprocessor)
88
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
89
prompt = question
90
stop_token_ids = [151329, 151336, 151338]
91
return llm, prompt, stop_token_ids
@@ -101,7 +101,7 @@ def run_h2ovl(question: str, modality: str):
101
model=model_name,
102
trust_remote_code=True,
103
max_model_len=8192,
104
- mm_cache_preprocessor=args.mm_cache_preprocessor,
104
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
105
)
106
107
tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -134,7 +134,7 @@ def run_idefics3(question: str, modality: str):
134
"longest_edge": 3 * 364
135
},
136
},
137
- mm_cache_preprocessor=args.mm_cache_preprocessor,
137
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
138
)
139
prompt = (
140
f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
@@ -153,7 +153,7 @@ def run_internvl(question: str, modality: str):
153
model=model_name,
154
trust_remote_code=True,
155
max_model_len=4096,
156
- mm_cache_preprocessor=args.mm_cache_preprocessor,
156
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
157
)
158
159
tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -180,7 +180,7 @@ def run_llava(question: str, modality: str):
180
181
llm = LLM(model="llava-hf/llava-1.5-7b-hf",
182
max_model_len=4096,
183
- mm_cache_preprocessor=args.mm_cache_preprocessor)
183
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
184
stop_token_ids = None
185
return llm, prompt, stop_token_ids
186
@@ -192,7 +192,7 @@ def run_llava_next(question: str, modality: str):
192
prompt = f"[INST] <image>\n{question} [/INST]"
193
llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf",
194
max_model_len=8192,
195
- mm_cache_preprocessor=args.mm_cache_preprocessor)
195
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
196
stop_token_ids = None
197
return llm, prompt, stop_token_ids
198
@@ -205,7 +205,7 @@ def run_llava_next_video(question: str, modality: str):
205
prompt = f"USER: <video>\n{question} ASSISTANT:"
206
llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf",
207
max_model_len=8192,
208
- mm_cache_preprocessor=args.mm_cache_preprocessor)
208
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
209
stop_token_ids = None
210
return llm, prompt, stop_token_ids
211
@@ -223,7 +223,7 @@ def run_llava_onevision(question: str, modality: str):
223
224
llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
225
max_model_len=16384,
226
- mm_cache_preprocessor=args.mm_cache_preprocessor)
226
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
227
stop_token_ids = None
228
return llm, prompt, stop_token_ids
229
@@ -239,7 +239,7 @@ def run_mantis(question: str, modality: str):
239
model="TIGER-Lab/Mantis-8B-siglip-llama3",
240
max_model_len=4096,
241
hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
242
- mm_cache_preprocessor=args.mm_cache_preprocessor,
242
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
243
)
244
stop_token_ids = [128009]
245
return llm, prompt, stop_token_ids
@@ -266,7 +266,7 @@ def run_minicpmv(question: str, modality: str):
266
max_model_len=4096,
267
max_num_seqs=2,
268
trust_remote_code=True,
269
- mm_cache_preprocessor=args.mm_cache_preprocessor,
269
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
270
)
271
# NOTE The stop_token_ids are different for various versions of MiniCPM-V
272
# 2.0
@@ -305,7 +305,7 @@ def run_mllama(question: str, modality: str):
305
max_model_len=4096,
306
max_num_seqs=16,
307
enforce_eager=True,
308
- mm_cache_preprocessor=args.mm_cache_preprocessor,
308
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
309
)
310
311
prompt = f"<|image|><|begin_of_text|>{question}"
@@ -323,7 +323,7 @@ def run_molmo(question, modality):
323
model=model_name,
324
trust_remote_code=True,
325
dtype="bfloat16",
326
- mm_cache_preprocessor=args.mm_cache_preprocessor,
326
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
327
)
328
329
prompt = question
@@ -343,7 +343,7 @@ def run_nvlm_d(question: str, modality: str):
343
trust_remote_code=True,
344
max_model_len=4096,
345
tensor_parallel_size=4,
346
- mm_cache_preprocessor=args.mm_cache_preprocessor,
346
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
347
)
348
349
tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -363,7 +363,7 @@ def run_paligemma(question: str, modality: str):
363
# PaliGemma has special prompt format for VQA
364
prompt = "caption en"
365
llm = LLM(model="google/paligemma-3b-mix-224",
366
- mm_cache_preprocessor=args.mm_cache_preprocessor)
366
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
367
stop_token_ids = None
368
return llm, prompt, stop_token_ids
369
@@ -375,7 +375,7 @@ def run_paligemma2(question: str, modality: str):
375
# PaliGemma 2 has special prompt format for VQA
376
prompt = "caption en"
377
llm = LLM(model="google/paligemma2-3b-ft-docci-448",
378
- mm_cache_preprocessor=args.mm_cache_preprocessor)
378
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
379
stop_token_ids = None
380
return llm, prompt, stop_token_ids
381
@@ -405,7 +405,7 @@ def run_phi3v(question: str, modality: str):
405
max_num_seqs=2,
406
# Note - mm_processor_kwargs can also be passed to generate/chat calls
407
mm_processor_kwargs={"num_crops": 16},
408
- mm_cache_preprocessor=args.mm_cache_preprocessor,
408
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
409
)
410
stop_token_ids = None
411
return llm, prompt, stop_token_ids
@@ -420,7 +420,7 @@ def run_pixtral_hf(question: str, modality: str):
420
llm = LLM(
421
model=model_name,
422
max_model_len=8192,
423
- mm_cache_preprocessor=args.mm_cache_preprocessor,
423
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
424
)
425
426
prompt = f"<s>[INST]{question}\n[IMG][/INST]"
@@ -437,7 +437,7 @@ def run_qwen_vl(question: str, modality: str):
437
trust_remote_code=True,
438
max_model_len=1024,
439
max_num_seqs=2,
440
- mm_cache_preprocessor=args.mm_cache_preprocessor,
440
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
441
)
442
443
prompt = f"{question}Picture 1: <img></img>\n"
@@ -460,7 +460,7 @@ def run_qwen2_vl(question: str, modality: str):
460
"min_pixels": 28 * 28,
461
"max_pixels": 1280 * 28 * 28,
462
},
463
- mm_cache_preprocessor=args.mm_cache_preprocessor,
463
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
464
)
465
466
prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
@@ -651,9 +651,9 @@ if __name__ == "__main__":
651
' (if enabled)')
652
653
parser.add_argument(
654
- '--mm-cache-preprocessor',
654
+ '--disable-mm-preprocessor-cache',
655
action='store_true',
656
- help='If True, enable caching of multi-modal preprocessor/mapper.')
656
+ help='If True, disables caching of multi-modal preprocessor/mapper.')
657
658
parser.add_argument(
659
'--time-generate',
vllm/config.py
+5
-6
@@ -148,9 +148,8 @@ class ModelConfig:
148
HuggingFace config.
149
mm_processor_kwargs: Arguments to be forwarded to the model's processor
150
for multi-modal data, e.g., image processor.
151
- mm_cache_preprocessor: If true, then enables caching of the multi-modal
152
- preprocessor/mapper. Otherwise, the mapper executes each time, and
153
- for better performance consider enabling frontend process.
151
+ disable_mm_preprocessor_cache: If true, then disables caching of the
152
+ multi-modal preprocessor/mapper. (not recommended)
153
override_neuron_config: Initialize non default neuron config or
154
override default neuron config that are specific to Neuron devices,
155
this argument will be used to configure the neuron config that
@@ -216,7 +215,7 @@ class ModelConfig:
215
config_format: ConfigFormat = ConfigFormat.AUTO,
216
hf_overrides: Optional[HfOverrides] = None,
217
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
219
- mm_cache_preprocessor: bool = False,
218
+ disable_mm_preprocessor_cache: bool = False,
219
override_neuron_config: Optional[Dict[str, Any]] = None,
220
override_pooler_config: Optional["PoolerConfig"] = None,
221
logits_processor_pattern: Optional[str] = None) -> None:
@@ -286,7 +285,7 @@ class ModelConfig:
285
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
286
self.use_async_output_proc = use_async_output_proc
287
self.mm_processor_kwargs = mm_processor_kwargs
289
- self.mm_cache_preprocessor = mm_cache_preprocessor
288
+ self.disable_mm_preprocessor_cache = disable_mm_preprocessor_cache
289
290
# Set enforce_eager to False if the value is unset.
291
if self.enforce_eager is None:
@@ -3155,7 +3154,7 @@ class VllmConfig:
3154
f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "
3155
f"chunked_prefill_enabled={self.scheduler_config.chunked_prefill_enabled}, " # noqa
3156
f"use_async_output_proc={self.model_config.use_async_output_proc}, "
3158
- f"mm_cache_preprocessor={self.model_config.mm_cache_preprocessor!r}, " # noqa
3157
+ f"disable_mm_preprocessor_cache={self.model_config.disable_mm_preprocessor_cache!r}, " # noqa
3158
f"mm_processor_kwargs={self.model_config.mm_processor_kwargs}, "
3159
f"pooler_config={self.model_config.pooler_config!r}, "
3160
f"compilation_config={self.compilation_config!r}")
vllm/engine/arg_utils.py
+5
-6
@@ -141,7 +141,7 @@ class EngineArgs:
141
tokenizer_pool_extra_config: Optional[Dict[str, Any]] = None
142
limit_mm_per_prompt: Optional[Mapping[str, int]] = None
143
mm_processor_kwargs: Optional[Dict[str, Any]] = None
144
- mm_cache_preprocessor: bool = False
144
+ disable_mm_preprocessor_cache: bool = False
145
enable_lora: bool = False
146
enable_lora_bias: bool = False
147
max_loras: int = 1
@@ -606,11 +606,10 @@ class EngineArgs:
606
help=('Overrides for the multimodal input mapping/processing, '
607
'e.g., image processor. For example: {"num_crops": 4}.'))
608
parser.add_argument(
609
- '--mm-cache-preprocessor',
609
+ '--disable-mm-preprocessor-cache',
610
action='store_true',
611
- help='If true, then enables caching of the multi-modal '
612
- 'preprocessor/mapper. Otherwise, the mapper executes each time'
613
- ', and for better performance consider enabling frontend process.')
611
+ help='If true, then disables caching of the multi-modal '
612
+ 'preprocessor/mapper. (not recommended)')
613
614
# LoRA related configs
615
parser.add_argument('--enable-lora',
@@ -983,7 +982,7 @@ class EngineArgs:
982
use_async_output_proc=not self.disable_async_output_proc,
983
config_format=self.config_format,
984
mm_processor_kwargs=self.mm_processor_kwargs,
986
- mm_cache_preprocessor=self.mm_cache_preprocessor,
985
+ disable_mm_preprocessor_cache=self.disable_mm_preprocessor_cache,
986
override_neuron_config=self.override_neuron_config,
987
override_pooler_config=self.override_pooler_config,
988
logits_processor_pattern=self.logits_processor_pattern)
vllm/v1/core/kv_cache_utils.py
+1
-1
@@ -191,7 +191,7 @@ def generate_block_hash_extra_keys(
191
raise ValueError(
192
"The number of multi-modal positions and hashes must match. This "
193
"is likely because you do not enable MM preprocessor hashing. "
194
- "Please set mm_cache_preprocessor=True.")
194
+ "Please set disable_mm_preprocessor_cache=False.")
195
196
# Note that we assume mm_positions is sorted by offset.
197
# We do not need to check all mm inputs if the start token index is out of
vllm/v1/engine/mm_input_mapper.py
+17
-3
@@ -43,7 +43,7 @@ class MMInputMapperClient:
43
self.mm_registry.init_mm_limits_per_prompt(model_config)
44
45
# Init cache
46
- self.use_cache = model_config.mm_cache_preprocessor
46
+ self.use_cache = not model_config.disable_mm_preprocessor_cache
47
self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
48
49
# DEBUG: Set to None to disable
@@ -119,7 +119,7 @@ class MMInputMapperClient:
119
class MMInputMapperServer:
120
121
def __init__(self, model_config):
122
- self.use_cache = model_config.mm_cache_preprocessor
122
+ self.use_cache = not model_config.disable_mm_preprocessor_cache
123
self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
124
125
def process_inputs(
@@ -151,12 +151,26 @@ class MMHasher:
151
def __init__(self):
152
pass
153
154
- def hash(self, prompt: PromptType) -> Optional[List[str]]:
154
+ def hash_mm_data(
155
+ self,
156
+ mm_data: Optional[MultiModalDataDict]) -> Optional[List[str]]:
157
+ if mm_data is None:
158
+ return None
159
+
160
+ image_inputs = mm_data['image']
161
+
162
+ return self.hash_images(image_inputs)
163
+
164
+ def hash_prompt(self, prompt: PromptType) -> Optional[List[str]]:
165
if "multi_modal_data" not in prompt:
166
return None
167
168
mm_data = prompt["multi_modal_data"]
169
image_inputs = mm_data["image"]
170
+
171
+ return self.hash_images(image_inputs)
172
+
173
+ def hash_images(self, image_inputs) -> Optional[List[str]]:
174
if not isinstance(image_inputs, list):
175
image_inputs = [image_inputs]
176
assert len(image_inputs) > 0
vllm/v1/engine/processor.py
+2
-2
@@ -46,7 +46,7 @@ class Processor:
46
self.mm_input_mapper_client = MMInputMapperClient(model_config)
47
48
# Multi-modal hasher (for images)
49
- self.use_hash = model_config.mm_cache_preprocessor or \
49
+ self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
50
cache_config.enable_prefix_caching
51
self.mm_hasher = MMHasher()
52
@@ -80,7 +80,7 @@ class Processor:
80
# Compute MM hashes (if enabled)
81
mm_hashes = None
82
if self.use_hash:
83
- mm_hashes = self.mm_hasher.hash(prompt)
83
+ mm_hashes = self.mm_hasher.hash_prompt(prompt)
84
85
# Process inputs.
86
preprocessed_inputs = self.input_preprocessor.preprocess(
vllm/v1/worker/gpu_model_runner.py
+17
-5
@@ -19,7 +19,7 @@ from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19
LayerBlockType, cdiv, is_pin_memory_available)
20
from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21
FlashAttentionMetadata)
22
-from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
22
+from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
23
from vllm.v1.outputs import ModelRunnerOutput
24
from vllm.v1.sample.metadata import SamplingMetadata
25
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -79,8 +79,14 @@ class GPUModelRunner:
79
# Multi-modal data support
80
self.input_registry = INPUT_REGISTRY
81
self.mm_registry = MULTIMODAL_REGISTRY
82
- # NOTE: mm_input_mapper is only used for memory profiling.
83
- self.mm_input_mapper = MMInputMapperClient(self.model_config)
82
+
83
+ # NOTE: mm_input_mapper_client and mm_hasher are only used for memory
84
+ # profiling.
85
+ self.mm_input_mapper_client = MMInputMapperClient(self.model_config)
86
+ self.mm_hasher = MMHasher()
87
+ self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
88
+ cache_config.enable_prefix_caching
89
+
90
self.max_num_encoder_input_tokens = self.scheduler_config.max_num_encoder_input_tokens # noqa: E501
91
self.encoder_cache_size = self.scheduler_config.encoder_cache_size
92
@@ -628,9 +634,15 @@ class GPUModelRunner:
634
mm_registry=self.mm_registry,
635
)
636
dummy_mm_data = dummy_request_data.multi_modal_data
631
- dummy_mm_kwargs, _ = self.mm_input_mapper.process_inputs(
637
+
638
+ # Compute MM hashes (if enabled)
639
+ mm_hashes = None
640
+ if self.use_hash:
641
+ mm_hashes = self.mm_hasher.hash_mm_data(dummy_mm_data)
642
+
643
+ dummy_mm_kwargs = self.mm_input_mapper_client.process_inputs(
644
mm_data=dummy_mm_data,
633
- mm_hashes=None,
645
+ mm_hashes=mm_hashes,
646
mm_processor_kwargs=None,
647
precomputed_mm_inputs=None)
648