@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fdea8ec167
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] VLM - enable processor cache by default (#11305)

Signed-off-by: Alexander Matveev <alexm@neuralmagic.com>

Alexander Matveev committed Dec 18, 2024 at 18:54 UTC fdea8ec16775e1645620b5ff46b799d60df4624c
7 files changed +72 -48
examples/offline_inference_vision_language.py
+25 -25
@@ -28,7 +28,7 @@ def run_aria(question: str, modality: str):
28 tokenizer_mode="slow",
29 trust_remote_code=True,
30 dtype="bfloat16",
31 - mm_cache_preprocessor=args.mm_cache_preprocessor)
31 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
32
33 prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
34 "<|im_end|>\n<|im_start|>assistant\n")
@@ -45,7 +45,7 @@ def run_blip2(question: str, modality: str):
45 # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
46 prompt = f"Question: {question} Answer:"
47 llm = LLM(model="Salesforce/blip2-opt-2.7b",
48 - mm_cache_preprocessor=args.mm_cache_preprocessor)
48 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
49 stop_token_ids = None
50 return llm, prompt, stop_token_ids
51
@@ -57,7 +57,7 @@ def run_chameleon(question: str, modality: str):
57 prompt = f"{question}<image>"
58 llm = LLM(model="facebook/chameleon-7b",
59 max_model_len=4096,
60 - mm_cache_preprocessor=args.mm_cache_preprocessor)
60 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
61 stop_token_ids = None
62 return llm, prompt, stop_token_ids
63
@@ -70,7 +70,7 @@ def run_fuyu(question: str, modality: str):
70 llm = LLM(model="adept/fuyu-8b",
71 max_model_len=2048,
72 max_num_seqs=2,
73 - mm_cache_preprocessor=args.mm_cache_preprocessor)
73 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
74 stop_token_ids = None
75 return llm, prompt, stop_token_ids
76
@@ -85,7 +85,7 @@ def run_glm4v(question: str, modality: str):
85 max_num_seqs=2,
86 trust_remote_code=True,
87 enforce_eager=True,
88 - mm_cache_preprocessor=args.mm_cache_preprocessor)
88 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
89 prompt = question
90 stop_token_ids = [151329, 151336, 151338]
91 return llm, prompt, stop_token_ids
@@ -101,7 +101,7 @@ def run_h2ovl(question: str, modality: str):
101 model=model_name,
102 trust_remote_code=True,
103 max_model_len=8192,
104 - mm_cache_preprocessor=args.mm_cache_preprocessor,
104 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
105 )
106
107 tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -134,7 +134,7 @@ def run_idefics3(question: str, modality: str):
134 "longest_edge": 3 * 364
135 },
136 },
137 - mm_cache_preprocessor=args.mm_cache_preprocessor,
137 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
138 )
139 prompt = (
140 f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
@@ -153,7 +153,7 @@ def run_internvl(question: str, modality: str):
153 model=model_name,
154 trust_remote_code=True,
155 max_model_len=4096,
156 - mm_cache_preprocessor=args.mm_cache_preprocessor,
156 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
157 )
158
159 tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -180,7 +180,7 @@ def run_llava(question: str, modality: str):
180
181 llm = LLM(model="llava-hf/llava-1.5-7b-hf",
182 max_model_len=4096,
183 - mm_cache_preprocessor=args.mm_cache_preprocessor)
183 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
184 stop_token_ids = None
185 return llm, prompt, stop_token_ids
186
@@ -192,7 +192,7 @@ def run_llava_next(question: str, modality: str):
192 prompt = f"[INST] <image>\n{question} [/INST]"
193 llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf",
194 max_model_len=8192,
195 - mm_cache_preprocessor=args.mm_cache_preprocessor)
195 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
196 stop_token_ids = None
197 return llm, prompt, stop_token_ids
198
@@ -205,7 +205,7 @@ def run_llava_next_video(question: str, modality: str):
205 prompt = f"USER: <video>\n{question} ASSISTANT:"
206 llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf",
207 max_model_len=8192,
208 - mm_cache_preprocessor=args.mm_cache_preprocessor)
208 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
209 stop_token_ids = None
210 return llm, prompt, stop_token_ids
211
@@ -223,7 +223,7 @@ def run_llava_onevision(question: str, modality: str):
223
224 llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
225 max_model_len=16384,
226 - mm_cache_preprocessor=args.mm_cache_preprocessor)
226 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
227 stop_token_ids = None
228 return llm, prompt, stop_token_ids
229
@@ -239,7 +239,7 @@ def run_mantis(question: str, modality: str):
239 model="TIGER-Lab/Mantis-8B-siglip-llama3",
240 max_model_len=4096,
241 hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
242 - mm_cache_preprocessor=args.mm_cache_preprocessor,
242 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
243 )
244 stop_token_ids = [128009]
245 return llm, prompt, stop_token_ids
@@ -266,7 +266,7 @@ def run_minicpmv(question: str, modality: str):
266 max_model_len=4096,
267 max_num_seqs=2,
268 trust_remote_code=True,
269 - mm_cache_preprocessor=args.mm_cache_preprocessor,
269 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
270 )
271 # NOTE The stop_token_ids are different for various versions of MiniCPM-V
272 # 2.0
@@ -305,7 +305,7 @@ def run_mllama(question: str, modality: str):
305 max_model_len=4096,
306 max_num_seqs=16,
307 enforce_eager=True,
308 - mm_cache_preprocessor=args.mm_cache_preprocessor,
308 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
309 )
310
311 prompt = f"<|image|><|begin_of_text|>{question}"
@@ -323,7 +323,7 @@ def run_molmo(question, modality):
323 model=model_name,
324 trust_remote_code=True,
325 dtype="bfloat16",
326 - mm_cache_preprocessor=args.mm_cache_preprocessor,
326 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
327 )
328
329 prompt = question
@@ -343,7 +343,7 @@ def run_nvlm_d(question: str, modality: str):
343 trust_remote_code=True,
344 max_model_len=4096,
345 tensor_parallel_size=4,
346 - mm_cache_preprocessor=args.mm_cache_preprocessor,
346 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
347 )
348
349 tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -363,7 +363,7 @@ def run_paligemma(question: str, modality: str):
363 # PaliGemma has special prompt format for VQA
364 prompt = "caption en"
365 llm = LLM(model="google/paligemma-3b-mix-224",
366 - mm_cache_preprocessor=args.mm_cache_preprocessor)
366 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
367 stop_token_ids = None
368 return llm, prompt, stop_token_ids
369
@@ -375,7 +375,7 @@ def run_paligemma2(question: str, modality: str):
375 # PaliGemma 2 has special prompt format for VQA
376 prompt = "caption en"
377 llm = LLM(model="google/paligemma2-3b-ft-docci-448",
378 - mm_cache_preprocessor=args.mm_cache_preprocessor)
378 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
379 stop_token_ids = None
380 return llm, prompt, stop_token_ids
381
@@ -405,7 +405,7 @@ def run_phi3v(question: str, modality: str):
405 max_num_seqs=2,
406 # Note - mm_processor_kwargs can also be passed to generate/chat calls
407 mm_processor_kwargs={"num_crops": 16},
408 - mm_cache_preprocessor=args.mm_cache_preprocessor,
408 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
409 )
410 stop_token_ids = None
411 return llm, prompt, stop_token_ids
@@ -420,7 +420,7 @@ def run_pixtral_hf(question: str, modality: str):
420 llm = LLM(
421 model=model_name,
422 max_model_len=8192,
423 - mm_cache_preprocessor=args.mm_cache_preprocessor,
423 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
424 )
425
426 prompt = f"<s>[INST]{question}\n[IMG][/INST]"
@@ -437,7 +437,7 @@ def run_qwen_vl(question: str, modality: str):
437 trust_remote_code=True,
438 max_model_len=1024,
439 max_num_seqs=2,
440 - mm_cache_preprocessor=args.mm_cache_preprocessor,
440 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
441 )
442
443 prompt = f"{question}Picture 1: <img></img>\n"
@@ -460,7 +460,7 @@ def run_qwen2_vl(question: str, modality: str):
460 "min_pixels": 28 * 28,
461 "max_pixels": 1280 * 28 * 28,
462 },
463 - mm_cache_preprocessor=args.mm_cache_preprocessor,
463 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
464 )
465
466 prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
@@ -651,9 +651,9 @@ if __name__ == "__main__":
651 ' (if enabled)')
652
653 parser.add_argument(
654 - '--mm-cache-preprocessor',
654 + '--disable-mm-preprocessor-cache',
655 action='store_true',
656 - help='If True, enable caching of multi-modal preprocessor/mapper.')
656 + help='If True, disables caching of multi-modal preprocessor/mapper.')
657
658 parser.add_argument(
659 '--time-generate',
vllm/config.py
+5 -6
@@ -148,9 +148,8 @@ class ModelConfig:
148 HuggingFace config.
149 mm_processor_kwargs: Arguments to be forwarded to the model's processor
150 for multi-modal data, e.g., image processor.
151 - mm_cache_preprocessor: If true, then enables caching of the multi-modal
152 - preprocessor/mapper. Otherwise, the mapper executes each time, and
153 - for better performance consider enabling frontend process.
151 + disable_mm_preprocessor_cache: If true, then disables caching of the
152 + multi-modal preprocessor/mapper. (not recommended)
153 override_neuron_config: Initialize non default neuron config or
154 override default neuron config that are specific to Neuron devices,
155 this argument will be used to configure the neuron config that
@@ -216,7 +215,7 @@ class ModelConfig:
215 config_format: ConfigFormat = ConfigFormat.AUTO,
216 hf_overrides: Optional[HfOverrides] = None,
217 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
219 - mm_cache_preprocessor: bool = False,
218 + disable_mm_preprocessor_cache: bool = False,
219 override_neuron_config: Optional[Dict[str, Any]] = None,
220 override_pooler_config: Optional["PoolerConfig"] = None,
221 logits_processor_pattern: Optional[str] = None) -> None:
@@ -286,7 +285,7 @@ class ModelConfig:
285 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
286 self.use_async_output_proc = use_async_output_proc
287 self.mm_processor_kwargs = mm_processor_kwargs
289 - self.mm_cache_preprocessor = mm_cache_preprocessor
288 + self.disable_mm_preprocessor_cache = disable_mm_preprocessor_cache
289
290 # Set enforce_eager to False if the value is unset.
291 if self.enforce_eager is None:
@@ -3155,7 +3154,7 @@ class VllmConfig:
3154 f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "
3155 f"chunked_prefill_enabled={self.scheduler_config.chunked_prefill_enabled}, " # noqa
3156 f"use_async_output_proc={self.model_config.use_async_output_proc}, "
3158 - f"mm_cache_preprocessor={self.model_config.mm_cache_preprocessor!r}, " # noqa
3157 + f"disable_mm_preprocessor_cache={self.model_config.disable_mm_preprocessor_cache!r}, " # noqa
3158 f"mm_processor_kwargs={self.model_config.mm_processor_kwargs}, "
3159 f"pooler_config={self.model_config.pooler_config!r}, "
3160 f"compilation_config={self.compilation_config!r}")
vllm/engine/arg_utils.py
+5 -6
@@ -141,7 +141,7 @@ class EngineArgs:
141 tokenizer_pool_extra_config: Optional[Dict[str, Any]] = None
142 limit_mm_per_prompt: Optional[Mapping[str, int]] = None
143 mm_processor_kwargs: Optional[Dict[str, Any]] = None
144 - mm_cache_preprocessor: bool = False
144 + disable_mm_preprocessor_cache: bool = False
145 enable_lora: bool = False
146 enable_lora_bias: bool = False
147 max_loras: int = 1
@@ -606,11 +606,10 @@ class EngineArgs:
606 help=('Overrides for the multimodal input mapping/processing, '
607 'e.g., image processor. For example: {"num_crops": 4}.'))
608 parser.add_argument(
609 - '--mm-cache-preprocessor',
609 + '--disable-mm-preprocessor-cache',
610 action='store_true',
611 - help='If true, then enables caching of the multi-modal '
612 - 'preprocessor/mapper. Otherwise, the mapper executes each time'
613 - ', and for better performance consider enabling frontend process.')
611 + help='If true, then disables caching of the multi-modal '
612 + 'preprocessor/mapper. (not recommended)')
613
614 # LoRA related configs
615 parser.add_argument('--enable-lora',
@@ -983,7 +982,7 @@ class EngineArgs:
982 use_async_output_proc=not self.disable_async_output_proc,
983 config_format=self.config_format,
984 mm_processor_kwargs=self.mm_processor_kwargs,
986 - mm_cache_preprocessor=self.mm_cache_preprocessor,
985 + disable_mm_preprocessor_cache=self.disable_mm_preprocessor_cache,
986 override_neuron_config=self.override_neuron_config,
987 override_pooler_config=self.override_pooler_config,
988 logits_processor_pattern=self.logits_processor_pattern)
vllm/v1/core/kv_cache_utils.py
+1 -1
@@ -191,7 +191,7 @@ def generate_block_hash_extra_keys(
191 raise ValueError(
192 "The number of multi-modal positions and hashes must match. This "
193 "is likely because you do not enable MM preprocessor hashing. "
194 - "Please set mm_cache_preprocessor=True.")
194 + "Please set disable_mm_preprocessor_cache=False.")
195
196 # Note that we assume mm_positions is sorted by offset.
197 # We do not need to check all mm inputs if the start token index is out of
vllm/v1/engine/mm_input_mapper.py
+17 -3
@@ -43,7 +43,7 @@ class MMInputMapperClient:
43 self.mm_registry.init_mm_limits_per_prompt(model_config)
44
45 # Init cache
46 - self.use_cache = model_config.mm_cache_preprocessor
46 + self.use_cache = not model_config.disable_mm_preprocessor_cache
47 self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
48
49 # DEBUG: Set to None to disable
@@ -119,7 +119,7 @@ class MMInputMapperClient:
119 class MMInputMapperServer:
120
121 def __init__(self, model_config):
122 - self.use_cache = model_config.mm_cache_preprocessor
122 + self.use_cache = not model_config.disable_mm_preprocessor_cache
123 self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
124
125 def process_inputs(
@@ -151,12 +151,26 @@ class MMHasher:
151 def __init__(self):
152 pass
153
154 - def hash(self, prompt: PromptType) -> Optional[List[str]]:
154 + def hash_mm_data(
155 + self,
156 + mm_data: Optional[MultiModalDataDict]) -> Optional[List[str]]:
157 + if mm_data is None:
158 + return None
159 +
160 + image_inputs = mm_data['image']
161 +
162 + return self.hash_images(image_inputs)
163 +
164 + def hash_prompt(self, prompt: PromptType) -> Optional[List[str]]:
165 if "multi_modal_data" not in prompt:
166 return None
167
168 mm_data = prompt["multi_modal_data"]
169 image_inputs = mm_data["image"]
170 +
171 + return self.hash_images(image_inputs)
172 +
173 + def hash_images(self, image_inputs) -> Optional[List[str]]:
174 if not isinstance(image_inputs, list):
175 image_inputs = [image_inputs]
176 assert len(image_inputs) > 0
vllm/v1/engine/processor.py
+2 -2
@@ -46,7 +46,7 @@ class Processor:
46 self.mm_input_mapper_client = MMInputMapperClient(model_config)
47
48 # Multi-modal hasher (for images)
49 - self.use_hash = model_config.mm_cache_preprocessor or \
49 + self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
50 cache_config.enable_prefix_caching
51 self.mm_hasher = MMHasher()
52
@@ -80,7 +80,7 @@ class Processor:
80 # Compute MM hashes (if enabled)
81 mm_hashes = None
82 if self.use_hash:
83 - mm_hashes = self.mm_hasher.hash(prompt)
83 + mm_hashes = self.mm_hasher.hash_prompt(prompt)
84
85 # Process inputs.
86 preprocessed_inputs = self.input_preprocessor.preprocess(
vllm/v1/worker/gpu_model_runner.py
+17 -5
@@ -19,7 +19,7 @@ from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19 LayerBlockType, cdiv, is_pin_memory_available)
20 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21 FlashAttentionMetadata)
22 -from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
22 +from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
23 from vllm.v1.outputs import ModelRunnerOutput
24 from vllm.v1.sample.metadata import SamplingMetadata
25 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -79,8 +79,14 @@ class GPUModelRunner:
79 # Multi-modal data support
80 self.input_registry = INPUT_REGISTRY
81 self.mm_registry = MULTIMODAL_REGISTRY
82 - # NOTE: mm_input_mapper is only used for memory profiling.
83 - self.mm_input_mapper = MMInputMapperClient(self.model_config)
82 +
83 + # NOTE: mm_input_mapper_client and mm_hasher are only used for memory
84 + # profiling.
85 + self.mm_input_mapper_client = MMInputMapperClient(self.model_config)
86 + self.mm_hasher = MMHasher()
87 + self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
88 + cache_config.enable_prefix_caching
89 +
90 self.max_num_encoder_input_tokens = self.scheduler_config.max_num_encoder_input_tokens # noqa: E501
91 self.encoder_cache_size = self.scheduler_config.encoder_cache_size
92
@@ -628,9 +634,15 @@ class GPUModelRunner:
634 mm_registry=self.mm_registry,
635 )
636 dummy_mm_data = dummy_request_data.multi_modal_data
631 - dummy_mm_kwargs, _ = self.mm_input_mapper.process_inputs(
637 +
638 + # Compute MM hashes (if enabled)
639 + mm_hashes = None
640 + if self.use_hash:
641 + mm_hashes = self.mm_hasher.hash_mm_data(dummy_mm_data)
642 +
643 + dummy_mm_kwargs = self.mm_input_mapper_client.process_inputs(
644 mm_data=dummy_mm_data,
633 - mm_hashes=None,
645 + mm_hashes=mm_hashes,
646 mm_processor_kwargs=None,
647 precomputed_mm_inputs=None)
648