@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 59c9b6ebeb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1][VLM] Proper memory profiling for image language models (#11210)

Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: ywang96 <ywang@example.com>

Roger Wang committed Dec 16, 2024 at 22:10 UTC 59c9b6ebeba79b2d744eec86734a7e13b03dcab7
6 files changed +98 -13
vllm/config.py
+8
@@ -1280,6 +1280,14 @@ class SchedulerConfig:
1280
1281 is_multimodal_model: bool = False
1282
1283 + # FIXME(woosuk & ywang96): Below are placeholder values. We need to
1284 + # calculate the actual values from the configurations.
1285 + # Multimodal encoder run compute budget, only used in V1
1286 + max_num_encoder_input_tokens = 16384
1287 +
1288 + # Multimodal encoder cache size, only used in V1
1289 + encoder_cache_size = 16384
1290 +
1291 # Whether to perform preemption by swapping or
1292 # recomputation. If not specified, we determine the mode as follows:
1293 # We use recomputation by default since it incurs lower overhead than
vllm/model_executor/models/pixtral.py
+5
@@ -245,6 +245,11 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
245 # Do not split, return as tensor of shape [1, fs, hs]
246 return image_embeds.unsqueeze(0)
247
248 + # If the last split index is the last index in image_tokens, we
249 + # ignore it to avoid empty split tensor
250 + if split_indices[-1] == len(image_tokens):
251 + split_indices = split_indices[:-1]
252 +
253 image_embeds = image_embeds.tensor_split(split_indices.cpu())
254 return image_embeds
255
vllm/multimodal/registry.py
+20 -3
@@ -200,6 +200,23 @@ class MultiModalRegistry:
200 """
201 return self.register_max_multimodal_tokens("image", max_mm_tokens)
202
203 + def get_max_tokens_per_item_by_modality(
204 + self,
205 + model_config: "ModelConfig",
206 + ) -> Mapping[str, int]:
207 + """
208 + Get the maximum number of tokens per data item from each modality
209 + for profiling the memory usage of a model.
210 +
211 + Note:
212 + This is currently directly used only in V1.
213 + """
214 +
215 + return {
216 + key: plugin.get_max_multimodal_tokens(model_config)
217 + for key, plugin in self._plugins.items()
218 + }
219 +
220 def get_max_tokens_by_modality(
221 self,
222 model_config: "ModelConfig",
@@ -216,9 +233,9 @@ class MultiModalRegistry:
233 limits_per_plugin = self._limits_by_model[model_config]
234
235 return {
219 - key: (limits_per_plugin[key] *
220 - plugin.get_max_multimodal_tokens(model_config))
221 - for key, plugin in self._plugins.items()
236 + key: limits_per_plugin[key] * max_tokens_per_mm_item
237 + for key, max_tokens_per_mm_item in
238 + self.get_max_tokens_per_item_by_modality(model_config).items()
239 }
240
241 def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
vllm/v1/core/scheduler.py
+3 -4
@@ -73,14 +73,13 @@ class Scheduler:
73 # NOTE(woosuk): Here, "encoder" includes the vision encoder (and
74 # projector if needed). Currently, we assume that the encoder also
75 # has the Transformer architecture (e.g., ViT).
76 - # FIXME(woosuk): Below are placeholder values. We need to calculate the
77 - # actual values from the configurations.
78 - self.max_num_encoder_input_tokens = 16384
76 + self.max_num_encoder_input_tokens = self.scheduler_config.max_num_encoder_input_tokens #noqa: E501
77 # NOTE(woosuk): For the models without encoder (e.g., text-only models),
78 # the encoder cache will not be initialized and used, regardless of
79 # the cache size. This is because the memory space for the encoder cache
80 # is preallocated in the profiling run.
83 - self.encoder_cache_manager = EncoderCacheManager(cache_size=16384)
81 + self.encoder_cache_manager = EncoderCacheManager(
82 + cache_size=self.scheduler_config.encoder_cache_size)
83
84 def schedule(self) -> "SchedulerOutput":
85 # NOTE(woosuk) on the scheduling algorithm:
vllm/v1/engine/mm_input_mapper.py
+1
@@ -54,6 +54,7 @@ class MMInputMapperClient:
54 logger.debug("MMInputMapper: cache_hit_ratio = %.2f ",
55 self.mm_cache_hits / self.mm_cache_total)
56
57 + # TODO: Support modalities beyond image.
58 def process_inputs(
59 self,
60 mm_data: MultiModalDataDict,
vllm/v1/worker/gpu_model_runner.py
+61 -6
@@ -10,15 +10,16 @@ import torch.nn as nn
10 from vllm.config import CompilationLevel, VllmConfig
11 from vllm.distributed.parallel_state import graph_capture
12 from vllm.forward_context import set_forward_context
13 -from vllm.inputs import INPUT_REGISTRY, InputRegistry
13 +from vllm.inputs import INPUT_REGISTRY
14 from vllm.logger import init_logger
15 from vllm.model_executor.model_loader import get_model
16 -from vllm.multimodal import MultiModalKwargs
16 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
17 from vllm.sampling_params import SamplingType
18 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19 LayerBlockType, cdiv, is_pin_memory_available)
20 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21 FlashAttentionMetadata)
22 +from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
23 from vllm.v1.outputs import ModelRunnerOutput
24 from vllm.v1.sample.metadata import SamplingMetadata
25 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -35,7 +36,6 @@ class GPUModelRunner:
36 self,
37 vllm_config: VllmConfig,
38 device: torch.device,
38 - input_registry: InputRegistry = INPUT_REGISTRY,
39 ):
40 self.vllm_config = vllm_config
41 self.model_config = vllm_config.model_config
@@ -77,7 +77,12 @@ class GPUModelRunner:
77 self.hidden_size = model_config.get_hidden_size()
78
79 # Multi-modal data support
80 - self.input_registry = input_registry
80 + self.input_registry = INPUT_REGISTRY
81 + self.mm_registry = MULTIMODAL_REGISTRY
82 + # NOTE: mm_input_mapper is only used for memory profiling.
83 + self.mm_input_mapper = MMInputMapperClient(self.model_config)
84 + self.max_num_encoder_input_tokens = self.scheduler_config.max_num_encoder_input_tokens # noqa: E501
85 + self.encoder_cache_size = self.scheduler_config.encoder_cache_size
86
87 # Lazy initialization
88 # self.model: nn.Module # Set after load_model
@@ -599,8 +604,6 @@ class GPUModelRunner:
604 return hidden_states
605
606 def profile_run(self) -> None:
602 - # TODO(woosuk): Profile the max memory usage of the encoder and
603 - # the encoder cache.
607 # use an empty tensor instead of `None`` to force Dynamo to pass
608 # it by reference, rather by specializing on the value `None`.
609 # the `dtype` argument does not matter, and we use `float32` as
@@ -612,6 +615,57 @@ class GPUModelRunner:
615 torch.tensor([], dtype=torch.float32, device=self.device)
616 for _ in range(self.num_attn_layers)
617 ]
618 +
619 + # Profile with multimodal encoder & encoder cache.
620 + # TODO (ywang96): generalize this beyond image modality since
621 + # mm_input_mapper only supports image inputs.
622 + if self.is_multimodal_model:
623 +
624 + # Create dummy batch of multimodal inputs.
625 + dummy_request_data = self.input_registry.dummy_data_for_profiling(
626 + model_config=self.model_config,
627 + seq_len=self.max_num_tokens,
628 + mm_registry=self.mm_registry,
629 + )
630 + dummy_mm_data = dummy_request_data.multi_modal_data
631 + dummy_mm_kwargs, _ = self.mm_input_mapper.process_inputs(
632 + mm_data=dummy_mm_data,
633 + mm_hashes=None,
634 + mm_processor_kwargs=None,
635 + precomputed_mm_inputs=None)
636 +
637 + # NOTE: Currently model is profiled with a single non-text
638 + # modality even when it supports multiple.
639 + max_tokens_per_mm_item = max(
640 + self.mm_registry.get_max_tokens_per_item_by_modality(
641 + self.model_config).values())
642 +
643 + max_num_mm_items = min(
644 + self.max_num_encoder_input_tokens,
645 + self.encoder_cache_size) // max_tokens_per_mm_item
646 +
647 + # Dummy data definition in V0 may contain multiple multimodal items
648 + # (e.g, multiple images) for a single request, therefore here we
649 + # always replicate first item by max_num_mm_items times since in V1
650 + # they are scheduled to be processed separately.
651 + batched_dummy_mm_inputs = MultiModalKwargs.batch(
652 + [dummy_mm_kwargs[0]] * max_num_mm_items)
653 + batched_dummy_mm_inputs = MultiModalKwargs.as_kwargs(
654 + batched_dummy_mm_inputs, device=self.device)
655 +
656 + # Run multimodal encoder.
657 + dummy_encoder_outputs = self.model.get_multimodal_embeddings(
658 + **batched_dummy_mm_inputs)
659 + assert len(dummy_encoder_outputs) == max_num_mm_items, (
660 + "Expected dimension 0 of encoder outputs to match the number "
661 + f"of multimodal data items: {max_num_mm_items}, got "
662 + f"{len(dummy_encoder_outputs)=} instead. This is most likely "
663 + "due to the 'get_multimodal_embeddings' method of the model "
664 + "not implemented correctly.")
665 +
666 + # Cache the dummy encoder outputs.
667 + self.encoder_cache["tmp"] = dict(enumerate(dummy_encoder_outputs))
668 +
669 # Trigger compilation for general shape.
670 hidden_states = self._dummy_run(self.model, self.max_num_tokens,
671 dummy_kv_caches)
@@ -620,6 +674,7 @@ class GPUModelRunner:
674 # TODO(woosuk): Consider the memory usage of the sampler.
675 torch.cuda.synchronize()
676 del hidden_states, logits
677 + self.encoder_cache.clear()
678 gc.collect()
679
680 def capture_model(self) -> None: