@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 98cef6a227
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core] Increase default `max_num_batched_tokens` for multimodal models (#8028)

Cyrus Leung committed Aug 30, 2024 at 23:20 UTC 98cef6a2278750ce7578ee6d6ae91e53d01c77a5
4 files changed +33 -12
vllm/config.py
+26 -10
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
32 logger = init_logger(__name__)
33
34 _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
35 +_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 4096
36
37 _PP_SUPPORTED_MODELS = [
38 "AquilaModel",
@@ -571,6 +572,10 @@ class ModelConfig:
572 """Extract the embedding model flag."""
573 return self.embedding_mode
574
575 + @property
576 + def is_multimodal_model(self) -> bool:
577 + return self.multimodal_config is not None
578 +
579
580 class CacheConfig:
581 """Configuration for the KV cache.
@@ -947,25 +952,36 @@ class SchedulerConfig:
952 num_lookahead_slots: int = 0,
953 delay_factor: float = 0.0,
954 enable_chunked_prefill: bool = False,
950 - embedding_mode: Optional[bool] = False,
955 + embedding_mode: bool = False,
956 + is_multimodal_model: bool = False,
957 preemption_mode: Optional[str] = None,
958 num_scheduler_steps: int = 1,
959 send_delta_data: bool = False) -> None:
954 - if max_num_batched_tokens is not None:
955 - self.max_num_batched_tokens = max_num_batched_tokens
956 - else:
960 + if max_num_batched_tokens is None:
961 if enable_chunked_prefill:
962 # It is the values that have the best balance between ITL
963 # and TTFT on A100. Note it is not optimized for throughput.
960 - self.max_num_batched_tokens = 512
961 - elif embedding_mode:
962 - # For embedding, choose specific value for higher throughput
963 - self.max_num_batched_tokens = max(
964 - max_model_len, _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS)
964 + max_num_batched_tokens = 512
965 else:
966 # If max_model_len is too short, use 2048 as the default value
967 # for higher throughput.
968 - self.max_num_batched_tokens = max(max_model_len, 2048)
968 + max_num_batched_tokens = max(max_model_len, 2048)
969 +
970 + if embedding_mode:
971 + # For embedding, choose specific value for higher throughput
972 + max_num_batched_tokens = max(
973 + max_num_batched_tokens,
974 + _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS,
975 + )
976 + if is_multimodal_model:
977 + # The value needs to be at least the number of multimodal tokens
978 + max_num_batched_tokens = max(
979 + max_num_batched_tokens,
980 + _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
981 + )
982 +
983 + self.max_num_batched_tokens = max_num_batched_tokens
984 +
985 if enable_chunked_prefill:
986 logger.info(
987 "Chunked prefill is enabled with max_num_batched_tokens=%d.",
vllm/engine/arg_utils.py
+1
@@ -921,6 +921,7 @@ class EngineArgs:
921 delay_factor=self.scheduler_delay_factor,
922 enable_chunked_prefill=self.enable_chunked_prefill,
923 embedding_mode=model_config.embedding_mode,
924 + is_multimodal_model=model_config.is_multimodal_model,
925 preemption_mode=self.preemption_mode,
926 num_scheduler_steps=self.num_scheduler_steps,
927 send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
vllm/engine/llm_engine.py
+5 -1
@@ -2019,7 +2019,7 @@ class LLMEngine:
2019 if prompt_ids is None or len(prompt_ids) == 0:
2020 raise ValueError("Prompt cannot be empty")
2021
2022 - if self.model_config.multimodal_config is not None:
2022 + if self.model_config.is_multimodal_model:
2023 max_prompt_len = self.model_config.max_model_len
2024
2025 if len(prompt_ids) > max_prompt_len:
@@ -2030,3 +2030,7 @@ class LLMEngine:
2030 "number of text tokens plus multimodal tokens. For image "
2031 "inputs, the number of image tokens depends on the number "
2032 "of images, and possibly their aspect ratios as well.")
2033 +
2034 + # TODO: Find out how many placeholder tokens are there so we can
2035 + # check that chunked prefill does not truncate them
2036 + # max_batch_len = self.scheduler_config.max_num_batched_tokens
vllm/worker/utils.py
+1 -1
@@ -39,7 +39,7 @@ def assert_enc_dec_mr_supported_scenario(
39 raise NotImplementedError(
40 STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PP'])
41
42 - if enc_dec_mr.model_config.multimodal_config is not None:
42 + if enc_dec_mr.model_config.is_multimodal_model:
43 raise NotImplementedError(
44 STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_MM'])
45