A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core] Increase default `max_num_batched_tokens` for multimodal models (#8028)
Cyrus Leung committed
Aug 30, 2024 at 23:20 UTC
98cef6a2278750ce7578ee6d6ae91e53d01c77a5
4 files changed
+33
-12
vllm/config.py
+26
-10
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
32
logger = init_logger(__name__)
33
34
_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
35
+_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 4096
36
37
_PP_SUPPORTED_MODELS = [
38
"AquilaModel",
@@ -571,6 +572,10 @@ class ModelConfig:
572
"""Extract the embedding model flag."""
573
return self.embedding_mode
574
575
+ @property
576
+ def is_multimodal_model(self) -> bool:
577
+ return self.multimodal_config is not None
578
+
579
580
class CacheConfig:
581
"""Configuration for the KV cache.
@@ -947,25 +952,36 @@ class SchedulerConfig:
952
num_lookahead_slots: int = 0,
953
delay_factor: float = 0.0,
954
enable_chunked_prefill: bool = False,
950
- embedding_mode: Optional[bool] = False,
955
+ embedding_mode: bool = False,
956
+ is_multimodal_model: bool = False,
957
preemption_mode: Optional[str] = None,
958
num_scheduler_steps: int = 1,
959
send_delta_data: bool = False) -> None:
954
- if max_num_batched_tokens is not None:
955
- self.max_num_batched_tokens = max_num_batched_tokens
956
- else:
960
+ if max_num_batched_tokens is None:
961
if enable_chunked_prefill:
962
# It is the values that have the best balance between ITL
963
# and TTFT on A100. Note it is not optimized for throughput.
960
- self.max_num_batched_tokens = 512
961
- elif embedding_mode:
962
- # For embedding, choose specific value for higher throughput
963
- self.max_num_batched_tokens = max(
964
- max_model_len, _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS)
964
+ max_num_batched_tokens = 512
965
else:
966
# If max_model_len is too short, use 2048 as the default value
967
# for higher throughput.
968
- self.max_num_batched_tokens = max(max_model_len, 2048)
968
+ max_num_batched_tokens = max(max_model_len, 2048)
969
+
970
+ if embedding_mode:
971
+ # For embedding, choose specific value for higher throughput
972
+ max_num_batched_tokens = max(
973
+ max_num_batched_tokens,
974
+ _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS,
975
+ )
976
+ if is_multimodal_model:
977
+ # The value needs to be at least the number of multimodal tokens
978
+ max_num_batched_tokens = max(
979
+ max_num_batched_tokens,
980
+ _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
981
+ )
982
+
983
+ self.max_num_batched_tokens = max_num_batched_tokens
984
+
985
if enable_chunked_prefill:
986
logger.info(
987
"Chunked prefill is enabled with max_num_batched_tokens=%d.",
vllm/engine/arg_utils.py
+1
@@ -921,6 +921,7 @@ class EngineArgs:
921
delay_factor=self.scheduler_delay_factor,
922
enable_chunked_prefill=self.enable_chunked_prefill,
923
embedding_mode=model_config.embedding_mode,
924
+ is_multimodal_model=model_config.is_multimodal_model,
925
preemption_mode=self.preemption_mode,
926
num_scheduler_steps=self.num_scheduler_steps,
927
send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
vllm/engine/llm_engine.py
+5
-1
@@ -2019,7 +2019,7 @@ class LLMEngine:
2019
if prompt_ids is None or len(prompt_ids) == 0:
2020
raise ValueError("Prompt cannot be empty")
2021
2022
- if self.model_config.multimodal_config is not None:
2022
+ if self.model_config.is_multimodal_model:
2023
max_prompt_len = self.model_config.max_model_len
2024
2025
if len(prompt_ids) > max_prompt_len:
@@ -2030,3 +2030,7 @@ class LLMEngine:
2030
"number of text tokens plus multimodal tokens. For image "
2031
"inputs, the number of image tokens depends on the number "
2032
"of images, and possibly their aspect ratios as well.")
2033
+
2034
+ # TODO: Find out how many placeholder tokens are there so we can
2035
+ # check that chunked prefill does not truncate them
2036
+ # max_batch_len = self.scheduler_config.max_num_batched_tokens
vllm/worker/utils.py
+1
-1
@@ -39,7 +39,7 @@ def assert_enc_dec_mr_supported_scenario(
39
raise NotImplementedError(
40
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PP'])
41
42
- if enc_dec_mr.model_config.multimodal_config is not None:
42
+ if enc_dec_mr.model_config.is_multimodal_model:
43
raise NotImplementedError(
44
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_MM'])
45