A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Misc] Enable chunked prefill by default for long context models (#6666)
Woosuk Kwon committed
Jul 22, 2024 at 20:03 UTC
729171ae58cce74753e628056bda2b6df6b65f4a
1 file changed
+52
-2
vllm/engine/arg_utils.py
+52
-2
@@ -10,6 +10,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
10
PromptAdapterConfig, SchedulerConfig,
11
SpeculativeConfig, TokenizerPoolConfig)
12
from vllm.executor.executor_base import ExecutorBase
13
+from vllm.logger import init_logger
14
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
15
from vllm.utils import FlexibleArgumentParser
16
@@ -17,6 +18,8 @@ if TYPE_CHECKING:
18
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
19
BaseTokenizerGroup)
20
21
+logger = init_logger(__name__)
22
+
23
24
def nullable_str(val: str):
25
if not val or val == "None":
@@ -95,7 +98,7 @@ class EngineArgs:
98
preemption_mode: Optional[str] = None
99
100
scheduler_delay_factor: float = 0.0
98
- enable_chunked_prefill: bool = False
101
+ enable_chunked_prefill: Optional[bool] = None
102
103
guided_decoding_backend: str = 'outlines'
104
# Speculative decoding configuration.
@@ -508,7 +511,10 @@ class EngineArgs:
511
'prompt latency) before scheduling next prompt.')
512
parser.add_argument(
513
'--enable-chunked-prefill',
511
- action='store_true',
514
+ action=StoreBoolean,
515
+ default=EngineArgs.enable_chunked_prefill,
516
+ nargs="?",
517
+ const="True",
518
help='If set, the prefill requests can be chunked based on the '
519
'max_num_batched_tokens.')
520
@@ -728,6 +734,38 @@ class EngineArgs:
734
ray_workers_use_nsight=self.ray_workers_use_nsight,
735
distributed_executor_backend=self.distributed_executor_backend)
736
737
+ max_model_len = model_config.max_model_len
738
+ use_long_context = max_model_len > 32768
739
+ if self.enable_chunked_prefill is None:
740
+ # If not explicitly set, enable chunked prefill by default for
741
+ # long context (> 32K) models. This is to avoid OOM errors in the
742
+ # initial memory profiling phase.
743
+ if use_long_context:
744
+ is_gpu = device_config.device_type == "cuda"
745
+ use_sliding_window = (model_config.get_sliding_window()
746
+ is not None)
747
+ use_spec_decode = self.speculative_model is not None
748
+ if (is_gpu and not use_sliding_window and not use_spec_decode
749
+ and not self.enable_lora
750
+ and not self.enable_prompt_adapter
751
+ and not self.enable_prefix_caching):
752
+ self.enable_chunked_prefill = True
753
+ logger.warning(
754
+ "Chunked prefill is enabled by default for models with "
755
+ "max_model_len > 32K. Currently, chunked prefill might "
756
+ "not work with some features or models. If you "
757
+ "encounter any issues, please disable chunked prefill "
758
+ "by setting --enable-chunked-prefill=False.")
759
+ if self.enable_chunked_prefill is None:
760
+ self.enable_chunked_prefill = False
761
+
762
+ if not self.enable_chunked_prefill and use_long_context:
763
+ logger.warning(
764
+ "The model has a long context length (%s). This may cause OOM "
765
+ "errors during the initial memory profiling phase, or result "
766
+ "in low performance due to small KV cache space. Consider "
767
+ "setting --max-model-len to a smaller value.", max_model_len)
768
+
769
speculative_config = SpeculativeConfig.maybe_create_spec_config(
770
target_model_config=model_config,
771
target_parallel_config=parallel_config,
@@ -843,6 +881,18 @@ class AsyncEngineArgs(EngineArgs):
881
return parser
882
883
884
+class StoreBoolean(argparse.Action):
885
+
886
+ def __call__(self, parser, namespace, values, option_string=None):
887
+ if values.lower() == "true":
888
+ setattr(namespace, self.dest, True)
889
+ elif values.lower() == "false":
890
+ setattr(namespace, self.dest, False)
891
+ else:
892
+ raise ValueError(f"Invalid boolean value: {values}. "
893
+ "Expected 'true' or 'false'.")
894
+
895
+
896
# These functions are used by sphinx to build the documentation
897
def _engine_args_parser():
898
return EngineArgs.add_cli_args(FlexibleArgumentParser())