@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 134810b3d9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1][Bugfix] Always set enable_chunked_prefill = True for V1 (#11061)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 10, 2024 at 14:41 UTC 134810b3d9a05510622282479f0f9e2114b88017
1 file changed +12 -9
vllm/engine/arg_utils.py
+12 -9
@@ -122,7 +122,7 @@ class EngineArgs:
122 cpu_offload_gb: float = 0 # GiB
123 gpu_memory_utilization: float = 0.90
124 max_num_batched_tokens: Optional[int] = None
125 - max_num_seqs: int = 256
125 + max_num_seqs: Optional[int] = None
126 max_logprobs: int = 20 # Default value for OpenAI Chat Completions API
127 disable_log_stats: bool = False
128 revision: Optional[str] = None
@@ -205,6 +205,9 @@ class EngineArgs:
205 # by user.
206 if self.enable_prefix_caching is None:
207 self.enable_prefix_caching = bool(envs.VLLM_USE_V1)
208 + # Override max_num_seqs if it's not set by user.
209 + if self.max_num_seqs is None:
210 + self.max_num_seqs = 256 if not envs.VLLM_USE_V1 else 1024
211
212 # support `EngineArgs(compilation_config={...})`
213 # without having to manually construct a
@@ -1225,19 +1228,19 @@ class EngineArgs:
1228 """
1229 assert envs.VLLM_USE_V1, "V1 is not enabled"
1230
1231 + # V1 always uses chunked prefills.
1232 + self.enable_chunked_prefill = True
1233 + # When no user override, set the default values based on the usage
1234 + # context.
1235 + # TODO(woosuk): Tune the default values for different hardware.
1236 if self.max_num_batched_tokens is None:
1229 - # When no user override, set the default values based on the
1230 - # usage context.
1237 if usage_context == UsageContext.LLM_CLASS:
1232 - logger.warning("Setting max_num_batched_tokens to 8192 "
1233 - "for LLM_CLASS usage context.")
1234 - self.max_num_seqs = 1024
1238 self.max_num_batched_tokens = 8192
1239 elif usage_context == UsageContext.OPENAI_API_SERVER:
1237 - logger.warning("Setting max_num_batched_tokens to 2048 "
1238 - "for OPENAI_API_SERVER usage context.")
1239 - self.max_num_seqs = 1024
1240 self.max_num_batched_tokens = 2048
1241 + logger.warning(
1242 + "Setting max_num_batched_tokens to %d for %s usage context.",
1243 + self.max_num_batched_tokens, usage_context.value)
1244
1245 def _override_v1_engine_config(self, engine_config: VllmConfig) -> None:
1246 """