@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 02a43f82a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

Update default max_num_batch_tokens for chunked prefill to 2048 (#10544)

Michael Goin committed Nov 23, 2024 at 00:14 UTC 02a43f82a97e37581b48f1c177d3393aca4fe3f2
1 file changed +3 -3
vllm/config.py
+3 -3
@@ -1133,9 +1133,9 @@ class SchedulerConfig:
1133 # max_num_batched_tokens.
1134 self.max_num_batched_tokens = max(self.max_model_len, 2048)
1135 else:
1136 - # It is the values that have the best balance between ITL
1137 - # and TTFT on A100. Note it is not optimized for throughput.
1138 - self.max_num_batched_tokens = 512
1136 + # This value is chosen to have a balance between ITL
1137 + # and TTFT. Note it is not optimized for throughput.
1138 + self.max_num_batched_tokens = 2048
1139 else:
1140 # If max_model_len is too short, use 2048 as the default value
1141 # for higher throughput.