A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Core] Make BlockSpaceManagerV2 the default BlockManager to use. (#8678)
sroy745 committed
Oct 3, 2024 at 09:44 UTC
f5d72b2fc6771de19c351945f1fbbb0198d53b8e
2 files changed
+8
-5
vllm/config.py
+1
-1
@@ -970,7 +970,7 @@ class SchedulerConfig:
970
max_num_batched_tokens: Optional[int],
971
max_num_seqs: int,
972
max_model_len: int,
973
- use_v2_block_manager: bool = False,
973
+ use_v2_block_manager: bool = True,
974
num_lookahead_slots: int = 0,
975
delay_factor: float = 0.0,
976
enable_chunked_prefill: bool = False,
vllm/engine/arg_utils.py
+7
-4
@@ -107,7 +107,7 @@ class EngineArgs:
107
block_size: int = 16
108
enable_prefix_caching: bool = False
109
disable_sliding_window: bool = False
110
- use_v2_block_manager: bool = False
110
+ use_v2_block_manager: bool = True
111
swap_space: float = 4 # GiB
112
cpu_offload_gb: float = 0 # GiB
113
gpu_memory_utilization: float = 0.90
@@ -369,9 +369,12 @@ class EngineArgs:
369
action='store_true',
370
help='Disables sliding window, '
371
'capping to sliding window size')
372
- parser.add_argument('--use-v2-block-manager',
373
- action='store_true',
374
- help='Use BlockSpaceMangerV2.')
372
+ parser.add_argument(
373
+ '--use-v2-block-manager',
374
+ default=EngineArgs.use_v2_block_manager,
375
+ action='store_true',
376
+ help='Use BlockSpaceMangerV2. By default this is set to True. '
377
+ 'Set to False to use BlockSpaceManagerV1')
378
parser.add_argument(
379
'--num-lookahead-slots',
380
type=int,