@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f5d72b2fc6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Core] Make BlockSpaceManagerV2 the default BlockManager to use. (#8678)

sroy745 committed Oct 3, 2024 at 09:44 UTC f5d72b2fc6771de19c351945f1fbbb0198d53b8e
2 files changed +8 -5
vllm/config.py
+1 -1
@@ -970,7 +970,7 @@ class SchedulerConfig:
970 max_num_batched_tokens: Optional[int],
971 max_num_seqs: int,
972 max_model_len: int,
973 - use_v2_block_manager: bool = False,
973 + use_v2_block_manager: bool = True,
974 num_lookahead_slots: int = 0,
975 delay_factor: float = 0.0,
976 enable_chunked_prefill: bool = False,
vllm/engine/arg_utils.py
+7 -4
@@ -107,7 +107,7 @@ class EngineArgs:
107 block_size: int = 16
108 enable_prefix_caching: bool = False
109 disable_sliding_window: bool = False
110 - use_v2_block_manager: bool = False
110 + use_v2_block_manager: bool = True
111 swap_space: float = 4 # GiB
112 cpu_offload_gb: float = 0 # GiB
113 gpu_memory_utilization: float = 0.90
@@ -369,9 +369,12 @@ class EngineArgs:
369 action='store_true',
370 help='Disables sliding window, '
371 'capping to sliding window size')
372 - parser.add_argument('--use-v2-block-manager',
373 - action='store_true',
374 - help='Use BlockSpaceMangerV2.')
372 + parser.add_argument(
373 + '--use-v2-block-manager',
374 + default=EngineArgs.use_v2_block_manager,
375 + action='store_true',
376 + help='Use BlockSpaceMangerV2. By default this is set to True. '
377 + 'Set to False to use BlockSpaceManagerV1')
378 parser.add_argument(
379 '--num-lookahead-slots',
380 type=int,