A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Misc] Remove deprecated arg for cuda graph capture (#9864)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Oct 31, 2024 at 00:22 UTC
3ea2dc2ec49d1ddd7875045e2397ae76a8f50b38
4 files changed
+1
-23
vllm/config.py
-7
@@ -84,9 +84,6 @@ class ModelConfig:
84
disable CUDA graph and always execute the model in eager mode.
85
If False, we will use CUDA graph and eager execution in hybrid.
86
If None, the user did not specify, so default to False.
87
- max_context_len_to_capture: Maximum context len covered by CUDA graphs.
88
- When a sequence has context length larger than this, we fall back
89
- to eager mode (DEPRECATED. Use max_seq_len_to_capture instead).
87
max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
88
When a sequence has context length larger than this, we fall back
89
to eager mode. Additionally for encoder-decoder models, if the
@@ -147,7 +144,6 @@ class ModelConfig:
144
quantization: Optional[str] = None,
145
quantization_param_path: Optional[str] = None,
146
enforce_eager: Optional[bool] = None,
150
- max_context_len_to_capture: Optional[int] = None,
147
max_seq_len_to_capture: Optional[int] = None,
148
max_logprobs: int = 20,
149
disable_sliding_window: bool = False,
@@ -181,9 +177,6 @@ class ModelConfig:
177
self.quantization = quantization
178
self.quantization_param_path = quantization_param_path
179
self.enforce_eager = enforce_eager
184
- if max_context_len_to_capture is not None:
185
- raise ValueError("`max_context_len_to_capture` is deprecated. "
186
- "Use `max_seq_len_to_capture` instead.")
180
self.max_seq_len_to_capture = max_seq_len_to_capture
181
self.max_logprobs = max_logprobs
182
self.disable_sliding_window = disable_sliding_window
vllm/engine/arg_utils.py
-10
@@ -126,7 +126,6 @@ class EngineArgs:
126
tokenizer_revision: Optional[str] = None
127
quantization: Optional[str] = None
128
enforce_eager: Optional[bool] = None
129
- max_context_len_to_capture: Optional[int] = None
129
max_seq_len_to_capture: int = 8192
130
disable_custom_all_reduce: bool = False
131
tokenizer_pool_size: int = 0
@@ -504,14 +503,6 @@ class EngineArgs:
503
help='Always use eager-mode PyTorch. If False, '
504
'will use eager mode and CUDA graph in hybrid '
505
'for maximal performance and flexibility.')
507
- parser.add_argument('--max-context-len-to-capture',
508
- type=int,
509
- default=EngineArgs.max_context_len_to_capture,
510
- help='Maximum context length covered by CUDA '
511
- 'graphs. When a sequence has context length '
512
- 'larger than this, we fall back to eager mode. '
513
- '(DEPRECATED. Use --max-seq-len-to-capture instead'
514
- ')')
506
parser.add_argument('--max-seq-len-to-capture',
507
type=int,
508
default=EngineArgs.max_seq_len_to_capture,
@@ -939,7 +930,6 @@ class EngineArgs:
930
quantization=self.quantization,
931
quantization_param_path=self.quantization_param_path,
932
enforce_eager=self.enforce_eager,
942
- max_context_len_to_capture=self.max_context_len_to_capture,
933
max_seq_len_to_capture=self.max_seq_len_to_capture,
934
max_logprobs=self.max_logprobs,
935
disable_sliding_window=self.disable_sliding_window,
vllm/entrypoints/llm.py
-5
@@ -93,9 +93,6 @@ class LLM:
93
enforce_eager: Whether to enforce eager execution. If True, we will
94
disable CUDA graph and always execute the model in eager mode.
95
If False, we will use CUDA graph and eager execution in hybrid.
96
- max_context_len_to_capture: Maximum context len covered by CUDA graphs.
97
- When a sequence has context length larger than this, we fall back
98
- to eager mode (DEPRECATED. Use `max_seq_len_to_capture` instead).
96
max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
97
When a sequence has context length larger than this, we fall back
98
to eager mode. Additionally for encoder-decoder models, if the
@@ -152,7 +149,6 @@ class LLM:
149
swap_space: float = 4,
150
cpu_offload_gb: float = 0,
151
enforce_eager: Optional[bool] = None,
155
- max_context_len_to_capture: Optional[int] = None,
152
max_seq_len_to_capture: int = 8192,
153
disable_custom_all_reduce: bool = False,
154
disable_async_output_proc: bool = False,
@@ -193,7 +189,6 @@ class LLM:
189
swap_space=swap_space,
190
cpu_offload_gb=cpu_offload_gb,
191
enforce_eager=enforce_eager,
196
- max_context_len_to_capture=max_context_len_to_capture,
192
max_seq_len_to_capture=max_seq_len_to_capture,
193
disable_custom_all_reduce=disable_custom_all_reduce,
194
disable_async_output_proc=disable_async_output_proc,
vllm/worker/model_runner.py
+1
-1
@@ -995,7 +995,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
995
# Python can be expensive. To optimize this, we cache the block table
996
# in numpy and only copy the actual input content at every iteration.
997
# The shape of the cached block table will be
998
- # (max batch size to capture, max context len to capture / block size).
998
+ # (max batch size to capture, max seq len to capture / block size).
999
self.graph_block_tables = np.zeros(
1000
(self.max_batchsize_to_capture, self.get_max_block_per_batch()),
1001
dtype=np.int32)