@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3ea2dc2ec4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Misc] Remove deprecated arg for cuda graph capture (#9864)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Oct 31, 2024 at 00:22 UTC 3ea2dc2ec49d1ddd7875045e2397ae76a8f50b38
4 files changed +1 -23
vllm/config.py
-7
@@ -84,9 +84,6 @@ class ModelConfig:
84 disable CUDA graph and always execute the model in eager mode.
85 If False, we will use CUDA graph and eager execution in hybrid.
86 If None, the user did not specify, so default to False.
87 - max_context_len_to_capture: Maximum context len covered by CUDA graphs.
88 - When a sequence has context length larger than this, we fall back
89 - to eager mode (DEPRECATED. Use max_seq_len_to_capture instead).
87 max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
88 When a sequence has context length larger than this, we fall back
89 to eager mode. Additionally for encoder-decoder models, if the
@@ -147,7 +144,6 @@ class ModelConfig:
144 quantization: Optional[str] = None,
145 quantization_param_path: Optional[str] = None,
146 enforce_eager: Optional[bool] = None,
150 - max_context_len_to_capture: Optional[int] = None,
147 max_seq_len_to_capture: Optional[int] = None,
148 max_logprobs: int = 20,
149 disable_sliding_window: bool = False,
@@ -181,9 +177,6 @@ class ModelConfig:
177 self.quantization = quantization
178 self.quantization_param_path = quantization_param_path
179 self.enforce_eager = enforce_eager
184 - if max_context_len_to_capture is not None:
185 - raise ValueError("`max_context_len_to_capture` is deprecated. "
186 - "Use `max_seq_len_to_capture` instead.")
180 self.max_seq_len_to_capture = max_seq_len_to_capture
181 self.max_logprobs = max_logprobs
182 self.disable_sliding_window = disable_sliding_window
vllm/engine/arg_utils.py
-10
@@ -126,7 +126,6 @@ class EngineArgs:
126 tokenizer_revision: Optional[str] = None
127 quantization: Optional[str] = None
128 enforce_eager: Optional[bool] = None
129 - max_context_len_to_capture: Optional[int] = None
129 max_seq_len_to_capture: int = 8192
130 disable_custom_all_reduce: bool = False
131 tokenizer_pool_size: int = 0
@@ -504,14 +503,6 @@ class EngineArgs:
503 help='Always use eager-mode PyTorch. If False, '
504 'will use eager mode and CUDA graph in hybrid '
505 'for maximal performance and flexibility.')
507 - parser.add_argument('--max-context-len-to-capture',
508 - type=int,
509 - default=EngineArgs.max_context_len_to_capture,
510 - help='Maximum context length covered by CUDA '
511 - 'graphs. When a sequence has context length '
512 - 'larger than this, we fall back to eager mode. '
513 - '(DEPRECATED. Use --max-seq-len-to-capture instead'
514 - ')')
506 parser.add_argument('--max-seq-len-to-capture',
507 type=int,
508 default=EngineArgs.max_seq_len_to_capture,
@@ -939,7 +930,6 @@ class EngineArgs:
930 quantization=self.quantization,
931 quantization_param_path=self.quantization_param_path,
932 enforce_eager=self.enforce_eager,
942 - max_context_len_to_capture=self.max_context_len_to_capture,
933 max_seq_len_to_capture=self.max_seq_len_to_capture,
934 max_logprobs=self.max_logprobs,
935 disable_sliding_window=self.disable_sliding_window,
vllm/entrypoints/llm.py
-5
@@ -93,9 +93,6 @@ class LLM:
93 enforce_eager: Whether to enforce eager execution. If True, we will
94 disable CUDA graph and always execute the model in eager mode.
95 If False, we will use CUDA graph and eager execution in hybrid.
96 - max_context_len_to_capture: Maximum context len covered by CUDA graphs.
97 - When a sequence has context length larger than this, we fall back
98 - to eager mode (DEPRECATED. Use `max_seq_len_to_capture` instead).
96 max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
97 When a sequence has context length larger than this, we fall back
98 to eager mode. Additionally for encoder-decoder models, if the
@@ -152,7 +149,6 @@ class LLM:
149 swap_space: float = 4,
150 cpu_offload_gb: float = 0,
151 enforce_eager: Optional[bool] = None,
155 - max_context_len_to_capture: Optional[int] = None,
152 max_seq_len_to_capture: int = 8192,
153 disable_custom_all_reduce: bool = False,
154 disable_async_output_proc: bool = False,
@@ -193,7 +189,6 @@ class LLM:
189 swap_space=swap_space,
190 cpu_offload_gb=cpu_offload_gb,
191 enforce_eager=enforce_eager,
196 - max_context_len_to_capture=max_context_len_to_capture,
192 max_seq_len_to_capture=max_seq_len_to_capture,
193 disable_custom_all_reduce=disable_custom_all_reduce,
194 disable_async_output_proc=disable_async_output_proc,
vllm/worker/model_runner.py
+1 -1
@@ -995,7 +995,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
995 # Python can be expensive. To optimize this, we cache the block table
996 # in numpy and only copy the actual input content at every iteration.
997 # The shape of the cached block table will be
998 - # (max batch size to capture, max context len to capture / block size).
998 + # (max batch size to capture, max seq len to capture / block size).
999 self.graph_block_tables = np.zeros(
1000 (self.max_batchsize_to_capture, self.get_max_block_per_batch()),
1001 dtype=np.int32)