A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[torch.compile] remove compilation_context and simplify code (#10838)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 2, 2024 at 22:19 UTC
dc5ce861bf0e10fc002384859b93b1eebbd70933
14 files changed
+128
-143
tests/compile/piecewise/test_simple.py
+4
-5
@@ -7,7 +7,6 @@ import torch
7
from torch import nn
8
from torch.library import Library
9
10
-from vllm.compilation.compile_context import set_compile_context
10
from vllm.compilation.counter import compilation_counter
11
from vllm.compilation.decorators import support_torch_compile
12
from vllm.config import (CompilationConfig, CompilationLevel, VllmConfig,
@@ -81,6 +80,7 @@ def test_simple_piecewise_compile():
80
use_cudagraph=True,
81
splitting_ops=["silly.attention"],
82
cudagraph_copy_inputs=True,
83
+ cudagraph_capture_sizes=[1, 2],
84
))
85
with set_current_vllm_config(vllm_config):
86
model = SillyModel(vllm_config=vllm_config, prefix='')
@@ -96,11 +96,10 @@ def test_simple_piecewise_compile():
96
6, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
97
):
98
99
- with set_compile_context([1, 2]):
100
- model(inputs)
99
+ model(inputs)
100
102
- model(torch.randn(2).cuda())
103
- model(torch.randn(1).cuda())
101
+ model(torch.randn(2).cuda())
102
+ model(torch.randn(1).cuda())
103
104
input = torch.zeros(2).cuda()
105
global global_counter
tests/compile/piecewise/test_toy_llama.py
+17
-16
@@ -13,7 +13,6 @@ import torch
13
from torch import nn
14
from torch.library import Library
15
16
-from vllm.compilation.compile_context import set_compile_context
16
from vllm.compilation.counter import compilation_counter
17
from vllm.compilation.decorators import support_torch_compile
18
from vllm.config import (CompilationConfig, CompilationLevel, VllmConfig,
@@ -256,6 +255,7 @@ def run_model(llama_config,
255
compilation_config = CompilationConfig(
256
level=CompilationLevel.PIECEWISE,
257
use_cudagraph=True,
258
+ cudagraph_capture_sizes=[1, 2],
259
)
260
if split_attn:
261
compilation_config.splitting_ops = ["silly.attention"]
@@ -273,10 +273,9 @@ def run_model(llama_config,
273
input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
274
positions = torch.arange(B).cuda()
275
276
- with set_compile_context([1, 2]):
277
- model(input_ids, positions)
278
- model(input_ids[:2], positions[:2])
279
- model(input_ids[:1], positions[:1])
276
+ model(input_ids, positions)
277
+ model(input_ids[:2], positions[:2])
278
+ model(input_ids[:1], positions[:1])
279
280
input_ids[:2].zero_()
281
output = model(input_ids[:2], positions[:2])
@@ -379,10 +378,13 @@ def benchmark():
378
level=CompilationLevel.PIECEWISE,
379
use_cudagraph=True,
380
splitting_ops=["silly.attention"],
381
+ cudagraph_capture_sizes=cudagraph_sizes,
382
)
383
else:
384
compilation_config = CompilationConfig(
385
- level=CompilationLevel.PIECEWISE, )
385
+ level=CompilationLevel.PIECEWISE,
386
+ cudagraph_capture_sizes=cudagraph_sizes,
387
+ )
388
389
vllm_config = VllmConfig(compilation_config=compilation_config)
390
with set_current_vllm_config(vllm_config):
@@ -396,17 +398,16 @@ def benchmark():
398
399
graphs = {}
400
399
- with set_compile_context(cudagraph_sizes):
400
- model(input_ids, positions)
401
- for b in cudagraph_sizes[::-1]:
402
- if not piecewise:
403
- graph = torch.cuda.CUDAGraph()
404
- with torch.cuda.graph(graph, pool=pool):
405
- output = model(input_ids[:b], positions[:b])
406
- graphs[b] = (graph, output)
407
- else:
401
+ model(input_ids, positions)
402
+ for b in cudagraph_sizes[::-1]:
403
+ if not piecewise:
404
+ graph = torch.cuda.CUDAGraph()
405
+ with torch.cuda.graph(graph, pool=pool):
406
output = model(input_ids[:b], positions[:b])
409
- graphs[b] = (model, output)
407
+ graphs[b] = (graph, output)
408
+ else:
409
+ output = model(input_ids[:b], positions[:b])
410
+ graphs[b] = (model, output)
411
for b in cudagraph_sizes:
412
if piecewise:
413
# noqa is for `Function definition does not bind loop variable`
tests/models/decoder_only/language/test_jamba.py
+3
-2
@@ -1,8 +1,8 @@
1
import pytest
2
3
from tests.utils import multi_gpu_test
4
+from vllm.config import VllmConfig
5
from vllm.sampling_params import SamplingParams
5
-from vllm.worker.model_runner import _get_graph_batch_size
6
7
from ...utils import check_outputs_equal
8
@@ -189,7 +189,8 @@ def test_mamba_cache_cg_padding(
189
# This test is for verifying that mamba cache is padded to CG captured
190
# batch size. If it's not, a torch RuntimeError will be raised because
191
# tensor dimensions aren't compatible
192
- while len(example_prompts) == _get_graph_batch_size(len(example_prompts)):
192
+ while len(example_prompts) == VllmConfig.get_graph_batch_size(
193
+ len(example_prompts)):
194
example_prompts.append(example_prompts[0])
195
196
try:
tests/models/decoder_only/language/test_mamba.py
+3
-2
@@ -5,8 +5,8 @@ Run `pytest tests/models/test_mamba.py`.
5
import pytest
6
from transformers import AutoModelForCausalLM, AutoTokenizer
7
8
+from vllm.config import VllmConfig
9
from vllm.sampling_params import SamplingParams
9
-from vllm.worker.model_runner import _get_graph_batch_size
10
11
from ...utils import check_outputs_equal
12
@@ -200,7 +200,8 @@ def test_mamba_cache_cg_padding(
200
# This test is for verifying that mamba cache is padded to CG captured
201
# batch size. If it's not, a torch RuntimeError will be raised because
202
# tensor dimensions aren't compatible
203
- while len(example_prompts) == _get_graph_batch_size(len(example_prompts)):
203
+ while len(example_prompts) == VllmConfig.get_graph_batch_size(
204
+ len(example_prompts)):
205
example_prompts.append(example_prompts[0])
206
207
try:
tests/worker/test_encoder_decoder_model_runner.py
+2
-2
@@ -4,12 +4,12 @@ from typing import List
4
import pytest
5
import torch
6
7
+from vllm.config import VllmConfig
8
from vllm.engine.arg_utils import EngineArgs
9
from vllm.platforms import current_platform
10
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11
from vllm.utils import make_tensor_with_pad
12
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
12
-from vllm.worker.model_runner import _get_graph_batch_size
13
14
BATCH_SIZES = [1, 4, 16, 64, 256]
15
@@ -548,7 +548,7 @@ def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
548
# With CUDA Graph capture and replay enabled, the decoder and encoder
549
# input sequences will be padded. Create the expected padded tensors
550
# accordingly.
551
- graph_batch_size = _get_graph_batch_size(expanded_batch_size)
551
+ graph_batch_size = VllmConfig.get_graph_batch_size(expanded_batch_size)
552
cuda_graph_pad_size = graph_batch_size - expanded_batch_size
553
padded_seq_lens = seq_lens + list(itertools.repeat(1, cuda_graph_pad_size))
554
padded_encoder_seq_lens = encoder_seq_lens + list(
tests/worker/test_model_runner.py
+3
-2
@@ -3,13 +3,14 @@ from typing import List
3
import pytest
4
import torch
5
6
+from vllm.config import VllmConfig
7
from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
8
init_distributed_environment)
9
from vllm.engine.arg_utils import EngineArgs
10
from vllm.model_executor.sampling_metadata import SamplingMetadata
11
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12
from vllm.utils import get_open_port
12
-from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
13
+from vllm.worker.model_runner import ModelRunner
14
15
16
def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
@@ -176,7 +177,7 @@ def test_prepare_decode_cuda_graph(batch_size):
177
model_input.attn_metadata, model_input.attn_metadata.slot_mapping)
178
assert len(slot_mapping) == len(input_tokens)
179
179
- expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
180
+ expected_bs = VllmConfig.get_graph_batch_size(len(seq_group_metadata_list))
181
# Verify input metadata is correct for prompts.
182
device = model_runner.device
183
assert attn_metadata.num_prefills == 0
vllm/compilation/backends.py
-4
@@ -242,10 +242,6 @@ class VllmBackend:
242
assert not self._called, "VllmBackend can only be called once"
243
244
self.graph = graph
245
- # config is updated now, because only here can
246
- # we get the sizes to capture for cudagraph
247
- # from compilation context
248
- self.compilation_configs.init_during_runtime()
245
self.configure_post_pass()
246
247
self.split_gm, self.piecewise_graphs = split_graph(
vllm/compilation/compile_context.py
deleted
-23
@@ -1,23 +0,0 @@
1
-from contextlib import contextmanager
2
-from typing import Any
3
-
4
-_compile_context: Any = None
5
-
6
-
7
-def get_compile_context() -> Any:
8
- """Get the current compile context."""
9
- return _compile_context
10
-
11
-
12
-@contextmanager
13
-def set_compile_context(context: Any):
14
- """A context manager that stores the current compile context,
15
- usually it is a list of sizes to specialize.
16
- """
17
- global _compile_context
18
- prev_context = _compile_context
19
- _compile_context = context
20
- try:
21
- yield
22
- finally:
23
- _compile_context = prev_context
vllm/config.py
+75
-8
@@ -2357,15 +2357,10 @@ class CompilationConfig(BaseModel):
2357
from vllm.compilation.backends import VllmBackend
2358
return VllmBackend(self)
2359
2360
- def init_during_runtime(self):
2360
+ def init_with_cudagraph_sizes(self, sizes_to_specialize: List[int]):
2361
"""To complete the initialization of config,
2362
- we need to know the compile context, which is only available
2363
- during the first run of the model.
2364
- """
2365
- from vllm.compilation.compile_context import get_compile_context
2366
- context = get_compile_context()
2367
- context = copy.deepcopy(context) if context is not None else []
2368
- sizes_to_specialize: List[int] = context
2362
+ we need to know the cudagraph sizes."""
2363
+
2364
if self.cudagraph_capture_sizes is None:
2365
self.capture_sizes = sizes_to_specialize
2366
else:
@@ -2386,6 +2381,21 @@ class CompilationConfig(BaseModel):
2381
self.inductor_compile_sizes = []
2382
self.compile_sizes = self.inductor_compile_sizes
2383
2384
+ # sort to make sure cudagraph capture sizes are in descending order
2385
+ self.capture_sizes.sort(reverse=True)
2386
+
2387
+
2388
+_BATCH_SIZE_ALIGNMENT = 8
2389
+# all the token sizes that **can** be captured by cudagraph.
2390
+# they can be arbitrarily large.
2391
+# currently it includes: 1, 2, 4, 8, 16, 24, 32, 40, ..., 8192.
2392
+# the actual sizes to capture will be determined by the model,
2393
+# depending on the model's max_num_seqs.
2394
+# NOTE: get_graph_batch_size needs to be updated if this list is changed.
2395
+_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
2396
+ _BATCH_SIZE_ALIGNMENT * i for i in range(1, 1025)
2397
+]
2398
+
2399
2400
@dataclass
2401
class VllmConfig:
@@ -2413,6 +2423,41 @@ class VllmConfig:
2423
kv_transfer_config: KVTransferConfig = field(default=None,
2424
init=True) # type: ignore
2425
2426
+ @staticmethod
2427
+ def get_graph_batch_size(batch_size: int) -> int:
2428
+ """Returns the padded batch size given actual batch size.
2429
+
2430
+ Batch sizes are 1, 2, 4, _BATCH_SIZE_ALIGNMENT,
2431
+ 2*_BATCH_SIZE_ALIGNMENT, 3*_BATCH_SIZE_ALIGNMENT...
2432
+ """
2433
+ if batch_size <= 2:
2434
+ return batch_size
2435
+ elif batch_size <= 4:
2436
+ return 4
2437
+ else:
2438
+ return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
2439
+ _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
2440
+
2441
+ @staticmethod
2442
+ def get_max_graph_batch_size(max_num_seqs: int) -> int:
2443
+ """
2444
+ max_num_seqs: Maximum number of sequences in a batch.
2445
+ _BATCH_SIZES_TO_CAPTURE: all the sizes that we want to capture.
2446
+
2447
+ pad the max_num_seqs if necessary by calling get_graph_batch_size,
2448
+ which will deal with some edge cases like 1, 2, 4.
2449
+
2450
+ if the padded size is in _BATCH_SIZES_TO_CAPTURE, return the padded
2451
+ size. if not, it means the padded size is larger than the largest size
2452
+ in _BATCH_SIZES_TO_CAPTURE, return the largest size in
2453
+ _BATCH_SIZES_TO_CAPTURE.
2454
+ """
2455
+ padded_size = VllmConfig.get_graph_batch_size(max_num_seqs)
2456
+ if padded_size in _BATCH_SIZES_TO_CAPTURE:
2457
+ return padded_size
2458
+ assert padded_size > _BATCH_SIZES_TO_CAPTURE[-1]
2459
+ return _BATCH_SIZES_TO_CAPTURE[-1]
2460
+
2461
@staticmethod
2462
def _get_quantization_config(
2463
model_config: ModelConfig,
@@ -2496,6 +2541,28 @@ class VllmConfig:
2541
self.compilation_config.pass_config.enable_reshape = False
2542
self.compilation_config.level = CompilationLevel.PIECEWISE
2543
2544
+ if not envs.VLLM_USE_V1:
2545
+ max_batchsize_to_capture = 0
2546
+ if self.scheduler_config is not None and \
2547
+ self.model_config is not None and \
2548
+ not self.model_config.enforce_eager:
2549
+ max_batchsize_to_capture = \
2550
+ self.get_max_graph_batch_size(
2551
+ self.scheduler_config.max_num_seqs)
2552
+ batch_size_capture_list = [
2553
+ size for size in _BATCH_SIZES_TO_CAPTURE
2554
+ if size <= max_batchsize_to_capture
2555
+ ]
2556
+ else:
2557
+ batch_size_capture_list = []
2558
+ if self.model_config is not None and \
2559
+ not self.model_config.enforce_eager:
2560
+ batch_size_capture_list = [1, 2, 4
2561
+ ] + [i for i in range(8, 513, 8)]
2562
+
2563
+ self.compilation_config.init_with_cudagraph_sizes(
2564
+ batch_size_capture_list)
2565
+
2566
if self.cache_config is not None and \
2567
self.cache_config.cpu_offload_gb > 0 and \
2568
self.compilation_config.level != CompilationLevel.NO_COMPILATION:
vllm/model_executor/models/jamba.py
+2
-4
@@ -7,7 +7,7 @@ from transformers import JambaConfig
7
8
from vllm.attention.backends.abstract import AttentionMetadata
9
from vllm.attention.layer import Attention
10
-from vllm.config import CacheConfig, VllmConfig
10
+from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
11
from vllm.distributed import get_tensor_model_parallel_world_size
12
from vllm.model_executor.layers.fused_moe import FusedMoE
13
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -25,8 +25,6 @@ from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
25
MambaCacheParams)
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
27
from vllm.sequence import IntermediateTensors
28
-from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
29
- _get_graph_batch_size)
28
29
from .interfaces import HasInnerState, SupportsLoRA
30
from .utils import maybe_prefix
@@ -404,7 +402,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
402
inputs_embeds: Optional[torch.Tensor] = None,
403
**kwargs):
404
if self.mamba_cache is None:
407
- max_batch_size = (_get_graph_batch_size(
405
+ max_batch_size = (VllmConfig.get_graph_batch_size(
406
self.scheduler_config.max_num_seqs) if self.scheduler_config
407
else max(_BATCH_SIZES_TO_CAPTURE) + 2)
408
vllm/model_executor/models/mamba.py
+2
-4
@@ -6,7 +6,7 @@ from torch import nn
6
from transformers import MambaConfig
7
8
from vllm.attention.backends.abstract import AttentionMetadata
9
-from vllm.config import CacheConfig, VllmConfig
9
+from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
10
from vllm.distributed import get_tensor_model_parallel_world_size
11
from vllm.model_executor.layers.layernorm import RMSNorm
12
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -23,8 +23,6 @@ from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
23
MambaCacheParams)
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
from vllm.sequence import IntermediateTensors
26
-from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
27
- _get_graph_batch_size)
26
27
from .utils import maybe_prefix
28
@@ -187,7 +185,7 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
185
inputs_embeds: Optional[torch.Tensor] = None,
186
**kwargs):
187
if self.mamba_cache is None:
190
- max_batch_size = (_get_graph_batch_size(
188
+ max_batch_size = (VllmConfig.get_graph_batch_size(
189
self.scheduler_config.max_num_seqs) if self.scheduler_config
190
else max(_BATCH_SIZES_TO_CAPTURE) + 2)
191
self.mamba_cache = MambaCacheManager(
vllm/v1/worker/gpu_model_runner.py
+8
-6
@@ -8,7 +8,6 @@ import torch
8
import torch.distributed
9
import torch.nn as nn
10
11
-from vllm.compilation.compile_context import set_compile_context
11
from vllm.config import CompilationLevel, VllmConfig
12
from vllm.distributed.parallel_state import graph_capture
13
from vllm.forward_context import set_forward_context
@@ -100,7 +99,11 @@ class GPUModelRunner:
99
== CompilationLevel.PIECEWISE
100
and not self.model_config.enforce_eager)
101
# TODO(woosuk): Provide an option to tune the max cudagraph batch size.
103
- self.cudagraph_batch_sizes = [1, 2, 4] + [i for i in range(8, 513, 8)]
102
+ # The convention is different.
103
+ # self.cudagraph_batch_sizes sorts in ascending order.
104
+ # The batch sizes in the config are in descending order.
105
+ self.cudagraph_batch_sizes = list(
106
+ reversed(self.vllm_config.compilation_config.capture_sizes))
107
self.positions = torch.zeros(self.max_num_tokens,
108
dtype=torch.int64,
109
device=self.device)
@@ -548,10 +551,9 @@ class GPUModelRunner:
551
torch.tensor([], dtype=torch.float32, device=self.device)
552
for _ in range(self.num_attn_layers)
553
]
551
- with set_compile_context(self.cudagraph_batch_sizes):
552
- # Trigger compilation for general shape.
553
- hidden_states = self._dummy_run(self.model, self.max_num_tokens,
554
- dummy_kv_caches)
554
+ # Trigger compilation for general shape.
555
+ hidden_states = self._dummy_run(self.model, self.max_num_tokens,
556
+ dummy_kv_caches)
557
logits = self.model.compute_logits(hidden_states, None)
558
logits = logits[:self.max_num_tokens]
559
# TODO(woosuk): Consider the memory usage of the sampler.
vllm/worker/enc_dec_model_runner.py
+3
-3
@@ -25,8 +25,7 @@ from vllm.sequence import (IntermediateTensors, PoolerOutput,
25
from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
26
from vllm.worker.model_runner import (GPUModelRunnerBase,
27
ModelInputForGPUBuilder,
28
- ModelInputForGPUWithSamplingMetadata,
29
- _get_graph_batch_size)
28
+ ModelInputForGPUWithSamplingMetadata)
29
from vllm.worker.model_runner_base import (
30
_add_attn_metadata_broadcastable_dict,
31
_add_sampling_metadata_broadcastable_dict)
@@ -465,7 +464,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
464
# We will be using CUDA graph replay for this decode.
465
max_len_of_block_table = self.get_max_block_per_batch()
466
batch_size = len(encoder_seq_lens)
468
- graph_batch_size = _get_graph_batch_size(batch_size)
467
+ graph_batch_size = self.vllm_config.get_graph_batch_size(
468
+ batch_size)
469
assert graph_batch_size >= batch_size
470
cuda_graph_pad_size = graph_batch_size - batch_size
471
# extend the cross_block_tables and encoder_seq_lens to match
vllm/worker/model_runner.py
+6
-62
@@ -18,7 +18,6 @@ import vllm.envs as envs
18
from vllm.attention import AttentionMetadata, get_attn_backend
19
from vllm.attention.backends.abstract import AttentionState
20
from vllm.attention.backends.utils import CommonAttentionState
21
-from vllm.compilation.compile_context import set_compile_context
21
from vllm.config import CompilationLevel, VllmConfig
22
from vllm.core.scheduler import SchedulerOutputs
23
from vllm.distributed import get_kv_transfer_group, get_pp_group
@@ -63,16 +62,7 @@ if TYPE_CHECKING:
62
logger = init_logger(__name__)
63
64
LORA_WARMUP_RANK = 8
66
-_BATCH_SIZE_ALIGNMENT = 8
67
-# all the token sizes that **can** be captured by cudagraph.
68
-# they can be arbitrarily large.
69
-# currently it includes: 1, 2, 4, 8, 16, 24, 32, 40, ..., 8192.
70
-# the actual sizes to capture will be determined by the model,
71
-# depending on the model's max_num_seqs.
72
-# NOTE: _get_graph_batch_size needs to be updated if this list is changed.
73
-_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
74
- _BATCH_SIZE_ALIGNMENT * i for i in range(1, 1025)
75
-]
65
+
66
_NUM_WARMUP_ITERS = 2
67
68
TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU")
@@ -763,7 +753,6 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
753
max_decode_seq_len: int,
754
max_encoder_seq_len: int = 0) -> bool:
755
return (decode_only and not self.runner.model_config.enforce_eager
766
- and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
756
and max_decode_seq_len <= self.runner.max_seq_len_to_capture
757
and max_encoder_seq_len <= self.runner.max_seq_len_to_capture
758
and batch_size <= self.runner.max_batchsize_to_capture)
@@ -811,7 +800,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
800
max_encoder_seq_len):
801
return -1
802
814
- graph_batch_size = _get_graph_batch_size(batch_size)
803
+ graph_batch_size = VllmConfig.get_graph_batch_size(batch_size)
804
assert graph_batch_size >= batch_size
805
return graph_batch_size - batch_size
806
@@ -1023,7 +1012,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1012
self.sliding_window = model_config.get_sliding_window()
1013
self.block_size = cache_config.block_size
1014
self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
1026
- self.max_batchsize_to_capture = _get_max_graph_batch_size(
1015
+ self.max_batchsize_to_capture = VllmConfig.get_max_graph_batch_size(
1016
self.scheduler_config.max_num_seqs)
1017
1018
self.graph_runners: List[Dict[int, CUDAGraphRunner]] = [
@@ -1333,14 +1322,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1322
dtype=self.model_config.dtype,
1323
device=self.device)
1324
1336
- graph_batch_size = self.max_batchsize_to_capture
1337
- batch_size_capture_list = [
1338
- bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1339
- ]
1340
- if self.model_config.enforce_eager:
1341
- batch_size_capture_list = []
1342
- with set_compile_context(batch_size_capture_list):
1343
- self.execute_model(model_input, kv_caches, intermediate_tensors)
1325
+ self.execute_model(model_input, kv_caches, intermediate_tensors)
1326
torch.cuda.synchronize()
1327
return
1328
@@ -1459,18 +1441,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1441
dtype=self.model_config.dtype,
1442
device=self.device)
1443
1462
- graph_batch_size = self.max_batchsize_to_capture
1463
- batch_size_capture_list = [
1464
- bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1465
- ]
1466
-
1444
with self.attn_state.graph_capture(
1445
max_batch_size), graph_capture() as graph_capture_context:
1446
# NOTE: Capturing the largest batch size first may help reduce the
1447
# memory usage of CUDA graph.
1448
for virtual_engine in range(
1449
self.parallel_config.pipeline_parallel_size):
1473
- for batch_size in reversed(batch_size_capture_list):
1450
+ for batch_size in \
1451
+ self.vllm_config.compilation_config.capture_sizes:
1452
attn_metadata = (
1453
self.attn_state.graph_capture_get_metadata_for_batch(
1454
batch_size,
@@ -1993,37 +1971,3 @@ class CUDAGraphRunner(nn.Module):
1971
return self.output_buffers["hidden_states"]
1972
1973
return self.output_buffers
1996
-
1997
-
1998
-def _get_graph_batch_size(batch_size: int) -> int:
1999
- """Returns the padded batch size given actual batch size.
2000
-
2001
- Batch sizes are 1, 2, 4, _BATCH_SIZE_ALIGNMENT,
2002
- 2*_BATCH_SIZE_ALIGNMENT, 3*_BATCH_SIZE_ALIGNMENT...
2003
- """
2004
- if batch_size <= 2:
2005
- return batch_size
2006
- elif batch_size <= 4:
2007
- return 4
2008
- else:
2009
- return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
2010
- _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
2011
-
2012
-
2013
-def _get_max_graph_batch_size(max_num_seqs: int) -> int:
2014
- """
2015
- max_num_seqs: Maximum number of sequences in a batch.
2016
- _BATCH_SIZES_TO_CAPTURE: all the sizes that we want to capture.
2017
-
2018
- pad the max_num_seqs if necessary by calling _get_graph_batch_size,
2019
- which will deal with some edge cases like 1, 2, 4.
2020
-
2021
- if the padded size is in _BATCH_SIZES_TO_CAPTURE, return the padded size.
2022
- if not, it means the padded size is larger than the largest size in
2023
- _BATCH_SIZES_TO_CAPTURE, return the largest size in _BATCH_SIZES_TO_CAPTURE.
2024
- """
2025
- padded_size = _get_graph_batch_size(max_num_seqs)
2026
- if padded_size in _BATCH_SIZES_TO_CAPTURE:
2027
- return padded_size
2028
- assert padded_size > _BATCH_SIZES_TO_CAPTURE[-1]
2029
- return _BATCH_SIZES_TO_CAPTURE[-1]