@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dc5ce861bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[torch.compile] remove compilation_context and simplify code (#10838)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 2, 2024 at 22:19 UTC dc5ce861bf0e10fc002384859b93b1eebbd70933
14 files changed +128 -143
tests/compile/piecewise/test_simple.py
+4 -5
@@ -7,7 +7,6 @@ import torch
7 from torch import nn
8 from torch.library import Library
9
10 -from vllm.compilation.compile_context import set_compile_context
10 from vllm.compilation.counter import compilation_counter
11 from vllm.compilation.decorators import support_torch_compile
12 from vllm.config import (CompilationConfig, CompilationLevel, VllmConfig,
@@ -81,6 +80,7 @@ def test_simple_piecewise_compile():
80 use_cudagraph=True,
81 splitting_ops=["silly.attention"],
82 cudagraph_copy_inputs=True,
83 + cudagraph_capture_sizes=[1, 2],
84 ))
85 with set_current_vllm_config(vllm_config):
86 model = SillyModel(vllm_config=vllm_config, prefix='')
@@ -96,11 +96,10 @@ def test_simple_piecewise_compile():
96 6, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
97 ):
98
99 - with set_compile_context([1, 2]):
100 - model(inputs)
99 + model(inputs)
100
102 - model(torch.randn(2).cuda())
103 - model(torch.randn(1).cuda())
101 + model(torch.randn(2).cuda())
102 + model(torch.randn(1).cuda())
103
104 input = torch.zeros(2).cuda()
105 global global_counter
tests/compile/piecewise/test_toy_llama.py
+17 -16
@@ -13,7 +13,6 @@ import torch
13 from torch import nn
14 from torch.library import Library
15
16 -from vllm.compilation.compile_context import set_compile_context
16 from vllm.compilation.counter import compilation_counter
17 from vllm.compilation.decorators import support_torch_compile
18 from vllm.config import (CompilationConfig, CompilationLevel, VllmConfig,
@@ -256,6 +255,7 @@ def run_model(llama_config,
255 compilation_config = CompilationConfig(
256 level=CompilationLevel.PIECEWISE,
257 use_cudagraph=True,
258 + cudagraph_capture_sizes=[1, 2],
259 )
260 if split_attn:
261 compilation_config.splitting_ops = ["silly.attention"]
@@ -273,10 +273,9 @@ def run_model(llama_config,
273 input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
274 positions = torch.arange(B).cuda()
275
276 - with set_compile_context([1, 2]):
277 - model(input_ids, positions)
278 - model(input_ids[:2], positions[:2])
279 - model(input_ids[:1], positions[:1])
276 + model(input_ids, positions)
277 + model(input_ids[:2], positions[:2])
278 + model(input_ids[:1], positions[:1])
279
280 input_ids[:2].zero_()
281 output = model(input_ids[:2], positions[:2])
@@ -379,10 +378,13 @@ def benchmark():
378 level=CompilationLevel.PIECEWISE,
379 use_cudagraph=True,
380 splitting_ops=["silly.attention"],
381 + cudagraph_capture_sizes=cudagraph_sizes,
382 )
383 else:
384 compilation_config = CompilationConfig(
385 - level=CompilationLevel.PIECEWISE, )
385 + level=CompilationLevel.PIECEWISE,
386 + cudagraph_capture_sizes=cudagraph_sizes,
387 + )
388
389 vllm_config = VllmConfig(compilation_config=compilation_config)
390 with set_current_vllm_config(vllm_config):
@@ -396,17 +398,16 @@ def benchmark():
398
399 graphs = {}
400
399 - with set_compile_context(cudagraph_sizes):
400 - model(input_ids, positions)
401 - for b in cudagraph_sizes[::-1]:
402 - if not piecewise:
403 - graph = torch.cuda.CUDAGraph()
404 - with torch.cuda.graph(graph, pool=pool):
405 - output = model(input_ids[:b], positions[:b])
406 - graphs[b] = (graph, output)
407 - else:
401 + model(input_ids, positions)
402 + for b in cudagraph_sizes[::-1]:
403 + if not piecewise:
404 + graph = torch.cuda.CUDAGraph()
405 + with torch.cuda.graph(graph, pool=pool):
406 output = model(input_ids[:b], positions[:b])
409 - graphs[b] = (model, output)
407 + graphs[b] = (graph, output)
408 + else:
409 + output = model(input_ids[:b], positions[:b])
410 + graphs[b] = (model, output)
411 for b in cudagraph_sizes:
412 if piecewise:
413 # noqa is for `Function definition does not bind loop variable`
tests/models/decoder_only/language/test_jamba.py
+3 -2
@@ -1,8 +1,8 @@
1 import pytest
2
3 from tests.utils import multi_gpu_test
4 +from vllm.config import VllmConfig
5 from vllm.sampling_params import SamplingParams
5 -from vllm.worker.model_runner import _get_graph_batch_size
6
7 from ...utils import check_outputs_equal
8
@@ -189,7 +189,8 @@ def test_mamba_cache_cg_padding(
189 # This test is for verifying that mamba cache is padded to CG captured
190 # batch size. If it's not, a torch RuntimeError will be raised because
191 # tensor dimensions aren't compatible
192 - while len(example_prompts) == _get_graph_batch_size(len(example_prompts)):
192 + while len(example_prompts) == VllmConfig.get_graph_batch_size(
193 + len(example_prompts)):
194 example_prompts.append(example_prompts[0])
195
196 try:
tests/models/decoder_only/language/test_mamba.py
+3 -2
@@ -5,8 +5,8 @@ Run `pytest tests/models/test_mamba.py`.
5 import pytest
6 from transformers import AutoModelForCausalLM, AutoTokenizer
7
8 +from vllm.config import VllmConfig
9 from vllm.sampling_params import SamplingParams
9 -from vllm.worker.model_runner import _get_graph_batch_size
10
11 from ...utils import check_outputs_equal
12
@@ -200,7 +200,8 @@ def test_mamba_cache_cg_padding(
200 # This test is for verifying that mamba cache is padded to CG captured
201 # batch size. If it's not, a torch RuntimeError will be raised because
202 # tensor dimensions aren't compatible
203 - while len(example_prompts) == _get_graph_batch_size(len(example_prompts)):
203 + while len(example_prompts) == VllmConfig.get_graph_batch_size(
204 + len(example_prompts)):
205 example_prompts.append(example_prompts[0])
206
207 try:
tests/worker/test_encoder_decoder_model_runner.py
+2 -2
@@ -4,12 +4,12 @@ from typing import List
4 import pytest
5 import torch
6
7 +from vllm.config import VllmConfig
8 from vllm.engine.arg_utils import EngineArgs
9 from vllm.platforms import current_platform
10 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11 from vllm.utils import make_tensor_with_pad
12 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
12 -from vllm.worker.model_runner import _get_graph_batch_size
13
14 BATCH_SIZES = [1, 4, 16, 64, 256]
15
@@ -548,7 +548,7 @@ def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
548 # With CUDA Graph capture and replay enabled, the decoder and encoder
549 # input sequences will be padded. Create the expected padded tensors
550 # accordingly.
551 - graph_batch_size = _get_graph_batch_size(expanded_batch_size)
551 + graph_batch_size = VllmConfig.get_graph_batch_size(expanded_batch_size)
552 cuda_graph_pad_size = graph_batch_size - expanded_batch_size
553 padded_seq_lens = seq_lens + list(itertools.repeat(1, cuda_graph_pad_size))
554 padded_encoder_seq_lens = encoder_seq_lens + list(
tests/worker/test_model_runner.py
+3 -2
@@ -3,13 +3,14 @@ from typing import List
3 import pytest
4 import torch
5
6 +from vllm.config import VllmConfig
7 from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
8 init_distributed_environment)
9 from vllm.engine.arg_utils import EngineArgs
10 from vllm.model_executor.sampling_metadata import SamplingMetadata
11 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12 from vllm.utils import get_open_port
12 -from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
13 +from vllm.worker.model_runner import ModelRunner
14
15
16 def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
@@ -176,7 +177,7 @@ def test_prepare_decode_cuda_graph(batch_size):
177 model_input.attn_metadata, model_input.attn_metadata.slot_mapping)
178 assert len(slot_mapping) == len(input_tokens)
179
179 - expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
180 + expected_bs = VllmConfig.get_graph_batch_size(len(seq_group_metadata_list))
181 # Verify input metadata is correct for prompts.
182 device = model_runner.device
183 assert attn_metadata.num_prefills == 0
vllm/compilation/backends.py
-4
@@ -242,10 +242,6 @@ class VllmBackend:
242 assert not self._called, "VllmBackend can only be called once"
243
244 self.graph = graph
245 - # config is updated now, because only here can
246 - # we get the sizes to capture for cudagraph
247 - # from compilation context
248 - self.compilation_configs.init_during_runtime()
245 self.configure_post_pass()
246
247 self.split_gm, self.piecewise_graphs = split_graph(
vllm/compilation/compile_context.py deleted
-23
@@ -1,23 +0,0 @@
1 -from contextlib import contextmanager
2 -from typing import Any
3 -
4 -_compile_context: Any = None
5 -
6 -
7 -def get_compile_context() -> Any:
8 - """Get the current compile context."""
9 - return _compile_context
10 -
11 -
12 -@contextmanager
13 -def set_compile_context(context: Any):
14 - """A context manager that stores the current compile context,
15 - usually it is a list of sizes to specialize.
16 - """
17 - global _compile_context
18 - prev_context = _compile_context
19 - _compile_context = context
20 - try:
21 - yield
22 - finally:
23 - _compile_context = prev_context
vllm/config.py
+75 -8
@@ -2357,15 +2357,10 @@ class CompilationConfig(BaseModel):
2357 from vllm.compilation.backends import VllmBackend
2358 return VllmBackend(self)
2359
2360 - def init_during_runtime(self):
2360 + def init_with_cudagraph_sizes(self, sizes_to_specialize: List[int]):
2361 """To complete the initialization of config,
2362 - we need to know the compile context, which is only available
2363 - during the first run of the model.
2364 - """
2365 - from vllm.compilation.compile_context import get_compile_context
2366 - context = get_compile_context()
2367 - context = copy.deepcopy(context) if context is not None else []
2368 - sizes_to_specialize: List[int] = context
2362 + we need to know the cudagraph sizes."""
2363 +
2364 if self.cudagraph_capture_sizes is None:
2365 self.capture_sizes = sizes_to_specialize
2366 else:
@@ -2386,6 +2381,21 @@ class CompilationConfig(BaseModel):
2381 self.inductor_compile_sizes = []
2382 self.compile_sizes = self.inductor_compile_sizes
2383
2384 + # sort to make sure cudagraph capture sizes are in descending order
2385 + self.capture_sizes.sort(reverse=True)
2386 +
2387 +
2388 +_BATCH_SIZE_ALIGNMENT = 8
2389 +# all the token sizes that **can** be captured by cudagraph.
2390 +# they can be arbitrarily large.
2391 +# currently it includes: 1, 2, 4, 8, 16, 24, 32, 40, ..., 8192.
2392 +# the actual sizes to capture will be determined by the model,
2393 +# depending on the model's max_num_seqs.
2394 +# NOTE: get_graph_batch_size needs to be updated if this list is changed.
2395 +_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
2396 + _BATCH_SIZE_ALIGNMENT * i for i in range(1, 1025)
2397 +]
2398 +
2399
2400 @dataclass
2401 class VllmConfig:
@@ -2413,6 +2423,41 @@ class VllmConfig:
2423 kv_transfer_config: KVTransferConfig = field(default=None,
2424 init=True) # type: ignore
2425
2426 + @staticmethod
2427 + def get_graph_batch_size(batch_size: int) -> int:
2428 + """Returns the padded batch size given actual batch size.
2429 +
2430 + Batch sizes are 1, 2, 4, _BATCH_SIZE_ALIGNMENT,
2431 + 2*_BATCH_SIZE_ALIGNMENT, 3*_BATCH_SIZE_ALIGNMENT...
2432 + """
2433 + if batch_size <= 2:
2434 + return batch_size
2435 + elif batch_size <= 4:
2436 + return 4
2437 + else:
2438 + return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
2439 + _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
2440 +
2441 + @staticmethod
2442 + def get_max_graph_batch_size(max_num_seqs: int) -> int:
2443 + """
2444 + max_num_seqs: Maximum number of sequences in a batch.
2445 + _BATCH_SIZES_TO_CAPTURE: all the sizes that we want to capture.
2446 +
2447 + pad the max_num_seqs if necessary by calling get_graph_batch_size,
2448 + which will deal with some edge cases like 1, 2, 4.
2449 +
2450 + if the padded size is in _BATCH_SIZES_TO_CAPTURE, return the padded
2451 + size. if not, it means the padded size is larger than the largest size
2452 + in _BATCH_SIZES_TO_CAPTURE, return the largest size in
2453 + _BATCH_SIZES_TO_CAPTURE.
2454 + """
2455 + padded_size = VllmConfig.get_graph_batch_size(max_num_seqs)
2456 + if padded_size in _BATCH_SIZES_TO_CAPTURE:
2457 + return padded_size
2458 + assert padded_size > _BATCH_SIZES_TO_CAPTURE[-1]
2459 + return _BATCH_SIZES_TO_CAPTURE[-1]
2460 +
2461 @staticmethod
2462 def _get_quantization_config(
2463 model_config: ModelConfig,
@@ -2496,6 +2541,28 @@ class VllmConfig:
2541 self.compilation_config.pass_config.enable_reshape = False
2542 self.compilation_config.level = CompilationLevel.PIECEWISE
2543
2544 + if not envs.VLLM_USE_V1:
2545 + max_batchsize_to_capture = 0
2546 + if self.scheduler_config is not None and \
2547 + self.model_config is not None and \
2548 + not self.model_config.enforce_eager:
2549 + max_batchsize_to_capture = \
2550 + self.get_max_graph_batch_size(
2551 + self.scheduler_config.max_num_seqs)
2552 + batch_size_capture_list = [
2553 + size for size in _BATCH_SIZES_TO_CAPTURE
2554 + if size <= max_batchsize_to_capture
2555 + ]
2556 + else:
2557 + batch_size_capture_list = []
2558 + if self.model_config is not None and \
2559 + not self.model_config.enforce_eager:
2560 + batch_size_capture_list = [1, 2, 4
2561 + ] + [i for i in range(8, 513, 8)]
2562 +
2563 + self.compilation_config.init_with_cudagraph_sizes(
2564 + batch_size_capture_list)
2565 +
2566 if self.cache_config is not None and \
2567 self.cache_config.cpu_offload_gb > 0 and \
2568 self.compilation_config.level != CompilationLevel.NO_COMPILATION:
vllm/model_executor/models/jamba.py
+2 -4
@@ -7,7 +7,7 @@ from transformers import JambaConfig
7
8 from vllm.attention.backends.abstract import AttentionMetadata
9 from vllm.attention.layer import Attention
10 -from vllm.config import CacheConfig, VllmConfig
10 +from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
11 from vllm.distributed import get_tensor_model_parallel_world_size
12 from vllm.model_executor.layers.fused_moe import FusedMoE
13 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -25,8 +25,6 @@ from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
25 MambaCacheParams)
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 from vllm.sequence import IntermediateTensors
28 -from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
29 - _get_graph_batch_size)
28
29 from .interfaces import HasInnerState, SupportsLoRA
30 from .utils import maybe_prefix
@@ -404,7 +402,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
402 inputs_embeds: Optional[torch.Tensor] = None,
403 **kwargs):
404 if self.mamba_cache is None:
407 - max_batch_size = (_get_graph_batch_size(
405 + max_batch_size = (VllmConfig.get_graph_batch_size(
406 self.scheduler_config.max_num_seqs) if self.scheduler_config
407 else max(_BATCH_SIZES_TO_CAPTURE) + 2)
408
vllm/model_executor/models/mamba.py
+2 -4
@@ -6,7 +6,7 @@ from torch import nn
6 from transformers import MambaConfig
7
8 from vllm.attention.backends.abstract import AttentionMetadata
9 -from vllm.config import CacheConfig, VllmConfig
9 +from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
10 from vllm.distributed import get_tensor_model_parallel_world_size
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -23,8 +23,6 @@ from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
23 MambaCacheParams)
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 from vllm.sequence import IntermediateTensors
26 -from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
27 - _get_graph_batch_size)
26
27 from .utils import maybe_prefix
28
@@ -187,7 +185,7 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
185 inputs_embeds: Optional[torch.Tensor] = None,
186 **kwargs):
187 if self.mamba_cache is None:
190 - max_batch_size = (_get_graph_batch_size(
188 + max_batch_size = (VllmConfig.get_graph_batch_size(
189 self.scheduler_config.max_num_seqs) if self.scheduler_config
190 else max(_BATCH_SIZES_TO_CAPTURE) + 2)
191 self.mamba_cache = MambaCacheManager(
vllm/v1/worker/gpu_model_runner.py
+8 -6
@@ -8,7 +8,6 @@ import torch
8 import torch.distributed
9 import torch.nn as nn
10
11 -from vllm.compilation.compile_context import set_compile_context
11 from vllm.config import CompilationLevel, VllmConfig
12 from vllm.distributed.parallel_state import graph_capture
13 from vllm.forward_context import set_forward_context
@@ -100,7 +99,11 @@ class GPUModelRunner:
99 == CompilationLevel.PIECEWISE
100 and not self.model_config.enforce_eager)
101 # TODO(woosuk): Provide an option to tune the max cudagraph batch size.
103 - self.cudagraph_batch_sizes = [1, 2, 4] + [i for i in range(8, 513, 8)]
102 + # The convention is different.
103 + # self.cudagraph_batch_sizes sorts in ascending order.
104 + # The batch sizes in the config are in descending order.
105 + self.cudagraph_batch_sizes = list(
106 + reversed(self.vllm_config.compilation_config.capture_sizes))
107 self.positions = torch.zeros(self.max_num_tokens,
108 dtype=torch.int64,
109 device=self.device)
@@ -548,10 +551,9 @@ class GPUModelRunner:
551 torch.tensor([], dtype=torch.float32, device=self.device)
552 for _ in range(self.num_attn_layers)
553 ]
551 - with set_compile_context(self.cudagraph_batch_sizes):
552 - # Trigger compilation for general shape.
553 - hidden_states = self._dummy_run(self.model, self.max_num_tokens,
554 - dummy_kv_caches)
554 + # Trigger compilation for general shape.
555 + hidden_states = self._dummy_run(self.model, self.max_num_tokens,
556 + dummy_kv_caches)
557 logits = self.model.compute_logits(hidden_states, None)
558 logits = logits[:self.max_num_tokens]
559 # TODO(woosuk): Consider the memory usage of the sampler.
vllm/worker/enc_dec_model_runner.py
+3 -3
@@ -25,8 +25,7 @@ from vllm.sequence import (IntermediateTensors, PoolerOutput,
25 from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
26 from vllm.worker.model_runner import (GPUModelRunnerBase,
27 ModelInputForGPUBuilder,
28 - ModelInputForGPUWithSamplingMetadata,
29 - _get_graph_batch_size)
28 + ModelInputForGPUWithSamplingMetadata)
29 from vllm.worker.model_runner_base import (
30 _add_attn_metadata_broadcastable_dict,
31 _add_sampling_metadata_broadcastable_dict)
@@ -465,7 +464,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
464 # We will be using CUDA graph replay for this decode.
465 max_len_of_block_table = self.get_max_block_per_batch()
466 batch_size = len(encoder_seq_lens)
468 - graph_batch_size = _get_graph_batch_size(batch_size)
467 + graph_batch_size = self.vllm_config.get_graph_batch_size(
468 + batch_size)
469 assert graph_batch_size >= batch_size
470 cuda_graph_pad_size = graph_batch_size - batch_size
471 # extend the cross_block_tables and encoder_seq_lens to match
vllm/worker/model_runner.py
+6 -62
@@ -18,7 +18,6 @@ import vllm.envs as envs
18 from vllm.attention import AttentionMetadata, get_attn_backend
19 from vllm.attention.backends.abstract import AttentionState
20 from vllm.attention.backends.utils import CommonAttentionState
21 -from vllm.compilation.compile_context import set_compile_context
21 from vllm.config import CompilationLevel, VllmConfig
22 from vllm.core.scheduler import SchedulerOutputs
23 from vllm.distributed import get_kv_transfer_group, get_pp_group
@@ -63,16 +62,7 @@ if TYPE_CHECKING:
62 logger = init_logger(__name__)
63
64 LORA_WARMUP_RANK = 8
66 -_BATCH_SIZE_ALIGNMENT = 8
67 -# all the token sizes that **can** be captured by cudagraph.
68 -# they can be arbitrarily large.
69 -# currently it includes: 1, 2, 4, 8, 16, 24, 32, 40, ..., 8192.
70 -# the actual sizes to capture will be determined by the model,
71 -# depending on the model's max_num_seqs.
72 -# NOTE: _get_graph_batch_size needs to be updated if this list is changed.
73 -_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
74 - _BATCH_SIZE_ALIGNMENT * i for i in range(1, 1025)
75 -]
65 +
66 _NUM_WARMUP_ITERS = 2
67
68 TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU")
@@ -763,7 +753,6 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
753 max_decode_seq_len: int,
754 max_encoder_seq_len: int = 0) -> bool:
755 return (decode_only and not self.runner.model_config.enforce_eager
766 - and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
756 and max_decode_seq_len <= self.runner.max_seq_len_to_capture
757 and max_encoder_seq_len <= self.runner.max_seq_len_to_capture
758 and batch_size <= self.runner.max_batchsize_to_capture)
@@ -811,7 +800,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
800 max_encoder_seq_len):
801 return -1
802
814 - graph_batch_size = _get_graph_batch_size(batch_size)
803 + graph_batch_size = VllmConfig.get_graph_batch_size(batch_size)
804 assert graph_batch_size >= batch_size
805 return graph_batch_size - batch_size
806
@@ -1023,7 +1012,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1012 self.sliding_window = model_config.get_sliding_window()
1013 self.block_size = cache_config.block_size
1014 self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
1026 - self.max_batchsize_to_capture = _get_max_graph_batch_size(
1015 + self.max_batchsize_to_capture = VllmConfig.get_max_graph_batch_size(
1016 self.scheduler_config.max_num_seqs)
1017
1018 self.graph_runners: List[Dict[int, CUDAGraphRunner]] = [
@@ -1333,14 +1322,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1322 dtype=self.model_config.dtype,
1323 device=self.device)
1324
1336 - graph_batch_size = self.max_batchsize_to_capture
1337 - batch_size_capture_list = [
1338 - bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1339 - ]
1340 - if self.model_config.enforce_eager:
1341 - batch_size_capture_list = []
1342 - with set_compile_context(batch_size_capture_list):
1343 - self.execute_model(model_input, kv_caches, intermediate_tensors)
1325 + self.execute_model(model_input, kv_caches, intermediate_tensors)
1326 torch.cuda.synchronize()
1327 return
1328
@@ -1459,18 +1441,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1441 dtype=self.model_config.dtype,
1442 device=self.device)
1443
1462 - graph_batch_size = self.max_batchsize_to_capture
1463 - batch_size_capture_list = [
1464 - bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1465 - ]
1466 -
1444 with self.attn_state.graph_capture(
1445 max_batch_size), graph_capture() as graph_capture_context:
1446 # NOTE: Capturing the largest batch size first may help reduce the
1447 # memory usage of CUDA graph.
1448 for virtual_engine in range(
1449 self.parallel_config.pipeline_parallel_size):
1473 - for batch_size in reversed(batch_size_capture_list):
1450 + for batch_size in \
1451 + self.vllm_config.compilation_config.capture_sizes:
1452 attn_metadata = (
1453 self.attn_state.graph_capture_get_metadata_for_batch(
1454 batch_size,
@@ -1993,37 +1971,3 @@ class CUDAGraphRunner(nn.Module):
1971 return self.output_buffers["hidden_states"]
1972
1973 return self.output_buffers
1996 -
1997 -
1998 -def _get_graph_batch_size(batch_size: int) -> int:
1999 - """Returns the padded batch size given actual batch size.
2000 -
2001 - Batch sizes are 1, 2, 4, _BATCH_SIZE_ALIGNMENT,
2002 - 2*_BATCH_SIZE_ALIGNMENT, 3*_BATCH_SIZE_ALIGNMENT...
2003 - """
2004 - if batch_size <= 2:
2005 - return batch_size
2006 - elif batch_size <= 4:
2007 - return 4
2008 - else:
2009 - return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
2010 - _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
2011 -
2012 -
2013 -def _get_max_graph_batch_size(max_num_seqs: int) -> int:
2014 - """
2015 - max_num_seqs: Maximum number of sequences in a batch.
2016 - _BATCH_SIZES_TO_CAPTURE: all the sizes that we want to capture.
2017 -
2018 - pad the max_num_seqs if necessary by calling _get_graph_batch_size,
2019 - which will deal with some edge cases like 1, 2, 4.
2020 -
2021 - if the padded size is in _BATCH_SIZES_TO_CAPTURE, return the padded size.
2022 - if not, it means the padded size is larger than the largest size in
2023 - _BATCH_SIZES_TO_CAPTURE, return the largest size in _BATCH_SIZES_TO_CAPTURE.
2024 - """
2025 - padded_size = _get_graph_batch_size(max_num_seqs)
2026 - if padded_size in _BATCH_SIZES_TO_CAPTURE:
2027 - return padded_size
2028 - assert padded_size > _BATCH_SIZES_TO_CAPTURE[-1]
2029 - return _BATCH_SIZES_TO_CAPTURE[-1]