@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 75f89dc44c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[torch.compile] add a flag to track batchsize statistics (#11059)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 10, 2024 at 12:40 UTC 75f89dc44c6e44cc28bae59d5b40a588735b507b
4 files changed +37 -1
vllm/envs.py
+3
@@ -69,6 +69,7 @@ if TYPE_CHECKING:
69 VLLM_DISABLED_KERNELS: List[str] = []
70 VLLM_USE_V1: bool = False
71 VLLM_ENABLE_V1_MULTIPROCESSING: bool = False
72 + VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
73
74
75 def get_default_cache_root():
@@ -452,6 +453,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
453 # If set, enable multiprocessing in LLM for the V1 code path.
454 "VLLM_ENABLE_V1_MULTIPROCESSING":
455 lambda: bool(int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0"))),
456 + "VLLM_LOG_BATCHSIZE_INTERVAL":
457 + lambda: float(os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1")),
458 }
459
460 # end-env-vars-definition
vllm/forward_context.py
+31 -1
@@ -1,8 +1,19 @@
1 +import time
2 +from collections import Counter
3 from contextlib import contextmanager
4 from dataclasses import dataclass
5 from typing import Any, Dict, Optional
6
7 +import vllm.envs as envs
8 from vllm.config import VllmConfig
9 +from vllm.logger import init_logger
10 +
11 +logger = init_logger(__name__)
12 +
13 +track_batchsize: bool = envs.VLLM_LOG_BATCHSIZE_INTERVAL >= 0
14 +batchsize_counter: Counter = Counter()
15 +last_logging_time: float = 0
16 +batchsize_logging_interval: float = envs.VLLM_LOG_BATCHSIZE_INTERVAL
17
18
19 @dataclass
@@ -26,7 +37,26 @@ def get_forward_context() -> ForwardContext:
37 @contextmanager
38 def set_forward_context(context: Any, vllm_config: VllmConfig):
39 """A context manager that stores the current forward context,
29 - can be attention metadata, etc."""
40 + can be attention metadata, etc.
41 + Here we can inject common logic for every model forward pass.
42 + """
43 + global track_batchsize, batchsize_counter
44 + global last_logging_time, batchsize_logging_interval
45 + if track_batchsize and context is not None:
46 + if hasattr(context, "num_prefill_tokens"):
47 + # for v0 attention backends
48 + batchsize = context.num_prefill_tokens + context.num_decode_tokens
49 + else:
50 + # for v1 attention backends
51 + batchsize = context.num_input_tokens
52 + batchsize_counter[batchsize] += 1
53 + if time.monotonic() - last_logging_time > batchsize_logging_interval:
54 + last_logging_time = time.monotonic()
55 + sorted_data = sorted(batchsize_counter.items(),
56 + key=lambda x: x[1],
57 + reverse=True)
58 + logger.info("Batchsize distribution (batchsize, count): %s",
59 + sorted_data)
60 global _forward_context
61 prev_context = _forward_context
62 _forward_context = ForwardContext(
vllm/v1/attention/backends/flash_attn.py
+1
@@ -56,6 +56,7 @@ class FlashAttentionMetadata:
56 seq_start_loc: torch.Tensor
57 block_table: torch.Tensor
58 slot_mapping: torch.Tensor
59 + num_input_tokens: int = 0 # Number of tokens including padding.
60
61
62 class FlashAttentionImpl(AttentionImpl):
vllm/v1/worker/gpu_model_runner.py
+2
@@ -445,6 +445,8 @@ class GPUModelRunner:
445 # Eager mode.
446 num_input_tokens = num_scheduled_tokens
447
448 + attn_metadata.num_input_tokens = num_input_tokens
449 +
450 # Get the inputs embeds.
451 if encoder_outputs:
452 inputs_embeds = self.model.get_input_embeddings(