A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[torch.compile] add a flag to track batchsize statistics (#11059)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 10, 2024 at 12:40 UTC
75f89dc44c6e44cc28bae59d5b40a588735b507b
4 files changed
+37
-1
vllm/envs.py
+3
@@ -69,6 +69,7 @@ if TYPE_CHECKING:
69
VLLM_DISABLED_KERNELS: List[str] = []
70
VLLM_USE_V1: bool = False
71
VLLM_ENABLE_V1_MULTIPROCESSING: bool = False
72
+ VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
73
74
75
def get_default_cache_root():
@@ -452,6 +453,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
453
# If set, enable multiprocessing in LLM for the V1 code path.
454
"VLLM_ENABLE_V1_MULTIPROCESSING":
455
lambda: bool(int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0"))),
456
+ "VLLM_LOG_BATCHSIZE_INTERVAL":
457
+ lambda: float(os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1")),
458
}
459
460
# end-env-vars-definition
vllm/forward_context.py
+31
-1
@@ -1,8 +1,19 @@
1
+import time
2
+from collections import Counter
3
from contextlib import contextmanager
4
from dataclasses import dataclass
5
from typing import Any, Dict, Optional
6
7
+import vllm.envs as envs
8
from vllm.config import VllmConfig
9
+from vllm.logger import init_logger
10
+
11
+logger = init_logger(__name__)
12
+
13
+track_batchsize: bool = envs.VLLM_LOG_BATCHSIZE_INTERVAL >= 0
14
+batchsize_counter: Counter = Counter()
15
+last_logging_time: float = 0
16
+batchsize_logging_interval: float = envs.VLLM_LOG_BATCHSIZE_INTERVAL
17
18
19
@dataclass
@@ -26,7 +37,26 @@ def get_forward_context() -> ForwardContext:
37
@contextmanager
38
def set_forward_context(context: Any, vllm_config: VllmConfig):
39
"""A context manager that stores the current forward context,
29
- can be attention metadata, etc."""
40
+ can be attention metadata, etc.
41
+ Here we can inject common logic for every model forward pass.
42
+ """
43
+ global track_batchsize, batchsize_counter
44
+ global last_logging_time, batchsize_logging_interval
45
+ if track_batchsize and context is not None:
46
+ if hasattr(context, "num_prefill_tokens"):
47
+ # for v0 attention backends
48
+ batchsize = context.num_prefill_tokens + context.num_decode_tokens
49
+ else:
50
+ # for v1 attention backends
51
+ batchsize = context.num_input_tokens
52
+ batchsize_counter[batchsize] += 1
53
+ if time.monotonic() - last_logging_time > batchsize_logging_interval:
54
+ last_logging_time = time.monotonic()
55
+ sorted_data = sorted(batchsize_counter.items(),
56
+ key=lambda x: x[1],
57
+ reverse=True)
58
+ logger.info("Batchsize distribution (batchsize, count): %s",
59
+ sorted_data)
60
global _forward_context
61
prev_context = _forward_context
62
_forward_context = ForwardContext(
vllm/v1/attention/backends/flash_attn.py
+1
@@ -56,6 +56,7 @@ class FlashAttentionMetadata:
56
seq_start_loc: torch.Tensor
57
block_table: torch.Tensor
58
slot_mapping: torch.Tensor
59
+ num_input_tokens: int = 0 # Number of tokens including padding.
60
61
62
class FlashAttentionImpl(AttentionImpl):
vllm/v1/worker/gpu_model_runner.py
+2
@@ -445,6 +445,8 @@ class GPUModelRunner:
445
# Eager mode.
446
num_input_tokens = num_scheduled_tokens
447
448
+ attn_metadata.num_input_tokens = num_input_tokens
449
+
450
# Get the inputs embeds.
451
if encoder_outputs:
452
inputs_embeds = self.model.get_input_embeddings(