A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[torch.compile] allow tracking forward time (#11081)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 14, 2024 at 19:45 UTC
a1c02058baf47be1a91ee743378a340ee1b10416
1 file changed
+42
-19
vllm/forward_context.py
+42
-19
@@ -1,9 +1,11 @@
1
import time
2
-from collections import Counter
2
+from collections import defaultdict
3
from contextlib import contextmanager
4
from dataclasses import dataclass
5
from typing import Any, Dict, Optional
6
7
+import torch
8
+
9
import vllm.envs as envs
10
from vllm.config import VllmConfig
11
from vllm.logger import init_logger
@@ -11,9 +13,10 @@ from vllm.logger import init_logger
13
logger = init_logger(__name__)
14
15
track_batchsize: bool = envs.VLLM_LOG_BATCHSIZE_INTERVAL >= 0
14
-batchsize_counter: Counter = Counter()
16
last_logging_time: float = 0
17
+forward_start_time: float = 0
18
batchsize_logging_interval: float = envs.VLLM_LOG_BATCHSIZE_INTERVAL
19
+batchsize_forward_time: defaultdict = defaultdict(list)
20
21
22
@dataclass
@@ -40,23 +43,10 @@ def set_forward_context(context: Any, vllm_config: VllmConfig):
43
can be attention metadata, etc.
44
Here we can inject common logic for every model forward pass.
45
"""
43
- global track_batchsize, batchsize_counter
44
- global last_logging_time, batchsize_logging_interval
45
- if track_batchsize and context is not None:
46
- if hasattr(context, "num_prefill_tokens"):
47
- # for v0 attention backends
48
- batchsize = context.num_prefill_tokens + context.num_decode_tokens
49
- else:
50
- # for v1 attention backends
51
- batchsize = context.num_input_tokens
52
- batchsize_counter[batchsize] += 1
53
- if time.monotonic() - last_logging_time > batchsize_logging_interval:
54
- last_logging_time = time.monotonic()
55
- sorted_data = sorted(batchsize_counter.items(),
56
- key=lambda x: x[1],
57
- reverse=True)
58
- logger.info("Batchsize distribution (batchsize, count): %s",
59
- sorted_data)
46
+ global forward_start_time
47
+ need_to_track_batchsize = track_batchsize and context is not None
48
+ if need_to_track_batchsize:
49
+ forward_start_time = time.perf_counter()
50
global _forward_context
51
prev_context = _forward_context
52
_forward_context = ForwardContext(
@@ -66,4 +56,37 @@ def set_forward_context(context: Any, vllm_config: VllmConfig):
56
try:
57
yield
58
finally:
59
+ global batchsize_counter
60
+ global last_logging_time, batchsize_logging_interval
61
+ if need_to_track_batchsize:
62
+ if hasattr(context, "num_prefill_tokens"):
63
+ # for v0 attention backends
64
+ batchsize = context.num_prefill_tokens + \
65
+ context.num_decode_tokens
66
+ else:
67
+ # for v1 attention backends
68
+ batchsize = context.num_input_tokens
69
+ # we use synchronous scheduling right now,
70
+ # adding a sync point here should not affect
71
+ # scheduling of the next batch
72
+ torch.cuda.synchronize()
73
+ now = time.perf_counter()
74
+ # time measurement is in milliseconds
75
+ batchsize_forward_time[batchsize].append(
76
+ (now - forward_start_time) * 1000)
77
+ if now - last_logging_time > batchsize_logging_interval:
78
+ last_logging_time = now
79
+ forward_stats = []
80
+ for bs, times in batchsize_forward_time.items():
81
+ if len(times) <= 1:
82
+ # can be cudagraph / profiling run
83
+ continue
84
+ medium = torch.quantile(torch.tensor(times), q=0.5).item()
85
+ medium = round(medium, 2)
86
+ forward_stats.append((bs, len(times), medium))
87
+ forward_stats.sort(key=lambda x: x[1], reverse=True)
88
+ if forward_stats:
89
+ logger.info(("Batchsize forward time stats "
90
+ "(batchsize, count, median_time(ms)): %s"),
91
+ forward_stats)
92
_forward_context = prev_context