@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a1c02058ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[torch.compile] allow tracking forward time (#11081)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 14, 2024 at 19:45 UTC a1c02058baf47be1a91ee743378a340ee1b10416
1 file changed +42 -19
vllm/forward_context.py
+42 -19
@@ -1,9 +1,11 @@
1 import time
2 -from collections import Counter
2 +from collections import defaultdict
3 from contextlib import contextmanager
4 from dataclasses import dataclass
5 from typing import Any, Dict, Optional
6
7 +import torch
8 +
9 import vllm.envs as envs
10 from vllm.config import VllmConfig
11 from vllm.logger import init_logger
@@ -11,9 +13,10 @@ from vllm.logger import init_logger
13 logger = init_logger(__name__)
14
15 track_batchsize: bool = envs.VLLM_LOG_BATCHSIZE_INTERVAL >= 0
14 -batchsize_counter: Counter = Counter()
16 last_logging_time: float = 0
17 +forward_start_time: float = 0
18 batchsize_logging_interval: float = envs.VLLM_LOG_BATCHSIZE_INTERVAL
19 +batchsize_forward_time: defaultdict = defaultdict(list)
20
21
22 @dataclass
@@ -40,23 +43,10 @@ def set_forward_context(context: Any, vllm_config: VllmConfig):
43 can be attention metadata, etc.
44 Here we can inject common logic for every model forward pass.
45 """
43 - global track_batchsize, batchsize_counter
44 - global last_logging_time, batchsize_logging_interval
45 - if track_batchsize and context is not None:
46 - if hasattr(context, "num_prefill_tokens"):
47 - # for v0 attention backends
48 - batchsize = context.num_prefill_tokens + context.num_decode_tokens
49 - else:
50 - # for v1 attention backends
51 - batchsize = context.num_input_tokens
52 - batchsize_counter[batchsize] += 1
53 - if time.monotonic() - last_logging_time > batchsize_logging_interval:
54 - last_logging_time = time.monotonic()
55 - sorted_data = sorted(batchsize_counter.items(),
56 - key=lambda x: x[1],
57 - reverse=True)
58 - logger.info("Batchsize distribution (batchsize, count): %s",
59 - sorted_data)
46 + global forward_start_time
47 + need_to_track_batchsize = track_batchsize and context is not None
48 + if need_to_track_batchsize:
49 + forward_start_time = time.perf_counter()
50 global _forward_context
51 prev_context = _forward_context
52 _forward_context = ForwardContext(
@@ -66,4 +56,37 @@ def set_forward_context(context: Any, vllm_config: VllmConfig):
56 try:
57 yield
58 finally:
59 + global batchsize_counter
60 + global last_logging_time, batchsize_logging_interval
61 + if need_to_track_batchsize:
62 + if hasattr(context, "num_prefill_tokens"):
63 + # for v0 attention backends
64 + batchsize = context.num_prefill_tokens + \
65 + context.num_decode_tokens
66 + else:
67 + # for v1 attention backends
68 + batchsize = context.num_input_tokens
69 + # we use synchronous scheduling right now,
70 + # adding a sync point here should not affect
71 + # scheduling of the next batch
72 + torch.cuda.synchronize()
73 + now = time.perf_counter()
74 + # time measurement is in milliseconds
75 + batchsize_forward_time[batchsize].append(
76 + (now - forward_start_time) * 1000)
77 + if now - last_logging_time > batchsize_logging_interval:
78 + last_logging_time = now
79 + forward_stats = []
80 + for bs, times in batchsize_forward_time.items():
81 + if len(times) <= 1:
82 + # can be cudagraph / profiling run
83 + continue
84 + medium = torch.quantile(torch.tensor(times), q=0.5).item()
85 + medium = round(medium, 2)
86 + forward_stats.append((bs, len(times), medium))
87 + forward_stats.sort(key=lambda x: x[1], reverse=True)
88 + if forward_stats:
89 + logger.info(("Batchsize forward time stats "
90 + "(batchsize, count, median_time(ms)): %s"),
91 + forward_stats)
92 _forward_context = prev_context