@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 240 lines 5.88 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Default bucket boundaries for the Prometheus histograms emitted by vLLM.
4
5 Every core engine histogram created by the Prometheus stat logger draws its
6 bucket boundaries from exactly one of the families defined here, so each
7 default list has a single source of truth. Histograms owned by other
8 subsystems, such as the KV connector and offloading metrics, keep their own
9 boundaries.
10 """
11
12 from collections.abc import Mapping
13 from typing import Literal, get_args
14
15 BucketFamilyKey = Literal[
16 "request_latency",
17 "time_to_first_token",
18 "inter_token_latency",
19 "iteration_tokens",
20 "request_params_n",
21 "request_num_preemptions",
22 "request_tokens",
23 "kv_cache_residency",
24 ]
25 """Canonical keys for the histogram bucket families."""
26
27 BUCKET_FAMILY_KEYS: frozenset[str] = frozenset(get_args(BucketFamilyKey))
28 """All bucket family keys, for membership checks."""
29
30 REQUEST_LATENCY_BUCKETS: tuple[float, ...] = (
31 0.3,
32 0.5,
33 0.8,
34 1.0,
35 1.5,
36 2.0,
37 2.5,
38 5.0,
39 10.0,
40 15.0,
41 20.0,
42 30.0,
43 40.0,
44 50.0,
45 60.0,
46 120.0,
47 240.0,
48 480.0,
49 960.0,
50 1920.0,
51 7680.0,
52 )
53 """Sub-second scheduling delays through multi-hour batch requests; shared by
54 the accumulated request phase-timing histograms (e2e, queue, inference,
55 prefill, decode)."""
56
57 TIME_TO_FIRST_TOKEN_BUCKETS: tuple[float, ...] = (
58 0.001,
59 0.005,
60 0.01,
61 0.02,
62 0.04,
63 0.06,
64 0.08,
65 0.1,
66 0.25,
67 0.5,
68 0.75,
69 1.0,
70 2.5,
71 5.0,
72 7.5,
73 10.0,
74 20.0,
75 40.0,
76 80.0,
77 160.0,
78 640.0,
79 2560.0,
80 )
81 """Millisecond-scale prefill for tiny prompts up to ~40-minute worst cases,
82 with the densest resolution around interactive (sub-second) latencies."""
83
84 INTER_TOKEN_LATENCY_BUCKETS: tuple[float, ...] = (
85 0.01,
86 0.025,
87 0.05,
88 0.075,
89 0.1,
90 0.15,
91 0.2,
92 0.3,
93 0.4,
94 0.5,
95 0.75,
96 1.0,
97 2.5,
98 5.0,
99 7.5,
100 10.0,
101 20.0,
102 40.0,
103 80.0,
104 )
105 """Per-decode-step latencies: 10 ms fast decode up to multi-second stalls."""
106
107 ITERATION_TOKENS_BUCKETS: tuple[float, ...] = (
108 1,
109 8,
110 16,
111 32,
112 64,
113 128,
114 256,
115 512,
116 1024,
117 2048,
118 4096,
119 8192,
120 16384,
121 )
122 """Tokens processed per engine step: powers of two up to a typical
123 max-num-batched-tokens budget."""
124
125 REQUEST_PARAMS_N_BUCKETS: tuple[float, ...] = (1, 2, 5, 10, 20)
126 """Small integer counts for the ``n`` sampling parameter."""
127
128 REQUEST_NUM_PREEMPTIONS_BUCKETS: tuple[float, ...] = (1, 2, 3, 4, 5, 10, 20)
129 """Per-request preemption counts, recorded for every finished request: the
130 first bucket therefore holds never-preempted requests together with
131 singly-preempted ones, 2 through 5 are separated individually, and the tail
132 coarsens to 20. Distinct from `REQUEST_PARAMS_N_BUCKETS` because these are the
133 boundaries the metric shipped with, not a harmonized choice."""
134
135 KV_CACHE_RESIDENCY_BUCKETS: tuple[float, ...] = (
136 0.001,
137 0.002,
138 0.005,
139 0.01,
140 0.02,
141 0.05,
142 0.1,
143 0.2,
144 0.5,
145 1,
146 2,
147 5,
148 10,
149 20,
150 30,
151 60,
152 120,
153 300,
154 600,
155 1200,
156 1800,
157 )
158 """KV cache block residency times: millisecond-scale reuse gaps up to
159 30-minute block lifetimes."""
160
161 _STATIC_FAMILY_DEFAULTS: dict[str, tuple[float, ...]] = {
162 "request_latency": REQUEST_LATENCY_BUCKETS,
163 "time_to_first_token": TIME_TO_FIRST_TOKEN_BUCKETS,
164 "inter_token_latency": INTER_TOKEN_LATENCY_BUCKETS,
165 "iteration_tokens": ITERATION_TOKENS_BUCKETS,
166 "request_params_n": REQUEST_PARAMS_N_BUCKETS,
167 "request_num_preemptions": REQUEST_NUM_PREEMPTIONS_BUCKETS,
168 "kv_cache_residency": KV_CACHE_RESIDENCY_BUCKETS,
169 }
170
171
172 def build_buckets(mantissa_lst: list[int], max_value: int) -> list[float]:
173 """Build buckets from mantissas scaled by increasing powers of 10.
174
175 Args:
176 mantissa_lst: Mantissa values multiplied by each power of 10.
177 max_value: Largest bucket value to include.
178
179 Returns:
180 Bucket values in increasing order, capped at `max_value`.
181
182 """
183 exponent = 0
184 buckets: list[float] = []
185 while True:
186 for m in mantissa_lst:
187 value = m * 10**exponent
188 if value <= max_value:
189 buckets.append(value)
190 else:
191 return buckets
192 exponent += 1
193
194
195 def build_1_2_5_buckets(max_value: int) -> list[float]:
196 """Build a 1-2-5 series of buckets capped at `max_value`.
197
198 Example:
199 >>> build_1_2_5_buckets(100)
200 [1, 2, 5, 10, 20, 50, 100]
201
202 """
203 return build_buckets([1, 2, 5], max_value)
204
205
206 def histogram_buckets(
207 family: BucketFamilyKey,
208 max_model_len: int | None = None,
209 overrides: Mapping[str, list[float]] | None = None,
210 ) -> list[float]:
211 """Return the bucket boundaries for a histogram family.
212
213 Args:
214 family: Canonical bucket family key.
215 max_model_len: Cap for the token-count series of the
216 `request_tokens` family; required for that family and ignored
217 otherwise.
218 overrides: Mapping from family key to replacement bucket bounds.
219 A present key replaces the family's defaults verbatim. Values
220 must already be validated; the single validation point is
221 `ObservabilityConfig.custom_histogram_buckets`.
222
223 Returns:
224 A fresh list of bucket upper bounds; callers may mutate it freely.
225
226 Raises:
227 ValueError: If `family` is `request_tokens`, no override is given,
228 and `max_model_len` is None.
229
230
231 """
232 if overrides is not None and (custom := overrides.get(family)) is not None:
233 return list(custom)
234 if family == "request_tokens":
235 if max_model_len is None:
236 raise ValueError(
237 "max_model_len is required for the 'request_tokens' bucket family"
238 )
239 return build_1_2_5_buckets(max_model_len)
240 return list(_STATIC_FAMILY_DEFAULTS[family])