A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Misc] Change buckets of histogram_iteration_tokens to [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096] to represent number of tokens (#17033)
Signed-off-by: sfc-gh-zhwang <flex.wang@snowflake.com>
Flex Wang committed
Apr 27, 2025 at 05:30 UTC
18445edd0f19b3d734315f968ed9a554937aab20
2 files changed
+7
-17
vllm/engine/metrics.py
+3
-6
@@ -140,16 +140,13 @@ class Metrics:
140
name="vllm:generation_tokens_total",
141
documentation="Number of generation tokens processed.",
142
labelnames=labelnames)
143
- buckets = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
144
- if not vllm_config.model_config.enforce_eager:
145
- buckets = vllm_config.compilation_config.\
146
- cudagraph_capture_sizes.copy()
147
- buckets.sort()
143
self.histogram_iteration_tokens = self._histogram_cls(
144
name="vllm:iteration_tokens_total",
145
documentation="Histogram of number of tokens per engine_step.",
146
labelnames=labelnames,
152
- buckets=buckets)
147
+ buckets=[
148
+ 1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096, 16192
149
+ ])
150
self.histogram_time_to_first_token = self._histogram_cls(
151
name="vllm:time_to_first_token_seconds",
152
documentation="Histogram of time to first token in seconds.",
vllm/v1/metrics/loggers.py
+4
-11
@@ -232,7 +232,10 @@ class PrometheusStatLogger(StatLoggerBase):
232
prometheus_client.Histogram(
233
name="vllm:iteration_tokens_total",
234
documentation="Histogram of number of tokens per engine_step.",
235
- buckets=build_cudagraph_buckets(vllm_config),
235
+ buckets=[
236
+ 1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096,
237
+ 16192
238
+ ],
239
labelnames=labelnames).labels(*labelvalues)
240
241
self.histogram_max_num_generation_tokens_request = \
@@ -467,16 +470,6 @@ def build_1_2_5_buckets(max_value: int) -> list[int]:
470
return build_buckets([1, 2, 5], max_value)
471
472
470
-def build_cudagraph_buckets(vllm_config: VllmConfig) -> list[int]:
471
- if not vllm_config.model_config.enforce_eager:
472
- buckets = vllm_config.compilation_config.\
473
- cudagraph_capture_sizes.copy()
474
- buckets.sort()
475
- return buckets
476
- else:
477
- return [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
478
-
479
-
473
def setup_default_loggers(
474
vllm_config: VllmConfig,
475
log_stats: bool,