A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Misc] Add metrics for request queue time, forward time, and execute time (#9659)
科英 committed
Oct 30, 2024 at 01:32 UTC
74fc2d77aec13304550bb52b459bd8c6da756d39
4 files changed
+70
-15
vllm/config.py
-7
@@ -1892,13 +1892,6 @@ class ObservabilityConfig:
1892
"'otlp_traces_endpoint'. Ensure OpenTelemetry packages are "
1893
f"installed. Original error:\n{otel_import_error_traceback}")
1894
1895
- if ((self.collect_model_forward_time
1896
- or self.collect_model_execute_time)
1897
- and self.otlp_traces_endpoint is None):
1898
- raise ValueError(
1899
- "collect_model_forward_time or collect_model_execute_time "
1900
- "requires --otlp-traces-endpoint to be set.")
1901
-
1895
1896
@dataclass(frozen=True)
1897
class EngineConfig:
vllm/engine/llm_engine.py
+15
@@ -1645,6 +1645,9 @@ class LLMEngine:
1645
# Request stats
1646
# Latency
1647
time_e2e_requests: List[float] = []
1648
+ time_in_queue_requests: List[float] = []
1649
+ model_forward_time_requests: List[float] = []
1650
+ model_execute_time_requests: List[float] = []
1651
# Metadata
1652
num_prompt_tokens_requests: List[int] = []
1653
num_generation_tokens_requests: List[int] = []
@@ -1738,6 +1741,15 @@ class LLMEngine:
1741
# Latency timings
1742
time_e2e_requests.append(now -
1743
seq_group.metrics.arrival_time)
1744
+ if seq_group.metrics.time_in_queue is not None:
1745
+ time_in_queue_requests.append(
1746
+ seq_group.metrics.time_in_queue)
1747
+ if seq_group.metrics.model_forward_time is not None:
1748
+ model_forward_time_requests.append(
1749
+ seq_group.metrics.model_forward_time)
1750
+ if seq_group.metrics.model_execute_time is not None:
1751
+ model_execute_time_requests.append(
1752
+ seq_group.metrics.model_execute_time * 1000)
1753
# Metadata
1754
num_prompt_tokens_requests.append(
1755
len(seq_group.prompt_token_ids))
@@ -1795,6 +1807,9 @@ class LLMEngine:
1807
# Request stats
1808
# Latency
1809
time_e2e_requests=time_e2e_requests,
1810
+ time_in_queue_requests=time_in_queue_requests,
1811
+ model_forward_time_requests=model_forward_time_requests,
1812
+ model_execute_time_requests=model_execute_time_requests,
1813
# Metadata
1814
num_prompt_tokens_requests=num_prompt_tokens_requests,
1815
num_generation_tokens_requests=num_generation_tokens_requests,
vllm/engine/metrics.py
+52
-8
@@ -133,7 +133,31 @@ class Metrics:
133
name="vllm:e2e_request_latency_seconds",
134
documentation="Histogram of end to end request latency in seconds.",
135
labelnames=labelnames,
136
- buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
136
+ buckets=[
137
+ 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
138
+ 40.0, 50.0, 60.0
139
+ ])
140
+ self.histogram_time_in_queue_request = self._histogram_cls(
141
+ name="vllm:time_in_queue_requests",
142
+ documentation=
143
+ "Histogram of time the request spent in the queue in seconds.",
144
+ labelnames=labelnames,
145
+ buckets=[
146
+ 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
147
+ 40.0, 50.0, 60.0
148
+ ])
149
+ self.histogram_model_forward_time_request = self._histogram_cls(
150
+ name="vllm:model_forward_time_milliseconds",
151
+ documentation=
152
+ "Histogram of time spent in the model forward pass in ms.",
153
+ labelnames=labelnames,
154
+ buckets=build_1_2_3_5_8_buckets(3000))
155
+ self.histogram_model_execute_time_request = self._histogram_cls(
156
+ name="vllm:model_execute_time_milliseconds",
157
+ documentation=
158
+ "Histogram of time spent in the model execute function in ms.",
159
+ labelnames=labelnames,
160
+ buckets=build_1_2_3_5_8_buckets(3000))
161
# Metadata
162
self.histogram_num_prompt_tokens_request = self._histogram_cls(
163
name="vllm:request_prompt_tokens",
@@ -299,16 +323,12 @@ class RayMetrics(Metrics):
323
pass
324
325
302
-def build_1_2_5_buckets(max_value: int) -> List[int]:
326
+def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
327
"""
304
- Builds a list of buckets with increasing powers of 10 multiplied by
305
- mantissa values (1, 2, 5) until the value exceeds the specified maximum.
328
+ Builds a list of buckets with increasing powers of 10 multiplied by
329
+ mantissa values until the value exceeds the specified maximum.
330
307
- Example:
308
- >>> build_1_2_5_buckets(100)
309
- [1, 2, 5, 10, 20, 50, 100]
331
"""
311
- mantissa_lst = [1, 2, 5]
332
exponent = 0
333
buckets: List[int] = []
334
while True:
@@ -321,6 +341,24 @@ def build_1_2_5_buckets(max_value: int) -> List[int]:
341
exponent += 1
342
343
344
+def build_1_2_5_buckets(max_value: int) -> List[int]:
345
+ """
346
+ Example:
347
+ >>> build_1_2_5_buckets(100)
348
+ [1, 2, 5, 10, 20, 50, 100]
349
+ """
350
+ return build_buckets([1, 2, 5], max_value)
351
+
352
+
353
+def build_1_2_3_5_8_buckets(max_value: int) -> List[int]:
354
+ """
355
+ Example:
356
+ >>> build_1_2_3_5_8_buckets(100)
357
+ [1, 2, 3, 5, 8, 10, 20, 30, 50, 80, 100]
358
+ """
359
+ return build_buckets([1, 2, 3, 5, 8], max_value)
360
+
361
+
362
def local_interval_elapsed(now: float, last_log: float,
363
local_interval: float) -> bool:
364
elapsed_time = now - last_log
@@ -486,6 +524,12 @@ class PrometheusStatLogger(StatLoggerBase):
524
# Latency
525
self._log_histogram(self.metrics.histogram_e2e_time_request,
526
stats.time_e2e_requests)
527
+ self._log_histogram(self.metrics.histogram_time_in_queue_request,
528
+ stats.time_in_queue_requests)
529
+ self._log_histogram(self.metrics.histogram_model_forward_time_request,
530
+ stats.model_forward_time_requests)
531
+ self._log_histogram(self.metrics.histogram_model_execute_time_request,
532
+ stats.model_execute_time_requests)
533
# Metadata
534
finished_reason_counter = CollectionsCounter(
535
stats.finished_reason_requests)
vllm/engine/metrics_types.py
+3
@@ -46,6 +46,9 @@ class Stats:
46
# Request stats (should have _requests suffix)
47
# Latency
48
time_e2e_requests: List[float]
49
+ time_in_queue_requests: List[float]
50
+ model_forward_time_requests: List[float]
51
+ model_execute_time_requests: List[float]
52
# Metadata
53
num_prompt_tokens_requests: List[int]
54
num_generation_tokens_requests: List[int]