@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 74fc2d77ae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Misc] Add metrics for request queue time, forward time, and execute time (#9659)

科英 committed Oct 30, 2024 at 01:32 UTC 74fc2d77aec13304550bb52b459bd8c6da756d39
4 files changed +70 -15
vllm/config.py
-7
@@ -1892,13 +1892,6 @@ class ObservabilityConfig:
1892 "'otlp_traces_endpoint'. Ensure OpenTelemetry packages are "
1893 f"installed. Original error:\n{otel_import_error_traceback}")
1894
1895 - if ((self.collect_model_forward_time
1896 - or self.collect_model_execute_time)
1897 - and self.otlp_traces_endpoint is None):
1898 - raise ValueError(
1899 - "collect_model_forward_time or collect_model_execute_time "
1900 - "requires --otlp-traces-endpoint to be set.")
1901 -
1895
1896 @dataclass(frozen=True)
1897 class EngineConfig:
vllm/engine/llm_engine.py
+15
@@ -1645,6 +1645,9 @@ class LLMEngine:
1645 # Request stats
1646 # Latency
1647 time_e2e_requests: List[float] = []
1648 + time_in_queue_requests: List[float] = []
1649 + model_forward_time_requests: List[float] = []
1650 + model_execute_time_requests: List[float] = []
1651 # Metadata
1652 num_prompt_tokens_requests: List[int] = []
1653 num_generation_tokens_requests: List[int] = []
@@ -1738,6 +1741,15 @@ class LLMEngine:
1741 # Latency timings
1742 time_e2e_requests.append(now -
1743 seq_group.metrics.arrival_time)
1744 + if seq_group.metrics.time_in_queue is not None:
1745 + time_in_queue_requests.append(
1746 + seq_group.metrics.time_in_queue)
1747 + if seq_group.metrics.model_forward_time is not None:
1748 + model_forward_time_requests.append(
1749 + seq_group.metrics.model_forward_time)
1750 + if seq_group.metrics.model_execute_time is not None:
1751 + model_execute_time_requests.append(
1752 + seq_group.metrics.model_execute_time * 1000)
1753 # Metadata
1754 num_prompt_tokens_requests.append(
1755 len(seq_group.prompt_token_ids))
@@ -1795,6 +1807,9 @@ class LLMEngine:
1807 # Request stats
1808 # Latency
1809 time_e2e_requests=time_e2e_requests,
1810 + time_in_queue_requests=time_in_queue_requests,
1811 + model_forward_time_requests=model_forward_time_requests,
1812 + model_execute_time_requests=model_execute_time_requests,
1813 # Metadata
1814 num_prompt_tokens_requests=num_prompt_tokens_requests,
1815 num_generation_tokens_requests=num_generation_tokens_requests,
vllm/engine/metrics.py
+52 -8
@@ -133,7 +133,31 @@ class Metrics:
133 name="vllm:e2e_request_latency_seconds",
134 documentation="Histogram of end to end request latency in seconds.",
135 labelnames=labelnames,
136 - buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
136 + buckets=[
137 + 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
138 + 40.0, 50.0, 60.0
139 + ])
140 + self.histogram_time_in_queue_request = self._histogram_cls(
141 + name="vllm:time_in_queue_requests",
142 + documentation=
143 + "Histogram of time the request spent in the queue in seconds.",
144 + labelnames=labelnames,
145 + buckets=[
146 + 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
147 + 40.0, 50.0, 60.0
148 + ])
149 + self.histogram_model_forward_time_request = self._histogram_cls(
150 + name="vllm:model_forward_time_milliseconds",
151 + documentation=
152 + "Histogram of time spent in the model forward pass in ms.",
153 + labelnames=labelnames,
154 + buckets=build_1_2_3_5_8_buckets(3000))
155 + self.histogram_model_execute_time_request = self._histogram_cls(
156 + name="vllm:model_execute_time_milliseconds",
157 + documentation=
158 + "Histogram of time spent in the model execute function in ms.",
159 + labelnames=labelnames,
160 + buckets=build_1_2_3_5_8_buckets(3000))
161 # Metadata
162 self.histogram_num_prompt_tokens_request = self._histogram_cls(
163 name="vllm:request_prompt_tokens",
@@ -299,16 +323,12 @@ class RayMetrics(Metrics):
323 pass
324
325
302 -def build_1_2_5_buckets(max_value: int) -> List[int]:
326 +def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
327 """
304 - Builds a list of buckets with increasing powers of 10 multiplied by
305 - mantissa values (1, 2, 5) until the value exceeds the specified maximum.
328 + Builds a list of buckets with increasing powers of 10 multiplied by
329 + mantissa values until the value exceeds the specified maximum.
330
307 - Example:
308 - >>> build_1_2_5_buckets(100)
309 - [1, 2, 5, 10, 20, 50, 100]
331 """
311 - mantissa_lst = [1, 2, 5]
332 exponent = 0
333 buckets: List[int] = []
334 while True:
@@ -321,6 +341,24 @@ def build_1_2_5_buckets(max_value: int) -> List[int]:
341 exponent += 1
342
343
344 +def build_1_2_5_buckets(max_value: int) -> List[int]:
345 + """
346 + Example:
347 + >>> build_1_2_5_buckets(100)
348 + [1, 2, 5, 10, 20, 50, 100]
349 + """
350 + return build_buckets([1, 2, 5], max_value)
351 +
352 +
353 +def build_1_2_3_5_8_buckets(max_value: int) -> List[int]:
354 + """
355 + Example:
356 + >>> build_1_2_3_5_8_buckets(100)
357 + [1, 2, 3, 5, 8, 10, 20, 30, 50, 80, 100]
358 + """
359 + return build_buckets([1, 2, 3, 5, 8], max_value)
360 +
361 +
362 def local_interval_elapsed(now: float, last_log: float,
363 local_interval: float) -> bool:
364 elapsed_time = now - last_log
@@ -486,6 +524,12 @@ class PrometheusStatLogger(StatLoggerBase):
524 # Latency
525 self._log_histogram(self.metrics.histogram_e2e_time_request,
526 stats.time_e2e_requests)
527 + self._log_histogram(self.metrics.histogram_time_in_queue_request,
528 + stats.time_in_queue_requests)
529 + self._log_histogram(self.metrics.histogram_model_forward_time_request,
530 + stats.model_forward_time_requests)
531 + self._log_histogram(self.metrics.histogram_model_execute_time_request,
532 + stats.model_execute_time_requests)
533 # Metadata
534 finished_reason_counter = CollectionsCounter(
535 stats.finished_reason_requests)
vllm/engine/metrics_types.py
+3
@@ -46,6 +46,9 @@ class Stats:
46 # Request stats (should have _requests suffix)
47 # Latency
48 time_e2e_requests: List[float]
49 + time_in_queue_requests: List[float]
50 + model_forward_time_requests: List[float]
51 + model_execute_time_requests: List[float]
52 # Metadata
53 num_prompt_tokens_requests: List[int]
54 num_generation_tokens_requests: List[int]