A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add Production Metrics in Prometheus format (#1890)
Simon Mo committed
Dec 2, 2023 at 16:37 UTC
5313c2cb8b3bcf7f71c0e6024c59d120efe94d88
6 files changed
+89
-2
docs/source/index.rst
+1
@@ -67,6 +67,7 @@ Documentation
67
serving/deploying_with_triton
68
serving/deploying_with_docker
69
serving/serving_with_langchain
70
+ serving/metrics
71
72
.. toctree::
73
:maxdepth: 1
docs/source/serving/metrics.rst
new
+13
@@ -0,0 +1,13 @@
1
+Production Metrics
2
+==================
3
+
4
+vLLM exposes a number of metrics that can be used to monitor the health of the
5
+system. These metrics are exposed via the `/metrics` endpoint on the vLLM
6
+OpenAI compatible API server.
7
+
8
+The following metrics are exposed:
9
+
10
+.. literalinclude:: ../../../vllm/engine/metrics.py
11
+ :language: python
12
+ :start-after: begin-metrics-definitions
13
+ :end-before: end-metrics-definitions
requirements.txt
+1
@@ -12,3 +12,4 @@ xformers >= 0.0.22.post7 # Required for CUDA 12.1.
12
fastapi
13
uvicorn[standard]
14
pydantic == 1.10.13 # Required for OpenAI server.
15
+aioprometheus[starlette]
vllm/engine/llm_engine.py
+13
-2
@@ -7,6 +7,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7
SchedulerConfig)
8
from vllm.core.scheduler import Scheduler, SchedulerOutputs
9
from vllm.engine.arg_utils import EngineArgs
10
+from vllm.engine.metrics import record_metrics
11
from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
12
from vllm.logger import init_logger
13
from vllm.outputs import RequestOutput
@@ -591,8 +592,8 @@ class LLMEngine:
592
else:
593
self.num_generation_tokens.append((now, num_batched_tokens))
594
594
- elapsed_time = now - self.last_logging_time
595
- if elapsed_time < _LOGGING_INTERVAL_SEC:
595
+ should_log = now - self.last_logging_time >= _LOGGING_INTERVAL_SEC
596
+ if not should_log:
597
return
598
599
# Discard the old stats.
@@ -631,6 +632,16 @@ class LLMEngine:
632
else:
633
cpu_cache_usage = 0.0
634
635
+ record_metrics(
636
+ avg_prompt_throughput=avg_prompt_throughput,
637
+ avg_generation_throughput=avg_generation_throughput,
638
+ scheduler_running=len(self.scheduler.running),
639
+ scheduler_swapped=len(self.scheduler.swapped),
640
+ scheduler_waiting=len(self.scheduler.waiting),
641
+ gpu_cache_usage=gpu_cache_usage,
642
+ cpu_cache_usage=cpu_cache_usage,
643
+ )
644
+
645
logger.info("Avg prompt throughput: "
646
f"{avg_prompt_throughput:.1f} tokens/s, "
647
"Avg generation throughput: "
vllm/engine/metrics.py
new
+51
@@ -0,0 +1,51 @@
1
+from aioprometheus import Gauge
2
+
3
+# The begin-* and end* here are used by the documentation generator
4
+# to extract the metrics definitions.
5
+
6
+# begin-metrics-definitions
7
+gauge_avg_prompt_throughput = Gauge("vllm:avg_prompt_throughput_toks_per_s",
8
+ "Average prefill throughput in tokens/s.")
9
+gauge_avg_generation_throughput = Gauge(
10
+ "vllm:avg_generation_throughput_toks_per_s",
11
+ "Average generation throughput in tokens/s.")
12
+
13
+gauge_scheduler_running = Gauge(
14
+ "vllm:num_requests_running",
15
+ "Number of requests that is currently running for inference.")
16
+gauge_scheduler_swapped = Gauge("vllm:num_requests_swapped",
17
+ "Number requests swapped to CPU.")
18
+gauge_scheduler_waiting = Gauge("vllm:num_requests_waiting",
19
+ "Number of requests waiting to be processed.")
20
+
21
+gauge_gpu_cache_usage = Gauge(
22
+ "vllm:gpu_cache_usage_perc",
23
+ "GPU KV-cache usage. 1 means 100 percent usage.")
24
+gauge_cpu_cache_usage = Gauge(
25
+ "vllm:cpu_cache_usage_perc",
26
+ "CPU KV-cache usage. 1 means 100 percent usage.")
27
+# end-metrics-definitions
28
+
29
+labels = {}
30
+
31
+
32
+def add_global_metrics_labels(**kwargs):
33
+ labels.update(kwargs)
34
+
35
+
36
+def record_metrics(
37
+ avg_prompt_throughput: float,
38
+ avg_generation_throughput: float,
39
+ scheduler_running: int,
40
+ scheduler_swapped: int,
41
+ scheduler_waiting: int,
42
+ gpu_cache_usage: float,
43
+ cpu_cache_usage: float,
44
+):
45
+ gauge_avg_prompt_throughput.set(labels, avg_prompt_throughput)
46
+ gauge_avg_generation_throughput.set(labels, avg_generation_throughput)
47
+ gauge_scheduler_running.set(labels, scheduler_running)
48
+ gauge_scheduler_swapped.set(labels, scheduler_swapped)
49
+ gauge_scheduler_waiting.set(labels, scheduler_waiting)
50
+ gauge_gpu_cache_usage.set(labels, gpu_cache_usage)
51
+ gauge_cpu_cache_usage.set(labels, cpu_cache_usage)
vllm/entrypoints/openai/api_server.py
+10
@@ -9,6 +9,8 @@ import time
9
from http import HTTPStatus
10
from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union
11
12
+from aioprometheus import MetricsMiddleware
13
+from aioprometheus.asgi.starlette import metrics
14
import fastapi
15
import uvicorn
16
from fastapi import Request
@@ -18,6 +20,7 @@ from fastapi.responses import JSONResponse, StreamingResponse, Response
20
21
from vllm.engine.arg_utils import AsyncEngineArgs
22
from vllm.engine.async_llm_engine import AsyncLLMEngine
23
+from vllm.engine.metrics import add_global_metrics_labels
24
from vllm.entrypoints.openai.protocol import (
25
CompletionRequest, CompletionResponse, CompletionResponseChoice,
26
CompletionResponseStreamChoice, CompletionStreamResponse,
@@ -82,6 +85,10 @@ def parse_args():
85
return parser.parse_args()
86
87
88
+app.add_middleware(MetricsMiddleware) # Trace HTTP server metrics
89
+app.add_route("/metrics", metrics) # Exposes HTTP metrics
90
+
91
+
92
def create_error_response(status_code: HTTPStatus,
93
message: str) -> JSONResponse:
94
return JSONResponse(ErrorResponse(message=message,
@@ -722,6 +729,9 @@ if __name__ == "__main__":
729
trust_remote_code=engine_model_config.trust_remote_code)
730
load_chat_template(args, tokenizer)
731
732
+ # Register labels for metrics
733
+ add_global_metrics_labels(model_name=engine_args.model)
734
+
735
uvicorn.run(app,
736
host=args.host,
737
port=args.port,