A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[ Bugfix ] Fix Prometheus Metrics With `zeromq` Frontend (#7279)
Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Robert Shaw committed
Aug 18, 2024 at 16:19 UTC
e3b318216d13225221ffbf03cc815648104e37c5
7 files changed
+366
-116
tests/entrypoints/openai/test_basic.py
-9
@@ -50,12 +50,3 @@ async def test_check_health(client: openai.AsyncOpenAI):
50
response = requests.get(base_url + "/health")
51
52
assert response.status_code == HTTPStatus.OK
53
-
54
-
55
-@pytest.mark.asyncio
56
-async def test_log_metrics(client: openai.AsyncOpenAI):
57
- base_url = str(client.base_url)[:-3].strip("/")
58
-
59
- response = requests.get(base_url + "/metrics")
60
-
61
- assert response.status_code == HTTPStatus.OK
tests/entrypoints/openai/test_metrics.py
new
+179
@@ -0,0 +1,179 @@
1
+from http import HTTPStatus
2
+
3
+import openai
4
+import pytest
5
+import requests
6
+from prometheus_client.parser import text_string_to_metric_families
7
+from transformers import AutoTokenizer
8
+
9
+from ...utils import RemoteOpenAIServer
10
+
11
+MODEL_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
12
+
13
+
14
+@pytest.fixture(scope="module")
15
+def default_server_args():
16
+ return [
17
+ # use half precision for speed and memory savings in CI environment
18
+ "--dtype",
19
+ "bfloat16",
20
+ "--max-model-len",
21
+ "1024",
22
+ "--enforce-eager",
23
+ "--max-num-seqs",
24
+ "128",
25
+ ]
26
+
27
+
28
+@pytest.fixture(scope="module",
29
+ params=[
30
+ "",
31
+ "--enable-chunked-prefill",
32
+ "--disable-frontend-multiprocessing",
33
+ ])
34
+def client(default_server_args, request):
35
+ if request.param:
36
+ default_server_args.append(request.param)
37
+ with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
38
+ yield remote_server.get_async_client()
39
+
40
+
41
+_PROMPT = "Hello my name is Robert and I love magic"
42
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
43
+_TOKENIZED_PROMPT = tokenizer(_PROMPT)["input_ids"]
44
+
45
+_NUM_REQUESTS = 10
46
+_NUM_PROMPT_TOKENS_PER_REQUEST = len(_TOKENIZED_PROMPT)
47
+_NUM_GENERATION_TOKENS_PER_REQUEST = 10
48
+
49
+# {metric_family: [(suffix, expected_value)]}
50
+EXPECTED_VALUES = {
51
+ "vllm:time_to_first_token_seconds": [("_count", _NUM_REQUESTS)],
52
+ "vllm:time_per_output_token_seconds":
53
+ [("_count", _NUM_REQUESTS * (_NUM_GENERATION_TOKENS_PER_REQUEST - 1))],
54
+ "vllm:e2e_request_latency_seconds": [("_count", _NUM_REQUESTS)],
55
+ "vllm:request_prompt_tokens":
56
+ [("_sum", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST),
57
+ ("_count", _NUM_REQUESTS)],
58
+ "vllm:request_generation_tokens":
59
+ [("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
60
+ ("_count", _NUM_REQUESTS)],
61
+ "vllm:request_params_n": [("_count", _NUM_REQUESTS)],
62
+ "vllm:request_params_best_of": [("_count", _NUM_REQUESTS)],
63
+ "vllm:prompt_tokens": [("_total",
64
+ _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
65
+ "vllm:generation_tokens":
66
+ [("_total", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
67
+ "vllm:request_success": [("_total", _NUM_REQUESTS)],
68
+}
69
+
70
+
71
+@pytest.mark.asyncio
72
+async def test_metrics_counts(client: openai.AsyncOpenAI):
73
+ base_url = str(client.base_url)[:-3].strip("/")
74
+
75
+ for _ in range(_NUM_REQUESTS):
76
+ # sending a request triggers the metrics to be logged.
77
+ await client.completions.create(
78
+ model=MODEL_NAME,
79
+ prompt=_TOKENIZED_PROMPT,
80
+ max_tokens=_NUM_GENERATION_TOKENS_PER_REQUEST)
81
+
82
+ response = requests.get(base_url + "/metrics")
83
+ print(response.text)
84
+ assert response.status_code == HTTPStatus.OK
85
+
86
+ # Loop over all expected metric_families
87
+ for metric_family, suffix_values_list in EXPECTED_VALUES.items():
88
+ found_metric = False
89
+
90
+ # Check to see if the metric_family is found in the prom endpoint.
91
+ for family in text_string_to_metric_families(response.text):
92
+ if family.name == metric_family:
93
+ found_metric = True
94
+
95
+ # Check that each suffix is found in the prom endpoint.
96
+ for suffix, expected_value in suffix_values_list:
97
+ metric_name_w_suffix = f"{metric_family}{suffix}"
98
+ found_suffix = False
99
+
100
+ for sample in family.samples:
101
+ if sample.name == metric_name_w_suffix:
102
+ found_suffix = True
103
+
104
+ # For each suffix, value sure the value matches
105
+ # what we expect.
106
+ assert sample.value == expected_value, (
107
+ f"{metric_name_w_suffix} expected value of "
108
+ f"{expected_value} did not match found value "
109
+ f"{sample.value}")
110
+ break
111
+ assert found_suffix, (
112
+ f"Did not find {metric_name_w_suffix} in prom endpoint"
113
+ )
114
+ break
115
+
116
+ assert found_metric, (f"Did not find {metric_family} in prom endpoint")
117
+
118
+
119
+EXPECTED_METRICS = [
120
+ "vllm:num_requests_running",
121
+ "vllm:num_requests_swapped",
122
+ "vllm:num_requests_waiting",
123
+ "vllm:gpu_cache_usage_perc",
124
+ "vllm:cpu_cache_usage_perc",
125
+ "vllm:time_to_first_token_seconds_sum",
126
+ "vllm:time_to_first_token_seconds_bucket",
127
+ "vllm:time_to_first_token_seconds_count",
128
+ "vllm:time_per_output_token_seconds_sum",
129
+ "vllm:time_per_output_token_seconds_bucket",
130
+ "vllm:time_per_output_token_seconds_count",
131
+ "vllm:e2e_request_latency_seconds_sum",
132
+ "vllm:e2e_request_latency_seconds_bucket",
133
+ "vllm:e2e_request_latency_seconds_count",
134
+ "vllm:request_prompt_tokens_sum",
135
+ "vllm:request_prompt_tokens_bucket",
136
+ "vllm:request_prompt_tokens_count",
137
+ "vllm:request_generation_tokens_sum",
138
+ "vllm:request_generation_tokens_bucket",
139
+ "vllm:request_generation_tokens_count",
140
+ "vllm:request_params_n_sum",
141
+ "vllm:request_params_n_bucket",
142
+ "vllm:request_params_n_count",
143
+ "vllm:request_params_best_of_sum",
144
+ "vllm:request_params_best_of_bucket",
145
+ "vllm:request_params_best_of_count",
146
+ "vllm:num_preemptions_total",
147
+ "vllm:prompt_tokens_total",
148
+ "vllm:generation_tokens_total",
149
+ "vllm:request_success_total",
150
+ "vllm:cache_config_info",
151
+ # labels in cache_config_info
152
+ "block_size",
153
+ "cache_dtype",
154
+ "cpu_offload_gb",
155
+ "enable_prefix_caching",
156
+ "gpu_memory_utilization",
157
+ "num_cpu_blocks",
158
+ "num_gpu_blocks",
159
+ "num_gpu_blocks_override",
160
+ "sliding_window",
161
+ "swap_space_bytes",
162
+]
163
+
164
+
165
+@pytest.mark.asyncio
166
+async def test_metrics_exist(client: openai.AsyncOpenAI):
167
+ base_url = str(client.base_url)[:-3].strip("/")
168
+
169
+ # sending a request triggers the metrics to be logged.
170
+ await client.completions.create(model=MODEL_NAME,
171
+ prompt="Hello, my name is",
172
+ max_tokens=5,
173
+ temperature=0.0)
174
+
175
+ response = requests.get(base_url + "/metrics")
176
+ assert response.status_code == HTTPStatus.OK
177
+
178
+ for metric in EXPECTED_METRICS:
179
+ assert metric in response.text
vllm/engine/async_llm_engine.py
+1
-1
@@ -15,7 +15,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
15
from vllm.engine.async_timeout import asyncio_timeout
16
from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
17
PromptComponents)
18
-from vllm.engine.metrics import StatLoggerBase
18
+from vllm.engine.metrics_types import StatLoggerBase
19
from vllm.executor.executor_base import ExecutorAsyncBase
20
from vllm.executor.ray_utils import initialize_ray_cluster, ray
21
from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
vllm/engine/llm_engine.py
+8
-2
@@ -16,8 +16,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
16
from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
17
SchedulerOutputs)
18
from vllm.engine.arg_utils import EngineArgs
19
-from vllm.engine.metrics import (LoggingStatLogger, PrometheusStatLogger,
20
- StatLoggerBase, Stats)
19
+from vllm.engine.metrics_types import StatLoggerBase, Stats
20
from vllm.engine.output_processor.interfaces import (
21
SequenceGroupOutputProcessor)
22
from vllm.engine.output_processor.stop_checker import StopChecker
@@ -339,6 +338,13 @@ class LLMEngine:
338
if stat_loggers is not None:
339
self.stat_loggers = stat_loggers
340
else:
341
+ # Lazy import for prometheus multiprocessing.
342
+ # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
343
+ # before prometheus_client is imported.
344
+ # See https://prometheus.github.io/client_python/multiprocess/
345
+ from vllm.engine.metrics import (LoggingStatLogger,
346
+ PrometheusStatLogger)
347
+
348
self.stat_loggers = {
349
"logging":
350
LoggingStatLogger(
vllm/engine/metrics.py
+48
-101
@@ -1,13 +1,12 @@
1
-import time
2
-from abc import ABC, abstractmethod
3
-from dataclasses import dataclass
1
from typing import TYPE_CHECKING
2
from typing import Counter as CollectionsCounter
6
-from typing import Dict, List, Optional, Protocol, Union
3
+from typing import Dict, List, Optional, Union
4
5
import numpy as np
6
import prometheus_client
7
8
+from vllm.engine.metrics_types import (StatLoggerBase, Stats,
9
+ SupportsMetricsInfo)
10
from vllm.executor.ray_utils import ray
11
from vllm.logger import init_logger
12
@@ -29,41 +28,49 @@ prometheus_client.disable_created_metrics()
28
29
# begin-metrics-definitions
30
class Metrics:
31
+ """
32
+ vLLM uses a multiprocessing-based frontend for the OpenAI server.
33
+ This means that we need to run prometheus_client in multiprocessing mode
34
+ See https://prometheus.github.io/client_python/multiprocess/ for more
35
+ details on limitations.
36
+ """
37
labelname_finish_reason = "finished_reason"
38
_gauge_cls = prometheus_client.Gauge
39
_counter_cls = prometheus_client.Counter
40
_histogram_cls = prometheus_client.Histogram
41
42
def __init__(self, labelnames: List[str], max_model_len: int):
38
- # Unregister any existing vLLM collectors
43
+ # Unregister any existing vLLM collectors (for CI/CD)
44
self._unregister_vllm_metrics()
45
41
- # Config Information
42
- self._create_info_cache_config()
43
-
46
# System stats
47
# Scheduler State
48
self.gauge_scheduler_running = self._gauge_cls(
49
name="vllm:num_requests_running",
50
documentation="Number of requests currently running on GPU.",
49
- labelnames=labelnames)
51
+ labelnames=labelnames,
52
+ multiprocess_mode="sum")
53
self.gauge_scheduler_waiting = self._gauge_cls(
54
name="vllm:num_requests_waiting",
55
documentation="Number of requests waiting to be processed.",
53
- labelnames=labelnames)
56
+ labelnames=labelnames,
57
+ multiprocess_mode="sum")
58
self.gauge_scheduler_swapped = self._gauge_cls(
59
name="vllm:num_requests_swapped",
60
documentation="Number of requests swapped to CPU.",
57
- labelnames=labelnames)
61
+ labelnames=labelnames,
62
+ multiprocess_mode="sum")
63
# KV Cache Usage in %
64
self.gauge_gpu_cache_usage = self._gauge_cls(
65
name="vllm:gpu_cache_usage_perc",
66
documentation="GPU KV-cache usage. 1 means 100 percent usage.",
62
- labelnames=labelnames)
67
+ labelnames=labelnames,
68
+ multiprocess_mode="sum")
69
self.gauge_cpu_cache_usage = self._gauge_cls(
70
name="vllm:cpu_cache_usage_perc",
71
documentation="CPU KV-cache usage. 1 means 100 percent usage.",
66
- labelnames=labelnames)
72
+ labelnames=labelnames,
73
+ multiprocess_mode="sum")
74
75
# Iteration stats
76
self.counter_num_preemption = self._counter_cls(
@@ -137,11 +144,13 @@ class Metrics:
144
self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
145
name="vllm:spec_decode_draft_acceptance_rate",
146
documentation="Speulative token acceptance rate.",
140
- labelnames=labelnames)
147
+ labelnames=labelnames,
148
+ multiprocess_mode="sum")
149
self.gauge_spec_decode_efficiency = self._gauge_cls(
150
name="vllm:spec_decode_efficiency",
151
documentation="Speculative decoding system efficiency.",
144
- labelnames=labelnames)
152
+ labelnames=labelnames,
153
+ multiprocess_mode="sum")
154
self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
155
name="vllm:spec_decode_num_accepted_tokens_total",
156
documentation="Number of accepted tokens.",
@@ -160,19 +169,18 @@ class Metrics:
169
name="vllm:avg_prompt_throughput_toks_per_s",
170
documentation="Average prefill throughput in tokens/s.",
171
labelnames=labelnames,
172
+ multiprocess_mode="sum",
173
)
174
# Deprecated in favor of vllm:generation_tokens_total
175
self.gauge_avg_generation_throughput = self._gauge_cls(
176
name="vllm:avg_generation_throughput_toks_per_s",
177
documentation="Average generation throughput in tokens/s.",
178
labelnames=labelnames,
179
+ multiprocess_mode="sum",
180
)
181
171
- def _create_info_cache_config(self) -> None:
172
- # Config Information
173
- self.info_cache_config = prometheus_client.Info(
174
- name='vllm:cache_config',
175
- documentation='information of cache_config')
182
+
183
+# end-metrics-definitions
184
185
def _unregister_vllm_metrics(self) -> None:
186
for collector in list(prometheus_client.REGISTRY._collector_to_names):
@@ -180,9 +188,6 @@ class Metrics:
188
prometheus_client.REGISTRY.unregister(collector)
189
190
183
-# end-metrics-definitions
184
-
185
-
191
class _RayGaugeWrapper:
192
"""Wraps around ray.util.metrics.Gauge to provide same API as
193
prometheus_client.Gauge"""
@@ -190,7 +195,9 @@ class _RayGaugeWrapper:
195
def __init__(self,
196
name: str,
197
documentation: str = "",
193
- labelnames: Optional[List[str]] = None):
198
+ labelnames: Optional[List[str]] = None,
199
+ multiprocess_mode: str = ""):
200
+ del multiprocess_mode
201
labelnames_tuple = tuple(labelnames) if labelnames else None
202
self._gauge = ray_metrics.Gauge(name=name,
203
description=documentation,
@@ -268,10 +275,6 @@ class RayMetrics(Metrics):
275
# No-op on purpose
276
pass
277
271
- def _create_info_cache_config(self) -> None:
272
- # No-op on purpose
273
- pass
274
-
278
279
def build_1_2_5_buckets(max_value: int) -> List[int]:
280
"""
@@ -295,46 +298,6 @@ def build_1_2_5_buckets(max_value: int) -> List[int]:
298
exponent += 1
299
300
298
-@dataclass
299
-class Stats:
300
- """Created by LLMEngine for use by StatLogger."""
301
- now: float
302
-
303
- # System stats (should have _sys suffix)
304
- # Scheduler State
305
- num_running_sys: int
306
- num_waiting_sys: int
307
- num_swapped_sys: int
308
- # KV Cache Usage in %
309
- gpu_cache_usage_sys: float
310
- cpu_cache_usage_sys: float
311
-
312
- # Iteration stats (should have _iter suffix)
313
- num_prompt_tokens_iter: int
314
- num_generation_tokens_iter: int
315
- time_to_first_tokens_iter: List[float]
316
- time_per_output_tokens_iter: List[float]
317
- num_preemption_iter: int
318
-
319
- # Request stats (should have _requests suffix)
320
- # Latency
321
- time_e2e_requests: List[float]
322
- # Metadata
323
- num_prompt_tokens_requests: List[int]
324
- num_generation_tokens_requests: List[int]
325
- best_of_requests: List[int]
326
- n_requests: List[int]
327
- finished_reason_requests: List[str]
328
-
329
- spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
330
-
331
-
332
-class SupportsMetricsInfo(Protocol):
333
-
334
- def metrics_info(self) -> Dict[str, str]:
335
- ...
336
-
337
-
301
def local_interval_elapsed(now: float, last_log: float,
302
local_interval: float) -> bool:
303
elapsed_time = now - last_log
@@ -346,38 +309,9 @@ def get_throughput(tracked_stats: List[int], now: float,
309
return float(np.sum(tracked_stats) / (now - last_log))
310
311
349
-class StatLoggerBase(ABC):
350
- """Base class for StatLogger."""
351
-
352
- def __init__(self, local_interval: float) -> None:
353
- # Tracked stats over current local logging interval.
354
- self.num_prompt_tokens: List[int] = []
355
- self.num_generation_tokens: List[int] = []
356
- self.last_local_log = time.time()
357
- self.local_interval = local_interval
358
- self.spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
359
-
360
- @abstractmethod
361
- def info(self, type: str, obj: SupportsMetricsInfo) -> None:
362
- raise NotImplementedError
363
-
364
- @abstractmethod
365
- def log(self, stats: Stats) -> None:
366
- raise NotImplementedError
367
-
368
- def maybe_update_spec_decode_metrics(self, stats: Stats):
369
- """Save spec decode metrics (since they are unlikely
370
- to be emitted at same time as log interval)."""
371
- if stats.spec_decode_metrics is not None:
372
- self.spec_decode_metrics = stats.spec_decode_metrics
373
-
374
-
312
class LoggingStatLogger(StatLoggerBase):
313
"""LoggingStatLogger is used in LLMEngine to log to Stdout."""
314
378
- def info(self, type: str, obj: SupportsMetricsInfo) -> None:
379
- raise NotImplementedError
380
-
315
def log(self, stats: Stats) -> None:
316
"""Called by LLMEngine.
317
Logs to Stdout every self.local_interval seconds."""
@@ -440,10 +374,14 @@ class LoggingStatLogger(StatLoggerBase):
374
f"Number of draft tokens: {metrics.draft_tokens}, "
375
f"Number of emitted tokens: {metrics.emitted_tokens}.")
376
377
+ def info(self, type: str, obj: SupportsMetricsInfo) -> None:
378
+ raise NotImplementedError
379
+
380
381
class PrometheusStatLogger(StatLoggerBase):
382
"""PrometheusStatLogger is used LLMEngine to log to Promethus."""
383
_metrics_cls = Metrics
384
+ _gauge_cls = prometheus_client.Gauge
385
386
def __init__(self, local_interval: float, labels: Dict[str, str],
387
max_model_len: int) -> None:
@@ -453,10 +391,6 @@ class PrometheusStatLogger(StatLoggerBase):
391
self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
392
max_model_len=max_model_len)
393
456
- def info(self, type: str, obj: SupportsMetricsInfo) -> None:
457
- if type == "cache_config":
458
- self.metrics.info_cache_config.info(obj.metrics_info())
459
-
394
def _log_gauge(self, gauge, data: Union[int, float]) -> None:
395
# Convenience function for logging to gauge.
396
gauge.labels(**self.labels).set(data)
@@ -586,6 +520,19 @@ class PrometheusStatLogger(StatLoggerBase):
520
self.last_local_log = stats.now
521
self.spec_decode_metrics = None
522
523
+ def info(self, type: str, obj: SupportsMetricsInfo) -> None:
524
+ # Info type metrics are syntactic sugar for a gauge permanently set to 1
525
+ # Since prometheus multiprocessing mode does not support Info, emulate
526
+ # info here with a gauge.
527
+ if type == "cache_config":
528
+ metrics_info = obj.metrics_info()
529
+ info_gauge = self._gauge_cls(
530
+ name="vllm:cache_config_info",
531
+ documentation="Information of the LLMEngine CacheConfig",
532
+ labelnames=metrics_info.keys(),
533
+ multiprocess_mode="mostrecent")
534
+ info_gauge.labels(**metrics_info).set(1)
535
+
536
537
class RayPrometheusStatLogger(PrometheusStatLogger):
538
"""RayPrometheusStatLogger uses Ray metrics instead."""
vllm/engine/metrics_types.py
new
+85
@@ -0,0 +1,85 @@
1
+"""
2
+These types are defined in this file to avoid importing vllm.engine.metrics
3
+and therefore importing prometheus_client.
4
+
5
+This is required due to usage of Prometheus multiprocess mode to enable
6
+metrics after splitting out the uvicorn process from the engine process.
7
+
8
+Prometheus multiprocess mode requires setting PROMETHEUS_MULTIPROC_DIR
9
+before prometheus_client is imported. Typically, this is done by setting
10
+the env variable before launch, but since we are a library, we need to
11
+do this in Python code and lazily import prometheus_client.
12
+"""
13
+
14
+import time
15
+from abc import ABC, abstractmethod
16
+from dataclasses import dataclass
17
+from typing import Dict, List, Optional, Protocol
18
+
19
+from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
20
+
21
+
22
+@dataclass
23
+class Stats:
24
+ """Created by LLMEngine for use by StatLogger."""
25
+ now: float
26
+
27
+ # System stats (should have _sys suffix)
28
+ # Scheduler State
29
+ num_running_sys: int
30
+ num_waiting_sys: int
31
+ num_swapped_sys: int
32
+ # KV Cache Usage in %
33
+ gpu_cache_usage_sys: float
34
+ cpu_cache_usage_sys: float
35
+
36
+ # Iteration stats (should have _iter suffix)
37
+ num_prompt_tokens_iter: int
38
+ num_generation_tokens_iter: int
39
+ time_to_first_tokens_iter: List[float]
40
+ time_per_output_tokens_iter: List[float]
41
+ num_preemption_iter: int
42
+
43
+ # Request stats (should have _requests suffix)
44
+ # Latency
45
+ time_e2e_requests: List[float]
46
+ # Metadata
47
+ num_prompt_tokens_requests: List[int]
48
+ num_generation_tokens_requests: List[int]
49
+ best_of_requests: List[int]
50
+ n_requests: List[int]
51
+ finished_reason_requests: List[str]
52
+
53
+ spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
54
+
55
+
56
+class SupportsMetricsInfo(Protocol):
57
+
58
+ def metrics_info(self) -> Dict[str, str]:
59
+ ...
60
+
61
+
62
+class StatLoggerBase(ABC):
63
+ """Base class for StatLogger."""
64
+
65
+ def __init__(self, local_interval: float) -> None:
66
+ # Tracked stats over current local logging interval.
67
+ self.num_prompt_tokens: List[int] = []
68
+ self.num_generation_tokens: List[int] = []
69
+ self.last_local_log = time.time()
70
+ self.local_interval = local_interval
71
+ self.spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
72
+
73
+ @abstractmethod
74
+ def log(self, stats: Stats) -> None:
75
+ raise NotImplementedError
76
+
77
+ @abstractmethod
78
+ def info(self, type: str, obj: SupportsMetricsInfo) -> None:
79
+ raise NotImplementedError
80
+
81
+ def maybe_update_spec_decode_metrics(self, stats: Stats):
82
+ """Save spec decode metrics (since they are unlikely
83
+ to be emitted at same time as log interval)."""
84
+ if stats.spec_decode_metrics is not None:
85
+ self.spec_decode_metrics = stats.spec_decode_metrics
vllm/entrypoints/openai/api_server.py
+45
-3
@@ -2,7 +2,9 @@ import asyncio
2
import importlib
3
import inspect
4
import multiprocessing
5
+import os
6
import re
7
+import tempfile
8
from argparse import Namespace
9
from contextlib import asynccontextmanager
10
from http import HTTPStatus
@@ -12,7 +14,6 @@ from fastapi import APIRouter, FastAPI, Request
14
from fastapi.exceptions import RequestValidationError
15
from fastapi.middleware.cors import CORSMiddleware
16
from fastapi.responses import JSONResponse, Response, StreamingResponse
15
-from prometheus_client import make_asgi_app
17
from starlette.routing import Mount
18
19
import vllm.envs as envs
@@ -54,6 +55,7 @@ openai_serving_chat: OpenAIServingChat
55
openai_serving_completion: OpenAIServingCompletion
56
openai_serving_embedding: OpenAIServingEmbedding
57
openai_serving_tokenization: OpenAIServingTokenization
58
+prometheus_multiproc_dir: tempfile.TemporaryDirectory
59
60
logger = init_logger('vllm.entrypoints.openai.api_server')
61
@@ -109,6 +111,21 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
111
112
# Otherwise, use the multiprocessing AsyncLLMEngine.
113
else:
114
+ if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
115
+ # Make TemporaryDirectory for prometheus multiprocessing
116
+ # Note: global TemporaryDirectory will be automatically
117
+ # cleaned up upon exit.
118
+ global prometheus_multiproc_dir
119
+ prometheus_multiproc_dir = tempfile.TemporaryDirectory()
120
+ os.environ[
121
+ "PROMETHEUS_MULTIPROC_DIR"] = prometheus_multiproc_dir.name
122
+ else:
123
+ logger.warning(
124
+ "Found PROMETHEUS_MULTIPROC_DIR was set by user. "
125
+ "This directory must be wiped between vLLM runs or "
126
+ "you will find inaccurate metrics. Unset the variable "
127
+ "and vLLM will properly handle cleanup.")
128
+
129
# Select random path for IPC.
130
rpc_path = get_open_zmq_ipc_path()
131
logger.info("Multiprocessing frontend to use %s for RPC Path.",
@@ -149,13 +166,38 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
166
# Wait for server process to join
167
rpc_server_process.join()
168
169
+ # Lazy import for prometheus multiprocessing.
170
+ # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
171
+ # before prometheus_client is imported.
172
+ # See https://prometheus.github.io/client_python/multiprocess/
173
+ from prometheus_client import multiprocess
174
+ multiprocess.mark_process_dead(rpc_server_process.pid)
175
+
176
177
router = APIRouter()
178
179
180
def mount_metrics(app: FastAPI):
157
- # Add prometheus asgi middleware to route /metrics requests
158
- metrics_route = Mount("/metrics", make_asgi_app())
181
+ # Lazy import for prometheus multiprocessing.
182
+ # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
183
+ # before prometheus_client is imported.
184
+ # See https://prometheus.github.io/client_python/multiprocess/
185
+ from prometheus_client import (CollectorRegistry, make_asgi_app,
186
+ multiprocess)
187
+
188
+ prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None)
189
+ if prometheus_multiproc_dir_path is not None:
190
+ logger.info("vLLM to use %s as PROMETHEUS_MULTIPROC_DIR",
191
+ prometheus_multiproc_dir_path)
192
+ registry = CollectorRegistry()
193
+ multiprocess.MultiProcessCollector(registry)
194
+
195
+ # Add prometheus asgi middleware to route /metrics requests
196
+ metrics_route = Mount("/metrics", make_asgi_app(registry=registry))
197
+ else:
198
+ # Add prometheus asgi middleware to route /metrics requests
199
+ metrics_route = Mount("/metrics", make_asgi_app())
200
+
201
# Workaround for 307 Redirect for /metrics
202
metrics_route.path_regex = re.compile('^/metrics(?P<path>.*)$')
203
app.routes.append(metrics_route)