@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e3b318216d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[ Bugfix ] Fix Prometheus Metrics With `zeromq` Frontend (#7279)

Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Robert Shaw committed Aug 18, 2024 at 16:19 UTC e3b318216d13225221ffbf03cc815648104e37c5
7 files changed +366 -116
tests/entrypoints/openai/test_basic.py
-9
@@ -50,12 +50,3 @@ async def test_check_health(client: openai.AsyncOpenAI):
50 response = requests.get(base_url + "/health")
51
52 assert response.status_code == HTTPStatus.OK
53 -
54 -
55 -@pytest.mark.asyncio
56 -async def test_log_metrics(client: openai.AsyncOpenAI):
57 - base_url = str(client.base_url)[:-3].strip("/")
58 -
59 - response = requests.get(base_url + "/metrics")
60 -
61 - assert response.status_code == HTTPStatus.OK
tests/entrypoints/openai/test_metrics.py new
+179
@@ -0,0 +1,179 @@
1 +from http import HTTPStatus
2 +
3 +import openai
4 +import pytest
5 +import requests
6 +from prometheus_client.parser import text_string_to_metric_families
7 +from transformers import AutoTokenizer
8 +
9 +from ...utils import RemoteOpenAIServer
10 +
11 +MODEL_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
12 +
13 +
14 +@pytest.fixture(scope="module")
15 +def default_server_args():
16 + return [
17 + # use half precision for speed and memory savings in CI environment
18 + "--dtype",
19 + "bfloat16",
20 + "--max-model-len",
21 + "1024",
22 + "--enforce-eager",
23 + "--max-num-seqs",
24 + "128",
25 + ]
26 +
27 +
28 +@pytest.fixture(scope="module",
29 + params=[
30 + "",
31 + "--enable-chunked-prefill",
32 + "--disable-frontend-multiprocessing",
33 + ])
34 +def client(default_server_args, request):
35 + if request.param:
36 + default_server_args.append(request.param)
37 + with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
38 + yield remote_server.get_async_client()
39 +
40 +
41 +_PROMPT = "Hello my name is Robert and I love magic"
42 +tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
43 +_TOKENIZED_PROMPT = tokenizer(_PROMPT)["input_ids"]
44 +
45 +_NUM_REQUESTS = 10
46 +_NUM_PROMPT_TOKENS_PER_REQUEST = len(_TOKENIZED_PROMPT)
47 +_NUM_GENERATION_TOKENS_PER_REQUEST = 10
48 +
49 +# {metric_family: [(suffix, expected_value)]}
50 +EXPECTED_VALUES = {
51 + "vllm:time_to_first_token_seconds": [("_count", _NUM_REQUESTS)],
52 + "vllm:time_per_output_token_seconds":
53 + [("_count", _NUM_REQUESTS * (_NUM_GENERATION_TOKENS_PER_REQUEST - 1))],
54 + "vllm:e2e_request_latency_seconds": [("_count", _NUM_REQUESTS)],
55 + "vllm:request_prompt_tokens":
56 + [("_sum", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST),
57 + ("_count", _NUM_REQUESTS)],
58 + "vllm:request_generation_tokens":
59 + [("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
60 + ("_count", _NUM_REQUESTS)],
61 + "vllm:request_params_n": [("_count", _NUM_REQUESTS)],
62 + "vllm:request_params_best_of": [("_count", _NUM_REQUESTS)],
63 + "vllm:prompt_tokens": [("_total",
64 + _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
65 + "vllm:generation_tokens":
66 + [("_total", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
67 + "vllm:request_success": [("_total", _NUM_REQUESTS)],
68 +}
69 +
70 +
71 +@pytest.mark.asyncio
72 +async def test_metrics_counts(client: openai.AsyncOpenAI):
73 + base_url = str(client.base_url)[:-3].strip("/")
74 +
75 + for _ in range(_NUM_REQUESTS):
76 + # sending a request triggers the metrics to be logged.
77 + await client.completions.create(
78 + model=MODEL_NAME,
79 + prompt=_TOKENIZED_PROMPT,
80 + max_tokens=_NUM_GENERATION_TOKENS_PER_REQUEST)
81 +
82 + response = requests.get(base_url + "/metrics")
83 + print(response.text)
84 + assert response.status_code == HTTPStatus.OK
85 +
86 + # Loop over all expected metric_families
87 + for metric_family, suffix_values_list in EXPECTED_VALUES.items():
88 + found_metric = False
89 +
90 + # Check to see if the metric_family is found in the prom endpoint.
91 + for family in text_string_to_metric_families(response.text):
92 + if family.name == metric_family:
93 + found_metric = True
94 +
95 + # Check that each suffix is found in the prom endpoint.
96 + for suffix, expected_value in suffix_values_list:
97 + metric_name_w_suffix = f"{metric_family}{suffix}"
98 + found_suffix = False
99 +
100 + for sample in family.samples:
101 + if sample.name == metric_name_w_suffix:
102 + found_suffix = True
103 +
104 + # For each suffix, value sure the value matches
105 + # what we expect.
106 + assert sample.value == expected_value, (
107 + f"{metric_name_w_suffix} expected value of "
108 + f"{expected_value} did not match found value "
109 + f"{sample.value}")
110 + break
111 + assert found_suffix, (
112 + f"Did not find {metric_name_w_suffix} in prom endpoint"
113 + )
114 + break
115 +
116 + assert found_metric, (f"Did not find {metric_family} in prom endpoint")
117 +
118 +
119 +EXPECTED_METRICS = [
120 + "vllm:num_requests_running",
121 + "vllm:num_requests_swapped",
122 + "vllm:num_requests_waiting",
123 + "vllm:gpu_cache_usage_perc",
124 + "vllm:cpu_cache_usage_perc",
125 + "vllm:time_to_first_token_seconds_sum",
126 + "vllm:time_to_first_token_seconds_bucket",
127 + "vllm:time_to_first_token_seconds_count",
128 + "vllm:time_per_output_token_seconds_sum",
129 + "vllm:time_per_output_token_seconds_bucket",
130 + "vllm:time_per_output_token_seconds_count",
131 + "vllm:e2e_request_latency_seconds_sum",
132 + "vllm:e2e_request_latency_seconds_bucket",
133 + "vllm:e2e_request_latency_seconds_count",
134 + "vllm:request_prompt_tokens_sum",
135 + "vllm:request_prompt_tokens_bucket",
136 + "vllm:request_prompt_tokens_count",
137 + "vllm:request_generation_tokens_sum",
138 + "vllm:request_generation_tokens_bucket",
139 + "vllm:request_generation_tokens_count",
140 + "vllm:request_params_n_sum",
141 + "vllm:request_params_n_bucket",
142 + "vllm:request_params_n_count",
143 + "vllm:request_params_best_of_sum",
144 + "vllm:request_params_best_of_bucket",
145 + "vllm:request_params_best_of_count",
146 + "vllm:num_preemptions_total",
147 + "vllm:prompt_tokens_total",
148 + "vllm:generation_tokens_total",
149 + "vllm:request_success_total",
150 + "vllm:cache_config_info",
151 + # labels in cache_config_info
152 + "block_size",
153 + "cache_dtype",
154 + "cpu_offload_gb",
155 + "enable_prefix_caching",
156 + "gpu_memory_utilization",
157 + "num_cpu_blocks",
158 + "num_gpu_blocks",
159 + "num_gpu_blocks_override",
160 + "sliding_window",
161 + "swap_space_bytes",
162 +]
163 +
164 +
165 +@pytest.mark.asyncio
166 +async def test_metrics_exist(client: openai.AsyncOpenAI):
167 + base_url = str(client.base_url)[:-3].strip("/")
168 +
169 + # sending a request triggers the metrics to be logged.
170 + await client.completions.create(model=MODEL_NAME,
171 + prompt="Hello, my name is",
172 + max_tokens=5,
173 + temperature=0.0)
174 +
175 + response = requests.get(base_url + "/metrics")
176 + assert response.status_code == HTTPStatus.OK
177 +
178 + for metric in EXPECTED_METRICS:
179 + assert metric in response.text
vllm/engine/async_llm_engine.py
+1 -1
@@ -15,7 +15,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
15 from vllm.engine.async_timeout import asyncio_timeout
16 from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
17 PromptComponents)
18 -from vllm.engine.metrics import StatLoggerBase
18 +from vllm.engine.metrics_types import StatLoggerBase
19 from vllm.executor.executor_base import ExecutorAsyncBase
20 from vllm.executor.ray_utils import initialize_ray_cluster, ray
21 from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
vllm/engine/llm_engine.py
+8 -2
@@ -16,8 +16,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
16 from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
17 SchedulerOutputs)
18 from vllm.engine.arg_utils import EngineArgs
19 -from vllm.engine.metrics import (LoggingStatLogger, PrometheusStatLogger,
20 - StatLoggerBase, Stats)
19 +from vllm.engine.metrics_types import StatLoggerBase, Stats
20 from vllm.engine.output_processor.interfaces import (
21 SequenceGroupOutputProcessor)
22 from vllm.engine.output_processor.stop_checker import StopChecker
@@ -339,6 +338,13 @@ class LLMEngine:
338 if stat_loggers is not None:
339 self.stat_loggers = stat_loggers
340 else:
341 + # Lazy import for prometheus multiprocessing.
342 + # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
343 + # before prometheus_client is imported.
344 + # See https://prometheus.github.io/client_python/multiprocess/
345 + from vllm.engine.metrics import (LoggingStatLogger,
346 + PrometheusStatLogger)
347 +
348 self.stat_loggers = {
349 "logging":
350 LoggingStatLogger(
vllm/engine/metrics.py
+48 -101
@@ -1,13 +1,12 @@
1 -import time
2 -from abc import ABC, abstractmethod
3 -from dataclasses import dataclass
1 from typing import TYPE_CHECKING
2 from typing import Counter as CollectionsCounter
6 -from typing import Dict, List, Optional, Protocol, Union
3 +from typing import Dict, List, Optional, Union
4
5 import numpy as np
6 import prometheus_client
7
8 +from vllm.engine.metrics_types import (StatLoggerBase, Stats,
9 + SupportsMetricsInfo)
10 from vllm.executor.ray_utils import ray
11 from vllm.logger import init_logger
12
@@ -29,41 +28,49 @@ prometheus_client.disable_created_metrics()
28
29 # begin-metrics-definitions
30 class Metrics:
31 + """
32 + vLLM uses a multiprocessing-based frontend for the OpenAI server.
33 + This means that we need to run prometheus_client in multiprocessing mode
34 + See https://prometheus.github.io/client_python/multiprocess/ for more
35 + details on limitations.
36 + """
37 labelname_finish_reason = "finished_reason"
38 _gauge_cls = prometheus_client.Gauge
39 _counter_cls = prometheus_client.Counter
40 _histogram_cls = prometheus_client.Histogram
41
42 def __init__(self, labelnames: List[str], max_model_len: int):
38 - # Unregister any existing vLLM collectors
43 + # Unregister any existing vLLM collectors (for CI/CD)
44 self._unregister_vllm_metrics()
45
41 - # Config Information
42 - self._create_info_cache_config()
43 -
46 # System stats
47 # Scheduler State
48 self.gauge_scheduler_running = self._gauge_cls(
49 name="vllm:num_requests_running",
50 documentation="Number of requests currently running on GPU.",
49 - labelnames=labelnames)
51 + labelnames=labelnames,
52 + multiprocess_mode="sum")
53 self.gauge_scheduler_waiting = self._gauge_cls(
54 name="vllm:num_requests_waiting",
55 documentation="Number of requests waiting to be processed.",
53 - labelnames=labelnames)
56 + labelnames=labelnames,
57 + multiprocess_mode="sum")
58 self.gauge_scheduler_swapped = self._gauge_cls(
59 name="vllm:num_requests_swapped",
60 documentation="Number of requests swapped to CPU.",
57 - labelnames=labelnames)
61 + labelnames=labelnames,
62 + multiprocess_mode="sum")
63 # KV Cache Usage in %
64 self.gauge_gpu_cache_usage = self._gauge_cls(
65 name="vllm:gpu_cache_usage_perc",
66 documentation="GPU KV-cache usage. 1 means 100 percent usage.",
62 - labelnames=labelnames)
67 + labelnames=labelnames,
68 + multiprocess_mode="sum")
69 self.gauge_cpu_cache_usage = self._gauge_cls(
70 name="vllm:cpu_cache_usage_perc",
71 documentation="CPU KV-cache usage. 1 means 100 percent usage.",
66 - labelnames=labelnames)
72 + labelnames=labelnames,
73 + multiprocess_mode="sum")
74
75 # Iteration stats
76 self.counter_num_preemption = self._counter_cls(
@@ -137,11 +144,13 @@ class Metrics:
144 self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
145 name="vllm:spec_decode_draft_acceptance_rate",
146 documentation="Speulative token acceptance rate.",
140 - labelnames=labelnames)
147 + labelnames=labelnames,
148 + multiprocess_mode="sum")
149 self.gauge_spec_decode_efficiency = self._gauge_cls(
150 name="vllm:spec_decode_efficiency",
151 documentation="Speculative decoding system efficiency.",
144 - labelnames=labelnames)
152 + labelnames=labelnames,
153 + multiprocess_mode="sum")
154 self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
155 name="vllm:spec_decode_num_accepted_tokens_total",
156 documentation="Number of accepted tokens.",
@@ -160,19 +169,18 @@ class Metrics:
169 name="vllm:avg_prompt_throughput_toks_per_s",
170 documentation="Average prefill throughput in tokens/s.",
171 labelnames=labelnames,
172 + multiprocess_mode="sum",
173 )
174 # Deprecated in favor of vllm:generation_tokens_total
175 self.gauge_avg_generation_throughput = self._gauge_cls(
176 name="vllm:avg_generation_throughput_toks_per_s",
177 documentation="Average generation throughput in tokens/s.",
178 labelnames=labelnames,
179 + multiprocess_mode="sum",
180 )
181
171 - def _create_info_cache_config(self) -> None:
172 - # Config Information
173 - self.info_cache_config = prometheus_client.Info(
174 - name='vllm:cache_config',
175 - documentation='information of cache_config')
182 +
183 +# end-metrics-definitions
184
185 def _unregister_vllm_metrics(self) -> None:
186 for collector in list(prometheus_client.REGISTRY._collector_to_names):
@@ -180,9 +188,6 @@ class Metrics:
188 prometheus_client.REGISTRY.unregister(collector)
189
190
183 -# end-metrics-definitions
184 -
185 -
191 class _RayGaugeWrapper:
192 """Wraps around ray.util.metrics.Gauge to provide same API as
193 prometheus_client.Gauge"""
@@ -190,7 +195,9 @@ class _RayGaugeWrapper:
195 def __init__(self,
196 name: str,
197 documentation: str = "",
193 - labelnames: Optional[List[str]] = None):
198 + labelnames: Optional[List[str]] = None,
199 + multiprocess_mode: str = ""):
200 + del multiprocess_mode
201 labelnames_tuple = tuple(labelnames) if labelnames else None
202 self._gauge = ray_metrics.Gauge(name=name,
203 description=documentation,
@@ -268,10 +275,6 @@ class RayMetrics(Metrics):
275 # No-op on purpose
276 pass
277
271 - def _create_info_cache_config(self) -> None:
272 - # No-op on purpose
273 - pass
274 -
278
279 def build_1_2_5_buckets(max_value: int) -> List[int]:
280 """
@@ -295,46 +298,6 @@ def build_1_2_5_buckets(max_value: int) -> List[int]:
298 exponent += 1
299
300
298 -@dataclass
299 -class Stats:
300 - """Created by LLMEngine for use by StatLogger."""
301 - now: float
302 -
303 - # System stats (should have _sys suffix)
304 - # Scheduler State
305 - num_running_sys: int
306 - num_waiting_sys: int
307 - num_swapped_sys: int
308 - # KV Cache Usage in %
309 - gpu_cache_usage_sys: float
310 - cpu_cache_usage_sys: float
311 -
312 - # Iteration stats (should have _iter suffix)
313 - num_prompt_tokens_iter: int
314 - num_generation_tokens_iter: int
315 - time_to_first_tokens_iter: List[float]
316 - time_per_output_tokens_iter: List[float]
317 - num_preemption_iter: int
318 -
319 - # Request stats (should have _requests suffix)
320 - # Latency
321 - time_e2e_requests: List[float]
322 - # Metadata
323 - num_prompt_tokens_requests: List[int]
324 - num_generation_tokens_requests: List[int]
325 - best_of_requests: List[int]
326 - n_requests: List[int]
327 - finished_reason_requests: List[str]
328 -
329 - spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
330 -
331 -
332 -class SupportsMetricsInfo(Protocol):
333 -
334 - def metrics_info(self) -> Dict[str, str]:
335 - ...
336 -
337 -
301 def local_interval_elapsed(now: float, last_log: float,
302 local_interval: float) -> bool:
303 elapsed_time = now - last_log
@@ -346,38 +309,9 @@ def get_throughput(tracked_stats: List[int], now: float,
309 return float(np.sum(tracked_stats) / (now - last_log))
310
311
349 -class StatLoggerBase(ABC):
350 - """Base class for StatLogger."""
351 -
352 - def __init__(self, local_interval: float) -> None:
353 - # Tracked stats over current local logging interval.
354 - self.num_prompt_tokens: List[int] = []
355 - self.num_generation_tokens: List[int] = []
356 - self.last_local_log = time.time()
357 - self.local_interval = local_interval
358 - self.spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
359 -
360 - @abstractmethod
361 - def info(self, type: str, obj: SupportsMetricsInfo) -> None:
362 - raise NotImplementedError
363 -
364 - @abstractmethod
365 - def log(self, stats: Stats) -> None:
366 - raise NotImplementedError
367 -
368 - def maybe_update_spec_decode_metrics(self, stats: Stats):
369 - """Save spec decode metrics (since they are unlikely
370 - to be emitted at same time as log interval)."""
371 - if stats.spec_decode_metrics is not None:
372 - self.spec_decode_metrics = stats.spec_decode_metrics
373 -
374 -
312 class LoggingStatLogger(StatLoggerBase):
313 """LoggingStatLogger is used in LLMEngine to log to Stdout."""
314
378 - def info(self, type: str, obj: SupportsMetricsInfo) -> None:
379 - raise NotImplementedError
380 -
315 def log(self, stats: Stats) -> None:
316 """Called by LLMEngine.
317 Logs to Stdout every self.local_interval seconds."""
@@ -440,10 +374,14 @@ class LoggingStatLogger(StatLoggerBase):
374 f"Number of draft tokens: {metrics.draft_tokens}, "
375 f"Number of emitted tokens: {metrics.emitted_tokens}.")
376
377 + def info(self, type: str, obj: SupportsMetricsInfo) -> None:
378 + raise NotImplementedError
379 +
380
381 class PrometheusStatLogger(StatLoggerBase):
382 """PrometheusStatLogger is used LLMEngine to log to Promethus."""
383 _metrics_cls = Metrics
384 + _gauge_cls = prometheus_client.Gauge
385
386 def __init__(self, local_interval: float, labels: Dict[str, str],
387 max_model_len: int) -> None:
@@ -453,10 +391,6 @@ class PrometheusStatLogger(StatLoggerBase):
391 self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
392 max_model_len=max_model_len)
393
456 - def info(self, type: str, obj: SupportsMetricsInfo) -> None:
457 - if type == "cache_config":
458 - self.metrics.info_cache_config.info(obj.metrics_info())
459 -
394 def _log_gauge(self, gauge, data: Union[int, float]) -> None:
395 # Convenience function for logging to gauge.
396 gauge.labels(**self.labels).set(data)
@@ -586,6 +520,19 @@ class PrometheusStatLogger(StatLoggerBase):
520 self.last_local_log = stats.now
521 self.spec_decode_metrics = None
522
523 + def info(self, type: str, obj: SupportsMetricsInfo) -> None:
524 + # Info type metrics are syntactic sugar for a gauge permanently set to 1
525 + # Since prometheus multiprocessing mode does not support Info, emulate
526 + # info here with a gauge.
527 + if type == "cache_config":
528 + metrics_info = obj.metrics_info()
529 + info_gauge = self._gauge_cls(
530 + name="vllm:cache_config_info",
531 + documentation="Information of the LLMEngine CacheConfig",
532 + labelnames=metrics_info.keys(),
533 + multiprocess_mode="mostrecent")
534 + info_gauge.labels(**metrics_info).set(1)
535 +
536
537 class RayPrometheusStatLogger(PrometheusStatLogger):
538 """RayPrometheusStatLogger uses Ray metrics instead."""
vllm/engine/metrics_types.py new
+85
@@ -0,0 +1,85 @@
1 +"""
2 +These types are defined in this file to avoid importing vllm.engine.metrics
3 +and therefore importing prometheus_client.
4 +
5 +This is required due to usage of Prometheus multiprocess mode to enable
6 +metrics after splitting out the uvicorn process from the engine process.
7 +
8 +Prometheus multiprocess mode requires setting PROMETHEUS_MULTIPROC_DIR
9 +before prometheus_client is imported. Typically, this is done by setting
10 +the env variable before launch, but since we are a library, we need to
11 +do this in Python code and lazily import prometheus_client.
12 +"""
13 +
14 +import time
15 +from abc import ABC, abstractmethod
16 +from dataclasses import dataclass
17 +from typing import Dict, List, Optional, Protocol
18 +
19 +from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
20 +
21 +
22 +@dataclass
23 +class Stats:
24 + """Created by LLMEngine for use by StatLogger."""
25 + now: float
26 +
27 + # System stats (should have _sys suffix)
28 + # Scheduler State
29 + num_running_sys: int
30 + num_waiting_sys: int
31 + num_swapped_sys: int
32 + # KV Cache Usage in %
33 + gpu_cache_usage_sys: float
34 + cpu_cache_usage_sys: float
35 +
36 + # Iteration stats (should have _iter suffix)
37 + num_prompt_tokens_iter: int
38 + num_generation_tokens_iter: int
39 + time_to_first_tokens_iter: List[float]
40 + time_per_output_tokens_iter: List[float]
41 + num_preemption_iter: int
42 +
43 + # Request stats (should have _requests suffix)
44 + # Latency
45 + time_e2e_requests: List[float]
46 + # Metadata
47 + num_prompt_tokens_requests: List[int]
48 + num_generation_tokens_requests: List[int]
49 + best_of_requests: List[int]
50 + n_requests: List[int]
51 + finished_reason_requests: List[str]
52 +
53 + spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
54 +
55 +
56 +class SupportsMetricsInfo(Protocol):
57 +
58 + def metrics_info(self) -> Dict[str, str]:
59 + ...
60 +
61 +
62 +class StatLoggerBase(ABC):
63 + """Base class for StatLogger."""
64 +
65 + def __init__(self, local_interval: float) -> None:
66 + # Tracked stats over current local logging interval.
67 + self.num_prompt_tokens: List[int] = []
68 + self.num_generation_tokens: List[int] = []
69 + self.last_local_log = time.time()
70 + self.local_interval = local_interval
71 + self.spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
72 +
73 + @abstractmethod
74 + def log(self, stats: Stats) -> None:
75 + raise NotImplementedError
76 +
77 + @abstractmethod
78 + def info(self, type: str, obj: SupportsMetricsInfo) -> None:
79 + raise NotImplementedError
80 +
81 + def maybe_update_spec_decode_metrics(self, stats: Stats):
82 + """Save spec decode metrics (since they are unlikely
83 + to be emitted at same time as log interval)."""
84 + if stats.spec_decode_metrics is not None:
85 + self.spec_decode_metrics = stats.spec_decode_metrics
vllm/entrypoints/openai/api_server.py
+45 -3
@@ -2,7 +2,9 @@ import asyncio
2 import importlib
3 import inspect
4 import multiprocessing
5 +import os
6 import re
7 +import tempfile
8 from argparse import Namespace
9 from contextlib import asynccontextmanager
10 from http import HTTPStatus
@@ -12,7 +14,6 @@ from fastapi import APIRouter, FastAPI, Request
14 from fastapi.exceptions import RequestValidationError
15 from fastapi.middleware.cors import CORSMiddleware
16 from fastapi.responses import JSONResponse, Response, StreamingResponse
15 -from prometheus_client import make_asgi_app
17 from starlette.routing import Mount
18
19 import vllm.envs as envs
@@ -54,6 +55,7 @@ openai_serving_chat: OpenAIServingChat
55 openai_serving_completion: OpenAIServingCompletion
56 openai_serving_embedding: OpenAIServingEmbedding
57 openai_serving_tokenization: OpenAIServingTokenization
58 +prometheus_multiproc_dir: tempfile.TemporaryDirectory
59
60 logger = init_logger('vllm.entrypoints.openai.api_server')
61
@@ -109,6 +111,21 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
111
112 # Otherwise, use the multiprocessing AsyncLLMEngine.
113 else:
114 + if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
115 + # Make TemporaryDirectory for prometheus multiprocessing
116 + # Note: global TemporaryDirectory will be automatically
117 + # cleaned up upon exit.
118 + global prometheus_multiproc_dir
119 + prometheus_multiproc_dir = tempfile.TemporaryDirectory()
120 + os.environ[
121 + "PROMETHEUS_MULTIPROC_DIR"] = prometheus_multiproc_dir.name
122 + else:
123 + logger.warning(
124 + "Found PROMETHEUS_MULTIPROC_DIR was set by user. "
125 + "This directory must be wiped between vLLM runs or "
126 + "you will find inaccurate metrics. Unset the variable "
127 + "and vLLM will properly handle cleanup.")
128 +
129 # Select random path for IPC.
130 rpc_path = get_open_zmq_ipc_path()
131 logger.info("Multiprocessing frontend to use %s for RPC Path.",
@@ -149,13 +166,38 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
166 # Wait for server process to join
167 rpc_server_process.join()
168
169 + # Lazy import for prometheus multiprocessing.
170 + # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
171 + # before prometheus_client is imported.
172 + # See https://prometheus.github.io/client_python/multiprocess/
173 + from prometheus_client import multiprocess
174 + multiprocess.mark_process_dead(rpc_server_process.pid)
175 +
176
177 router = APIRouter()
178
179
180 def mount_metrics(app: FastAPI):
157 - # Add prometheus asgi middleware to route /metrics requests
158 - metrics_route = Mount("/metrics", make_asgi_app())
181 + # Lazy import for prometheus multiprocessing.
182 + # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
183 + # before prometheus_client is imported.
184 + # See https://prometheus.github.io/client_python/multiprocess/
185 + from prometheus_client import (CollectorRegistry, make_asgi_app,
186 + multiprocess)
187 +
188 + prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None)
189 + if prometheus_multiproc_dir_path is not None:
190 + logger.info("vLLM to use %s as PROMETHEUS_MULTIPROC_DIR",
191 + prometheus_multiproc_dir_path)
192 + registry = CollectorRegistry()
193 + multiprocess.MultiProcessCollector(registry)
194 +
195 + # Add prometheus asgi middleware to route /metrics requests
196 + metrics_route = Mount("/metrics", make_asgi_app(registry=registry))
197 + else:
198 + # Add prometheus asgi middleware to route /metrics requests
199 + metrics_route = Mount("/metrics", make_asgi_app())
200 +
201 # Workaround for 307 Redirect for /metrics
202 metrics_route.path_regex = re.compile('^/metrics(?P<path>.*)$')
203 app.routes.append(metrics_route)