@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5f0b9933e6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix Ray Metrics API usage (#6354)

Antoni Baum committed Jul 17, 2024 at 12:40 UTC 5f0b9933e63839e816b9736a65a3c55005df2cfe
4 files changed +195 -40
tests/metrics/test_metrics.py
+54
@@ -1,11 +1,13 @@
1 from typing import List
2
3 import pytest
4 +import ray
5 from prometheus_client import REGISTRY
6
7 from vllm import EngineArgs, LLMEngine
8 from vllm.engine.arg_utils import AsyncEngineArgs
9 from vllm.engine.async_llm_engine import AsyncLLMEngine
10 +from vllm.engine.metrics import RayPrometheusStatLogger
11 from vllm.sampling_params import SamplingParams
12
13 MODELS = [
@@ -241,3 +243,55 @@ def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
243 labels)
244 assert (
245 metric_value == num_requests), "Metrics should be collected"
246 +
247 +
248 +@pytest.mark.parametrize("model", MODELS)
249 +@pytest.mark.parametrize("dtype", ["half"])
250 +@pytest.mark.parametrize("max_tokens", [16])
251 +def test_engine_log_metrics_ray(
252 + example_prompts,
253 + model: str,
254 + dtype: str,
255 + max_tokens: int,
256 +) -> None:
257 + # This test is quite weak - it only checks that we can use
258 + # RayPrometheusStatLogger without exceptions.
259 + # Checking whether the metrics are actually emitted is unfortunately
260 + # non-trivial.
261 +
262 + # We have to run in a Ray task for Ray metrics to be emitted correctly
263 + @ray.remote(num_gpus=1)
264 + def _inner():
265 +
266 + class _RayPrometheusStatLogger(RayPrometheusStatLogger):
267 +
268 + def __init__(self, *args, **kwargs):
269 + self._i = 0
270 + super().__init__(*args, **kwargs)
271 +
272 + def log(self, *args, **kwargs):
273 + self._i += 1
274 + return super().log(*args, **kwargs)
275 +
276 + engine_args = EngineArgs(
277 + model=model,
278 + dtype=dtype,
279 + disable_log_stats=False,
280 + )
281 + engine = LLMEngine.from_engine_args(engine_args)
282 + logger = _RayPrometheusStatLogger(
283 + local_interval=0.5,
284 + labels=dict(model_name=engine.model_config.served_model_name),
285 + max_model_len=engine.model_config.max_model_len)
286 + engine.add_logger("ray", logger)
287 + for i, prompt in enumerate(example_prompts):
288 + engine.add_request(
289 + f"request-id-{i}",
290 + prompt,
291 + SamplingParams(max_tokens=max_tokens),
292 + )
293 + while engine.has_unfinished_requests():
294 + engine.step()
295 + assert logger._i > 0, ".log must be called at least once"
296 +
297 + ray.get(_inner.remote())
vllm/engine/async_llm_engine.py
+19
@@ -12,6 +12,7 @@ from vllm.core.scheduler import SchedulerOutputs
12 from vllm.engine.arg_utils import AsyncEngineArgs
13 from vllm.engine.async_timeout import asyncio_timeout
14 from vllm.engine.llm_engine import LLMEngine
15 +from vllm.engine.metrics import StatLoggerBase
16 from vllm.executor.ray_utils import initialize_ray_cluster, ray
17 from vllm.inputs import LLMInputs, PromptInputs
18 from vllm.logger import init_logger
@@ -389,6 +390,7 @@ class AsyncLLMEngine:
390 engine_args: AsyncEngineArgs,
391 start_engine_loop: bool = True,
392 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
393 + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
394 ) -> "AsyncLLMEngine":
395 """Creates an async LLM engine from the engine arguments."""
396 # Create the engine configs.
@@ -451,6 +453,7 @@ class AsyncLLMEngine:
453 max_log_len=engine_args.max_log_len,
454 start_engine_loop=start_engine_loop,
455 usage_context=usage_context,
456 + stat_loggers=stat_loggers,
457 )
458 return engine
459
@@ -957,3 +960,19 @@ class AsyncLLMEngine:
960 )
961 else:
962 return self.engine.is_tracing_enabled()
963 +
964 + def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None:
965 + if self.engine_use_ray:
966 + ray.get(
967 + self.engine.add_logger.remote( # type: ignore
968 + logger_name=logger_name, logger=logger))
969 + else:
970 + self.engine.add_logger(logger_name=logger_name, logger=logger)
971 +
972 + def remove_logger(self, logger_name: str) -> None:
973 + if self.engine_use_ray:
974 + ray.get(
975 + self.engine.remove_logger.remote( # type: ignore
976 + logger_name=logger_name))
977 + else:
978 + self.engine.remove_logger(logger_name=logger_name)
vllm/engine/llm_engine.py
+2
@@ -379,6 +379,7 @@ class LLMEngine:
379 cls,
380 engine_args: EngineArgs,
381 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
382 + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
383 ) -> "LLMEngine":
384 """Creates an LLM engine from the engine arguments."""
385 # Create the engine configs.
@@ -423,6 +424,7 @@ class LLMEngine:
424 executor_class=executor_class,
425 log_stats=not engine_args.disable_log_stats,
426 usage_context=usage_context,
427 + stat_loggers=stat_loggers,
428 )
429 return engine
430
vllm/engine/metrics.py
+120 -40
@@ -30,55 +30,55 @@ prometheus_client.disable_created_metrics()
30 # begin-metrics-definitions
31 class Metrics:
32 labelname_finish_reason = "finished_reason"
33 - _base_library = prometheus_client
33 + _gauge_cls = prometheus_client.Gauge
34 + _counter_cls = prometheus_client.Counter
35 + _histogram_cls = prometheus_client.Histogram
36
37 def __init__(self, labelnames: List[str], max_model_len: int):
38 # Unregister any existing vLLM collectors
39 self._unregister_vllm_metrics()
40
41 # Config Information
40 - self.info_cache_config = prometheus_client.Info(
41 - name='vllm:cache_config',
42 - documentation='information of cache_config')
42 + self._create_info_cache_config()
43
44 # System stats
45 # Scheduler State
46 - self.gauge_scheduler_running = self._base_library.Gauge(
46 + self.gauge_scheduler_running = self._gauge_cls(
47 name="vllm:num_requests_running",
48 documentation="Number of requests currently running on GPU.",
49 labelnames=labelnames)
50 - self.gauge_scheduler_waiting = self._base_library.Gauge(
50 + self.gauge_scheduler_waiting = self._gauge_cls(
51 name="vllm:num_requests_waiting",
52 documentation="Number of requests waiting to be processed.",
53 labelnames=labelnames)
54 - self.gauge_scheduler_swapped = self._base_library.Gauge(
54 + self.gauge_scheduler_swapped = self._gauge_cls(
55 name="vllm:num_requests_swapped",
56 documentation="Number of requests swapped to CPU.",
57 labelnames=labelnames)
58 # KV Cache Usage in %
59 - self.gauge_gpu_cache_usage = self._base_library.Gauge(
59 + self.gauge_gpu_cache_usage = self._gauge_cls(
60 name="vllm:gpu_cache_usage_perc",
61 documentation="GPU KV-cache usage. 1 means 100 percent usage.",
62 labelnames=labelnames)
63 - self.gauge_cpu_cache_usage = self._base_library.Gauge(
63 + self.gauge_cpu_cache_usage = self._gauge_cls(
64 name="vllm:cpu_cache_usage_perc",
65 documentation="CPU KV-cache usage. 1 means 100 percent usage.",
66 labelnames=labelnames)
67
68 # Iteration stats
69 - self.counter_num_preemption = self._base_library.Counter(
69 + self.counter_num_preemption = self._counter_cls(
70 name="vllm:num_preemptions_total",
71 documentation="Cumulative number of preemption from the engine.",
72 labelnames=labelnames)
73 - self.counter_prompt_tokens = self._base_library.Counter(
73 + self.counter_prompt_tokens = self._counter_cls(
74 name="vllm:prompt_tokens_total",
75 documentation="Number of prefill tokens processed.",
76 labelnames=labelnames)
77 - self.counter_generation_tokens = self._base_library.Counter(
77 + self.counter_generation_tokens = self._counter_cls(
78 name="vllm:generation_tokens_total",
79 documentation="Number of generation tokens processed.",
80 labelnames=labelnames)
81 - self.histogram_time_to_first_token = self._base_library.Histogram(
81 + self.histogram_time_to_first_token = self._histogram_cls(
82 name="vllm:time_to_first_token_seconds",
83 documentation="Histogram of time to first token in seconds.",
84 labelnames=labelnames,
@@ -86,7 +86,7 @@ class Metrics:
86 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
87 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
88 ])
89 - self.histogram_time_per_output_token = self._base_library.Histogram(
89 + self.histogram_time_per_output_token = self._histogram_cls(
90 name="vllm:time_per_output_token_seconds",
91 documentation="Histogram of time per output token in seconds.",
92 labelnames=labelnames,
@@ -97,83 +97,157 @@ class Metrics:
97
98 # Request stats
99 # Latency
100 - self.histogram_e2e_time_request = self._base_library.Histogram(
100 + self.histogram_e2e_time_request = self._histogram_cls(
101 name="vllm:e2e_request_latency_seconds",
102 documentation="Histogram of end to end request latency in seconds.",
103 labelnames=labelnames,
104 buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
105 # Metadata
106 - self.histogram_num_prompt_tokens_request = self._base_library.Histogram(
106 + self.histogram_num_prompt_tokens_request = self._histogram_cls(
107 name="vllm:request_prompt_tokens",
108 documentation="Number of prefill tokens processed.",
109 labelnames=labelnames,
110 buckets=build_1_2_5_buckets(max_model_len),
111 )
112 self.histogram_num_generation_tokens_request = \
113 - self._base_library.Histogram(
113 + self._histogram_cls(
114 name="vllm:request_generation_tokens",
115 documentation="Number of generation tokens processed.",
116 labelnames=labelnames,
117 buckets=build_1_2_5_buckets(max_model_len),
118 )
119 - self.histogram_best_of_request = self._base_library.Histogram(
119 + self.histogram_best_of_request = self._histogram_cls(
120 name="vllm:request_params_best_of",
121 documentation="Histogram of the best_of request parameter.",
122 labelnames=labelnames,
123 buckets=[1, 2, 5, 10, 20],
124 )
125 - self.histogram_n_request = self._base_library.Histogram(
125 + self.histogram_n_request = self._histogram_cls(
126 name="vllm:request_params_n",
127 documentation="Histogram of the n request parameter.",
128 labelnames=labelnames,
129 buckets=[1, 2, 5, 10, 20],
130 )
131 - self.counter_request_success = self._base_library.Counter(
131 + self.counter_request_success = self._counter_cls(
132 name="vllm:request_success_total",
133 documentation="Count of successfully processed requests.",
134 labelnames=labelnames + [Metrics.labelname_finish_reason])
135
136 # Speculatie decoding stats
137 - self.gauge_spec_decode_draft_acceptance_rate = self._base_library.Gauge(
137 + self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
138 name="vllm:spec_decode_draft_acceptance_rate",
139 documentation="Speulative token acceptance rate.",
140 labelnames=labelnames)
141 - self.gauge_spec_decode_efficiency = self._base_library.Gauge(
141 + self.gauge_spec_decode_efficiency = self._gauge_cls(
142 name="vllm:spec_decode_efficiency",
143 documentation="Speculative decoding system efficiency.",
144 labelnames=labelnames)
145 - self.counter_spec_decode_num_accepted_tokens = (
146 - self._base_library.Counter(
147 - name="vllm:spec_decode_num_accepted_tokens_total",
148 - documentation="Number of accepted tokens.",
149 - labelnames=labelnames))
150 - self.counter_spec_decode_num_draft_tokens = self._base_library.Counter(
145 + self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
146 + name="vllm:spec_decode_num_accepted_tokens_total",
147 + documentation="Number of accepted tokens.",
148 + labelnames=labelnames))
149 + self.counter_spec_decode_num_draft_tokens = self._counter_cls(
150 name="vllm:spec_decode_num_draft_tokens_total",
151 documentation="Number of draft tokens.",
152 labelnames=labelnames)
154 - self.counter_spec_decode_num_emitted_tokens = (
155 - self._base_library.Counter(
156 - name="vllm:spec_decode_num_emitted_tokens_total",
157 - documentation="Number of emitted tokens.",
158 - labelnames=labelnames))
153 + self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
154 + name="vllm:spec_decode_num_emitted_tokens_total",
155 + documentation="Number of emitted tokens.",
156 + labelnames=labelnames))
157
158 # Deprecated in favor of vllm:prompt_tokens_total
161 - self.gauge_avg_prompt_throughput = self._base_library.Gauge(
159 + self.gauge_avg_prompt_throughput = self._gauge_cls(
160 name="vllm:avg_prompt_throughput_toks_per_s",
161 documentation="Average prefill throughput in tokens/s.",
162 labelnames=labelnames,
163 )
164 # Deprecated in favor of vllm:generation_tokens_total
167 - self.gauge_avg_generation_throughput = self._base_library.Gauge(
165 + self.gauge_avg_generation_throughput = self._gauge_cls(
166 name="vllm:avg_generation_throughput_toks_per_s",
167 documentation="Average generation throughput in tokens/s.",
168 labelnames=labelnames,
169 )
170
171 + def _create_info_cache_config(self) -> None:
172 + # Config Information
173 + self.info_cache_config = prometheus_client.Info(
174 + name='vllm:cache_config',
175 + documentation='information of cache_config')
176 +
177 def _unregister_vllm_metrics(self) -> None:
174 - for collector in list(self._base_library.REGISTRY._collector_to_names):
178 + for collector in list(prometheus_client.REGISTRY._collector_to_names):
179 if hasattr(collector, "_name") and "vllm" in collector._name:
176 - self._base_library.REGISTRY.unregister(collector)
180 + prometheus_client.REGISTRY.unregister(collector)
181 +
182 +
183 +# end-metrics-definitions
184 +
185 +
186 +class _RayGaugeWrapper:
187 + """Wraps around ray.util.metrics.Gauge to provide same API as
188 + prometheus_client.Gauge"""
189 +
190 + def __init__(self,
191 + name: str,
192 + documentation: str = "",
193 + labelnames: Optional[List[str]] = None):
194 + labelnames_tuple = tuple(labelnames) if labelnames else None
195 + self._gauge = ray_metrics.Gauge(name=name,
196 + description=documentation,
197 + tag_keys=labelnames_tuple)
198 +
199 + def labels(self, **labels):
200 + self._gauge.set_default_tags(labels)
201 + return self
202 +
203 + def set(self, value: Union[int, float]):
204 + return self._gauge.set(value)
205 +
206 +
207 +class _RayCounterWrapper:
208 + """Wraps around ray.util.metrics.Counter to provide same API as
209 + prometheus_client.Counter"""
210 +
211 + def __init__(self,
212 + name: str,
213 + documentation: str = "",
214 + labelnames: Optional[List[str]] = None):
215 + labelnames_tuple = tuple(labelnames) if labelnames else None
216 + self._counter = ray_metrics.Counter(name=name,
217 + description=documentation,
218 + tag_keys=labelnames_tuple)
219 +
220 + def labels(self, **labels):
221 + self._counter.set_default_tags(labels)
222 + return self
223 +
224 + def inc(self, value: Union[int, float] = 1.0):
225 + if value == 0:
226 + return
227 + return self._counter.inc(value)
228 +
229 +
230 +class _RayHistogramWrapper:
231 + """Wraps around ray.util.metrics.Histogram to provide same API as
232 + prometheus_client.Histogram"""
233 +
234 + def __init__(self,
235 + name: str,
236 + documentation: str = "",
237 + labelnames: Optional[List[str]] = None,
238 + buckets: Optional[List[float]] = None):
239 + labelnames_tuple = tuple(labelnames) if labelnames else None
240 + self._histogram = ray_metrics.Histogram(name=name,
241 + description=documentation,
242 + tag_keys=labelnames_tuple,
243 + boundaries=buckets)
244 +
245 + def labels(self, **labels):
246 + self._histogram.set_default_tags(labels)
247 + return self
248 +
249 + def observe(self, value: Union[int, float]):
250 + return self._histogram.observe(value)
251
252
253 class RayMetrics(Metrics):
@@ -181,7 +255,9 @@ class RayMetrics(Metrics):
255 RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
256 Provides the same metrics as Metrics but uses Ray's util.metrics library.
257 """
184 - _base_library = ray_metrics
258 + _gauge_cls = _RayGaugeWrapper
259 + _counter_cls = _RayCounterWrapper
260 + _histogram_cls = _RayHistogramWrapper
261
262 def __init__(self, labelnames: List[str], max_model_len: int):
263 if ray_metrics is None:
@@ -192,8 +268,9 @@ class RayMetrics(Metrics):
268 # No-op on purpose
269 pass
270
195 -
196 -# end-metrics-definitions
271 + def _create_info_cache_config(self) -> None:
272 + # No-op on purpose
273 + pass
274
275
276 def build_1_2_5_buckets(max_value: int) -> List[int]:
@@ -498,3 +575,6 @@ class PrometheusStatLogger(StatLoggerBase):
575 class RayPrometheusStatLogger(PrometheusStatLogger):
576 """RayPrometheusStatLogger uses Ray metrics instead."""
577 _metrics_cls = RayMetrics
578 +
579 + def info(self, type: str, obj: SupportsMetricsInfo) -> None:
580 + return None