A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc][DP] support customized aggregated logger for dp (#24354)
Signed-off-by: Lu Fang <fanglu@fb.com>
Lucia Fang committed
Oct 13, 2025 at 17:45 UTC
8317f723540a9aef2ad1390ac4952aaa13b46f46
8 files changed
+297
-85
examples/online_serving/multi_instance_data_parallel.py
+42
-13
@@ -1,11 +1,13 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
+import threading
5
6
from vllm.engine.arg_utils import AsyncEngineArgs
7
from vllm.engine.async_llm_engine import AsyncLLMEngine
8
from vllm.outputs import RequestOutput
9
from vllm.sampling_params import SamplingParams
10
+from vllm.v1.metrics.loggers import AggregatedLoggingStatLogger
11
12
"""
13
To run this example, run the following commands simultaneously with
@@ -21,37 +23,64 @@ send a request to the instance with DP rank 1.
23
"""
24
25
26
+def _do_background_logging(engine, interval, stop_event):
27
+ try:
28
+ while not stop_event.is_set():
29
+ asyncio.run(engine.do_log_stats())
30
+ stop_event.wait(interval)
31
+ except Exception as e:
32
+ print(f"vLLM background logging shutdown: {e}")
33
+ pass
34
+
35
+
36
async def main():
37
engine_args = AsyncEngineArgs(
38
model="ibm-research/PowerMoE-3b",
39
data_parallel_size=2,
40
+ tensor_parallel_size=1,
41
dtype="auto",
42
max_model_len=2048,
43
data_parallel_address="127.0.0.1",
44
data_parallel_rpc_port=62300,
45
data_parallel_size_local=1,
46
enforce_eager=True,
47
+ enable_log_requests=True,
48
+ disable_custom_all_reduce=True,
49
)
50
36
- engine_client = AsyncLLMEngine.from_engine_args(engine_args)
37
-
51
+ engine_client = AsyncLLMEngine.from_engine_args(
52
+ engine_args,
53
+ # Example: Using aggregated logger
54
+ stat_loggers=[AggregatedLoggingStatLogger],
55
+ )
56
+ stop_logging_event = threading.Event()
57
+ logging_thread = threading.Thread(
58
+ target=_do_background_logging,
59
+ args=(engine_client, 5, stop_logging_event),
60
+ daemon=True,
61
+ )
62
+ logging_thread.start()
63
sampling_params = SamplingParams(
64
temperature=0.7,
65
top_p=0.9,
66
max_tokens=100,
67
)
68
+ num_prompts = 10
69
+ for i in range(num_prompts):
70
+ prompt = "Who won the 2004 World Series?"
71
+ final_output: RequestOutput | None = None
72
+ async for output in engine_client.generate(
73
+ prompt=prompt,
74
+ sampling_params=sampling_params,
75
+ request_id=f"abcdef-{i}",
76
+ data_parallel_rank=1,
77
+ ):
78
+ final_output = output
79
+ if final_output:
80
+ print(final_output.outputs[0].text)
81
44
- prompt = "Who won the 2004 World Series?"
45
- final_output: RequestOutput | None = None
46
- async for output in engine_client.generate(
47
- prompt=prompt,
48
- sampling_params=sampling_params,
49
- request_id="abcdef",
50
- data_parallel_rank=1,
51
- ):
52
- final_output = output
53
- if final_output:
54
- print(final_output.outputs[0].text)
82
+ stop_logging_event.set()
83
+ logging_thread.join()
84
85
86
if __name__ == "__main__":
tests/v1/engine/test_async_llm.py
+51
-5
@@ -17,7 +17,12 @@ from vllm.platforms import current_platform
17
from vllm.sampling_params import RequestOutputKind
18
from vllm.utils import set_default_torch_num_threads
19
from vllm.v1.engine.async_llm import AsyncLLM
20
-from vllm.v1.metrics.loggers import LoggingStatLogger
20
+from vllm.v1.metrics.loggers import (
21
+ AggregatedLoggingStatLogger,
22
+ LoggingStatLogger,
23
+ PerEngineStatLoggerAdapter,
24
+ PrometheusStatLogger,
25
+)
26
27
if not current_platform.is_cuda():
28
pytest.skip(reason="V1 currently only supported on CUDA.", allow_module_level=True)
@@ -384,6 +389,12 @@ class MockLoggingStatLogger(LoggingStatLogger):
389
self.log = MagicMock()
390
391
392
+class MockAggregatedStatLogger(AggregatedLoggingStatLogger):
393
+ def __init__(self, vllm_config: VllmConfig, engine_indexes: list[int]):
394
+ super().__init__(vllm_config, engine_indexes)
395
+ self.log = MagicMock()
396
+
397
+
398
@pytest.mark.asyncio
399
async def test_customize_loggers(monkeypatch):
400
"""Test that we can customize the loggers.
@@ -401,10 +412,45 @@ async def test_customize_loggers(monkeypatch):
412
413
await engine.do_log_stats()
414
404
- stat_loggers = engine.logger_manager.per_engine_logger_dict
405
- assert len(stat_loggers) == 1
406
- assert len(stat_loggers[0]) == 2 # LoggingStatLogger + MockLoggingStatLogger
407
- stat_loggers[0][0].log.assert_called_once()
415
+ stat_loggers = engine.logger_manager.stat_loggers
416
+ assert (
417
+ len(stat_loggers) == 3
418
+ ) # MockLoggingStatLogger + LoggingStatLogger + Promethus Logger
419
+ print(f"{stat_loggers=}")
420
+ stat_loggers[0].per_engine_stat_loggers[0].log.assert_called_once()
421
+ assert isinstance(stat_loggers[1], PerEngineStatLoggerAdapter)
422
+ assert isinstance(stat_loggers[1].per_engine_stat_loggers[0], LoggingStatLogger)
423
+ assert isinstance(stat_loggers[2], PrometheusStatLogger)
424
+
425
+
426
+@pytest.mark.asyncio
427
+async def test_customize_aggregated_loggers(monkeypatch):
428
+ """Test that we can customize the aggregated loggers.
429
+ If a customized logger is provided at the init, it should
430
+ be added to the default loggers.
431
+ """
432
+
433
+ with monkeypatch.context() as m, ExitStack() as after:
434
+ m.setenv("VLLM_USE_V1", "1")
435
+
436
+ with set_default_torch_num_threads(1):
437
+ engine = AsyncLLM.from_engine_args(
438
+ TEXT_ENGINE_ARGS,
439
+ stat_loggers=[MockLoggingStatLogger, MockAggregatedStatLogger],
440
+ )
441
+ after.callback(engine.shutdown)
442
+
443
+ await engine.do_log_stats()
444
+
445
+ stat_loggers = engine.logger_manager.stat_loggers
446
+ assert len(stat_loggers) == 4
447
+ # MockLoggingStatLogger + MockAggregatedStatLogger
448
+ # + LoggingStatLogger + PrometheusStatLogger
449
+ stat_loggers[0].per_engine_stat_loggers[0].log.assert_called_once()
450
+ stat_loggers[1].log.assert_called_once()
451
+ assert isinstance(stat_loggers[2], PerEngineStatLoggerAdapter)
452
+ assert isinstance(stat_loggers[2].per_engine_stat_loggers[0], LoggingStatLogger)
453
+ assert isinstance(stat_loggers[3], PrometheusStatLogger)
454
455
456
@pytest.mark.asyncio(scope="module")
tests/v1/metrics/test_engine_logger_apis.py
+5
-3
@@ -54,7 +54,7 @@ async def test_async_llm_replace_default_loggers(log_stats_enabled_engine_args):
54
engine = AsyncLLM.from_engine_args(
55
log_stats_enabled_engine_args, stat_loggers=[RayPrometheusStatLogger]
56
)
57
- assert isinstance(engine.logger_manager.prometheus_logger, RayPrometheusStatLogger)
57
+ assert isinstance(engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger)
58
engine.shutdown()
59
60
@@ -73,9 +73,11 @@ async def test_async_llm_add_to_default_loggers(log_stats_enabled_engine_args):
73
disabled_log_engine_args, stat_loggers=[DummyStatLogger]
74
)
75
76
- assert len(engine.logger_manager.per_engine_logger_dict[0]) == 1
76
+ assert len(engine.logger_manager.stat_loggers) == 2
77
+ assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1
78
assert isinstance(
78
- engine.logger_manager.per_engine_logger_dict[0][0], DummyStatLogger
79
+ engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0],
80
+ DummyStatLogger,
81
)
82
83
# log_stats is still True, since custom stat loggers are used
vllm/engine/arg_utils.py
+7
@@ -410,6 +410,7 @@ class EngineArgs:
410
max_logprobs: int = ModelConfig.max_logprobs
411
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
412
disable_log_stats: bool = False
413
+ aggregate_engine_logging: bool = False
414
revision: str | None = ModelConfig.revision
415
code_revision: str | None = ModelConfig.code_revision
416
rope_scaling: dict[str, Any] = get_field(ModelConfig, "rope_scaling")
@@ -1043,6 +1044,12 @@ class EngineArgs:
1044
help="Disable logging statistics.",
1045
)
1046
1047
+ parser.add_argument(
1048
+ "--aggregate-engine-logging",
1049
+ action="store_true",
1050
+ help="Log aggregate rather than per-engine statistics "
1051
+ "when using data parallelism.",
1052
+ )
1053
return parser
1054
1055
@classmethod
vllm/entrypoints/openai/api_server.py
+1
@@ -239,6 +239,7 @@ async def build_async_engine_client_from_engine_args(
239
vllm_config=vllm_config,
240
usage_context=usage_context,
241
enable_log_requests=engine_args.enable_log_requests,
242
+ aggregate_engine_logging=engine_args.aggregate_engine_logging,
243
disable_log_stats=engine_args.disable_log_stats,
244
client_addresses=client_config,
245
client_count=client_count,
vllm/v1/engine/async_llm.py
+4
@@ -57,6 +57,7 @@ class AsyncLLM(EngineClient):
57
log_requests: bool = True,
58
start_engine_loop: bool = True,
59
stat_loggers: list[StatLoggerFactory] | None = None,
60
+ aggregate_engine_logging: bool = False,
61
client_addresses: dict[str, str] | None = None,
62
client_count: int = 1,
63
client_index: int = 0,
@@ -144,6 +145,7 @@ class AsyncLLM(EngineClient):
145
custom_stat_loggers=stat_loggers,
146
enable_default_loggers=log_stats,
147
client_count=client_count,
148
+ aggregate_engine_logging=aggregate_engine_logging,
149
)
150
self.logger_manager.log_engine_initialized()
151
@@ -187,6 +189,7 @@ class AsyncLLM(EngineClient):
189
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
190
stat_loggers: list[StatLoggerFactory] | None = None,
191
enable_log_requests: bool = False,
192
+ aggregate_engine_logging: bool = False,
193
disable_log_stats: bool = False,
194
client_addresses: dict[str, str] | None = None,
195
client_count: int = 1,
@@ -209,6 +212,7 @@ class AsyncLLM(EngineClient):
212
stat_loggers=stat_loggers,
213
log_requests=enable_log_requests,
214
log_stats=not disable_log_stats,
215
+ aggregate_engine_logging=aggregate_engine_logging,
216
usage_context=usage_context,
217
client_addresses=client_addresses,
218
client_count=client_count,
vllm/v1/engine/llm_engine.py
+2
@@ -51,6 +51,7 @@ class LLMEngine:
51
vllm_config: VllmConfig,
52
executor_class: type[Executor],
53
log_stats: bool,
54
+ aggregate_engine_logging: bool = False,
55
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
56
stat_loggers: list[StatLoggerFactory] | None = None,
57
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
@@ -132,6 +133,7 @@ class LLMEngine:
133
vllm_config=vllm_config,
134
custom_stat_loggers=stat_loggers,
135
enable_default_loggers=log_stats,
136
+ aggregate_engine_logging=aggregate_engine_logging,
137
)
138
self.logger_manager.log_engine_initialized()
139
vllm/v1/metrics/loggers.py
+185
-64
@@ -24,7 +24,9 @@ from vllm.v1.spec_decode.metrics import SpecDecodingLogging, SpecDecodingProm
24
25
logger = init_logger(__name__)
26
27
-StatLoggerFactory = Callable[[VllmConfig, int], "StatLoggerBase"]
27
+PerEngineStatLoggerFactory = Callable[[VllmConfig, int], "StatLoggerBase"]
28
+AggregateStatLoggerFactory = type["AggregateStatLoggerBase"]
29
+StatLoggerFactory = AggregateStatLoggerFactory | PerEngineStatLoggerFactory
30
31
32
class StatLoggerBase(ABC):
@@ -54,6 +56,14 @@ class StatLoggerBase(ABC):
56
pass
57
58
59
+class AggregateStatLoggerBase(StatLoggerBase):
60
+ """Abstract base class for loggers that
61
+ aggregate across multiple DP engines."""
62
+
63
+ @abstractmethod
64
+ def __init__(self, vllm_config: VllmConfig, engine_indexes: list[int]): ...
65
+
66
+
67
class LoggingStatLogger(StatLoggerBase):
68
def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
69
self.engine_index = engine_index
@@ -72,6 +82,8 @@ class LoggingStatLogger(StatLoggerBase):
82
self.kv_connector_logging = KVConnectorLogging(kv_tranfer_config)
83
self.last_prompt_throughput: float = 0.0
84
self.last_generation_throughput: float = 0.0
85
+ self.engine_is_idle = False
86
+ self.aggregated = False
87
88
def _reset(self, now):
89
self.last_log_time = now
@@ -92,6 +104,10 @@ class LoggingStatLogger(StatLoggerBase):
104
return 0.0
105
return float(tracked_stats / delta_time)
106
107
+ @property
108
+ def log_prefix(self):
109
+ return "Engine {:03d}: ".format(self.engine_index)
110
+
111
def record(
112
self,
113
scheduler_stats: SchedulerStats | None,
@@ -110,34 +126,37 @@ class LoggingStatLogger(StatLoggerBase):
126
self.spec_decoding_logging.observe(scheduler_stats.spec_decoding_stats)
127
if kv_connector_stats := scheduler_stats.kv_connector_stats:
128
self.kv_connector_logging.observe(kv_connector_stats)
113
- self.last_scheduler_stats = scheduler_stats
114
-
129
+ if not self.aggregated:
130
+ self.last_scheduler_stats = scheduler_stats
131
if mm_cache_stats:
132
self.mm_caching_metrics.observe(mm_cache_stats)
133
118
- def log(self):
134
+ def _update_stats(self):
135
now = time.monotonic()
136
prompt_throughput = self._get_throughput(self.num_prompt_tokens, now)
137
generation_throughput = self._get_throughput(self.num_generation_tokens, now)
138
139
self._reset(now)
124
-
125
- scheduler_stats = self.last_scheduler_stats
126
-
127
- log_fn = logger.info
128
- if not any(
140
+ self.engine_is_idle = not any(
141
(
142
prompt_throughput,
143
generation_throughput,
144
self.last_prompt_throughput,
145
self.last_generation_throughput,
146
)
135
- ):
136
- # Avoid log noise on an idle production system
137
- log_fn = logger.debug
147
+ )
148
self.last_generation_throughput = generation_throughput
149
self.last_prompt_throughput = prompt_throughput
150
151
+ def aggregate_scheduler_stats(self):
152
+ # noop for per engine loggers
153
+ return
154
+
155
+ def log(self):
156
+ self._update_stats()
157
+ self.aggregate_scheduler_stats()
158
+ # Avoid log noise on an idle production system
159
+ log_fn = logger.debug if self.engine_is_idle else logger.info
160
# Format and print output.
161
log_parts = [
162
"Avg prompt throughput: %.1f tokens/s",
@@ -148,11 +167,11 @@ class LoggingStatLogger(StatLoggerBase):
167
"Prefix cache hit rate: %.1f%%",
168
]
169
log_args = [
151
- prompt_throughput,
152
- generation_throughput,
153
- scheduler_stats.num_running_reqs,
154
- scheduler_stats.num_waiting_reqs,
155
- scheduler_stats.kv_cache_usage * 100,
170
+ self.last_prompt_throughput,
171
+ self.last_generation_throughput,
172
+ self.last_scheduler_stats.num_running_reqs,
173
+ self.last_scheduler_stats.num_waiting_reqs,
174
+ self.last_scheduler_stats.kv_cache_usage * 100,
175
self.prefix_caching_metrics.hit_rate * 100,
176
]
177
if not self.mm_caching_metrics.empty:
@@ -160,8 +179,7 @@ class LoggingStatLogger(StatLoggerBase):
179
log_args.append(self.mm_caching_metrics.hit_rate * 100)
180
181
log_fn(
163
- "Engine %03d: " + ", ".join(log_parts),
164
- self.engine_index,
182
+ self.log_prefix + ", ".join(log_parts),
183
*log_args,
184
)
185
@@ -178,7 +196,114 @@ class LoggingStatLogger(StatLoggerBase):
196
)
197
198
181
-class PrometheusStatLogger(StatLoggerBase):
199
+class AggregatedLoggingStatLogger(LoggingStatLogger, AggregateStatLoggerBase):
200
+ def __init__(
201
+ self,
202
+ vllm_config: VllmConfig,
203
+ engine_indexes: list[int],
204
+ ):
205
+ self.engine_indexes = engine_indexes
206
+ self.last_scheduler_stats_dict: dict[int, SchedulerStats] = {
207
+ idx: SchedulerStats() for idx in self.engine_indexes
208
+ }
209
+ LoggingStatLogger.__init__(self, vllm_config, engine_index=-1)
210
+ self.aggregated = True
211
+
212
+ @property
213
+ def log_prefix(self):
214
+ return "{} Engines Aggregated: ".format(len(self.engine_indexes))
215
+
216
+ def record(
217
+ self,
218
+ scheduler_stats: SchedulerStats | None,
219
+ iteration_stats: IterationStats | None,
220
+ mm_cache_stats: MultiModalCacheStats | None = None,
221
+ engine_idx: int = 0,
222
+ ):
223
+ if engine_idx not in self.engine_indexes:
224
+ logger.warning("Unexpected engine_idx: %d", engine_idx)
225
+ return
226
+ LoggingStatLogger.record(
227
+ self,
228
+ scheduler_stats,
229
+ iteration_stats,
230
+ mm_cache_stats=mm_cache_stats,
231
+ engine_idx=engine_idx,
232
+ )
233
+ if scheduler_stats is not None:
234
+ self.last_scheduler_stats_dict[engine_idx] = scheduler_stats
235
+
236
+ def aggregate_scheduler_stats(self):
237
+ self.last_scheduler_stats = SchedulerStats()
238
+ for last_scheduler_stats in self.last_scheduler_stats_dict.values():
239
+ self.last_scheduler_stats.num_waiting_reqs += (
240
+ last_scheduler_stats.num_waiting_reqs
241
+ )
242
+ self.last_scheduler_stats.num_running_reqs += (
243
+ last_scheduler_stats.num_running_reqs
244
+ )
245
+ self.last_scheduler_stats.num_corrupted_reqs += (
246
+ last_scheduler_stats.num_corrupted_reqs
247
+ )
248
+ self.last_scheduler_stats.kv_cache_usage += (
249
+ last_scheduler_stats.kv_cache_usage
250
+ )
251
+ self.last_scheduler_stats.kv_cache_usage /= len(self.last_scheduler_stats_dict)
252
+
253
+ def log(self):
254
+ LoggingStatLogger.log(self)
255
+
256
+ def log_engine_initialized(self):
257
+ if self.vllm_config.cache_config.num_gpu_blocks:
258
+ logger.info(
259
+ "%d Engines: vllm cache_config_info with initialization "
260
+ "after num_gpu_blocks is: %d",
261
+ len(self.engine_indexes),
262
+ self.vllm_config.cache_config.num_gpu_blocks,
263
+ )
264
+
265
+
266
+class PerEngineStatLoggerAdapter(AggregateStatLoggerBase):
267
+ def __init__(
268
+ self,
269
+ vllm_config: VllmConfig,
270
+ engine_indexes: list[int],
271
+ per_engine_stat_logger_factory: PerEngineStatLoggerFactory,
272
+ ) -> None:
273
+ self.per_engine_stat_loggers = {}
274
+ self.engine_indexes = engine_indexes
275
+ for engine_index in engine_indexes:
276
+ self.per_engine_stat_loggers[engine_index] = per_engine_stat_logger_factory(
277
+ vllm_config, engine_index
278
+ )
279
+
280
+ def record(
281
+ self,
282
+ scheduler_stats: SchedulerStats | None,
283
+ iteration_stats: IterationStats | None,
284
+ mm_cache_stats: MultiModalCacheStats | None = None,
285
+ engine_idx: int = 0,
286
+ ):
287
+ if engine_idx not in self.per_engine_stat_loggers:
288
+ logger.warning("Unexpected engine_idx: %d", engine_idx)
289
+ return
290
+ self.per_engine_stat_loggers[engine_idx].record(
291
+ scheduler_stats,
292
+ iteration_stats,
293
+ mm_cache_stats=mm_cache_stats,
294
+ engine_idx=engine_idx,
295
+ )
296
+
297
+ def log(self):
298
+ for per_engine_stat_logger in self.per_engine_stat_loggers.values():
299
+ per_engine_stat_logger.log()
300
+
301
+ def log_engine_initialized(self):
302
+ for per_engine_stat_logger in self.per_engine_stat_loggers.values():
303
+ per_engine_stat_logger.log_engine_initialized()
304
+
305
+
306
+class PrometheusStatLogger(AggregateStatLoggerBase):
307
_gauge_cls = Gauge
308
_counter_cls = Counter
309
_histogram_cls = Histogram
@@ -189,6 +314,7 @@ class PrometheusStatLogger(StatLoggerBase):
314
):
315
if engine_indexes is None:
316
engine_indexes = [0]
317
+
318
self.engine_indexes = engine_indexes
319
320
unregister_vllm_metrics()
@@ -880,14 +1006,14 @@ class StatLoggerManager:
1006
engine_idxs: list[int] | None = None,
1007
custom_stat_loggers: list[StatLoggerFactory] | None = None,
1008
enable_default_loggers: bool = True,
1009
+ aggregate_engine_logging: bool = False,
1010
client_count: int = 1,
1011
):
885
- self.engine_idxs = engine_idxs if engine_idxs else [0]
886
-
887
- factories: list[StatLoggerFactory] = []
1012
+ self.engine_indexes = engine_idxs if engine_idxs else [0]
1013
+ self.stat_loggers: list[AggregateStatLoggerBase] = []
1014
+ stat_logger_factories: list[StatLoggerFactory] = []
1015
if custom_stat_loggers is not None:
889
- factories.extend(custom_stat_loggers)
890
-
1016
+ stat_logger_factories.extend(custom_stat_loggers)
1017
if enable_default_loggers and logger.isEnabledFor(logging.INFO):
1018
if client_count > 1:
1019
logger.warning(
@@ -895,27 +1021,35 @@ class StatLoggerManager:
1021
"disabling stats logging to avoid incomplete stats."
1022
)
1023
else:
898
- factories.append(LoggingStatLogger)
899
-
900
- # engine_idx: StatLogger
901
- self.per_engine_logger_dict: dict[int, list[StatLoggerBase]] = {}
902
- prometheus_factory = PrometheusStatLogger
903
- for engine_idx in self.engine_idxs:
904
- loggers: list[StatLoggerBase] = []
905
- for logger_factory in factories:
906
- # If we get a custom prometheus logger, use that
907
- # instead. This is typically used for the ray case.
908
- if isinstance(logger_factory, type) and issubclass(
909
- logger_factory, PrometheusStatLogger
910
- ):
911
- prometheus_factory = logger_factory
912
- continue
913
- loggers.append(logger_factory(vllm_config, engine_idx)) # type: ignore
914
- self.per_engine_logger_dict[engine_idx] = loggers
915
-
916
- # For Prometheus, need to share the metrics between EngineCores.
917
- # Each EngineCore's metrics are expressed as a unique label.
918
- self.prometheus_logger = prometheus_factory(vllm_config, engine_idxs)
1024
+ default_logger_factory = (
1025
+ AggregatedLoggingStatLogger
1026
+ if aggregate_engine_logging
1027
+ else LoggingStatLogger
1028
+ )
1029
+ stat_logger_factories.append(default_logger_factory)
1030
+ custom_prometheus_logger: bool = False
1031
+ for stat_logger_factory in stat_logger_factories:
1032
+ if isinstance(stat_logger_factory, type) and issubclass(
1033
+ stat_logger_factory, AggregateStatLoggerBase
1034
+ ):
1035
+ global_stat_logger = stat_logger_factory(
1036
+ vllm_config=vllm_config,
1037
+ engine_indexes=self.engine_indexes,
1038
+ )
1039
+ if isinstance(global_stat_logger, PrometheusStatLogger):
1040
+ custom_prometheus_logger = True
1041
+ else:
1042
+ # per engine logger
1043
+ global_stat_logger = PerEngineStatLoggerAdapter(
1044
+ vllm_config=vllm_config,
1045
+ engine_indexes=self.engine_indexes,
1046
+ per_engine_stat_logger_factory=stat_logger_factory, # type: ignore[arg-type]
1047
+ )
1048
+ self.stat_loggers.append(global_stat_logger)
1049
+ if not custom_prometheus_logger:
1050
+ self.stat_loggers.append(
1051
+ PrometheusStatLogger(vllm_config, self.engine_indexes)
1052
+ )
1053
1054
def record(
1055
self,
@@ -926,9 +1060,7 @@ class StatLoggerManager:
1060
):
1061
if engine_idx is None:
1062
engine_idx = 0
929
-
930
- per_engine_loggers = self.per_engine_logger_dict[engine_idx]
931
- for logger in per_engine_loggers:
1063
+ for logger in self.stat_loggers:
1064
logger.record(
1065
scheduler_stats,
1066
iteration_stats,
@@ -936,21 +1068,10 @@ class StatLoggerManager:
1068
engine_idx=engine_idx,
1069
)
1070
939
- self.prometheus_logger.record(
940
- scheduler_stats,
941
- iteration_stats,
942
- mm_cache_stats=mm_cache_stats,
943
- engine_idx=engine_idx,
944
- )
945
-
1071
def log(self):
947
- for per_engine_loggers in self.per_engine_logger_dict.values():
948
- for logger in per_engine_loggers:
949
- logger.log()
1072
+ for logger in self.stat_loggers:
1073
+ logger.log()
1074
1075
def log_engine_initialized(self):
952
- self.prometheus_logger.log_engine_initialized()
953
-
954
- for per_engine_loggers in self.per_engine_logger_dict.values():
955
- for logger in per_engine_loggers:
956
- logger.log_engine_initialized()
1076
+ for agg_logger in self.stat_loggers:
1077
+ agg_logger.log_engine_initialized()