A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[V1][Bugfix]: vllm v1 verison metric num_gpu_blocks is None (#15755)
Signed-off-by: rongfu.leng <rongfu.leng@daocloud.io>
rongfu.leng committed
Apr 30, 2025 at 18:20 UTC
d8037867313eef09779b265ea4bace76d2d118ba
4 files changed
+37
-11
vllm/v1/engine/async_llm.py
+2
-1
@@ -120,7 +120,8 @@ class AsyncLLM(EngineClient):
120
executor_class=executor_class,
121
log_stats=self.log_stats,
122
)
123
-
123
+ for stat_logger in self.stat_loggers[0]:
124
+ stat_logger.log_engine_initialized()
125
self.output_handler: Optional[asyncio.Task] = None
126
try:
127
# Start output handler eagerly if we are in the asyncio eventloop.
vllm/v1/engine/core.py
+8
-1
@@ -1,4 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
+import json
3
import os
4
import queue
5
import signal
@@ -116,6 +117,7 @@ class EngineCore:
117
logger.info("Batch queue is enabled with size %d",
118
self.batch_queue_size)
119
self.batch_queue = queue.Queue(self.batch_queue_size)
120
+ self.vllm_config = vllm_config
121
122
def _initialize_kv_caches(
123
self, vllm_config: VllmConfig) -> tuple[int, int, KVCacheConfig]:
@@ -507,7 +509,12 @@ class EngineCoreProc(EngineCore):
509
bind=False) as socket:
510
511
# Send ready message to front-end once input socket is connected.
510
- socket.send(b'READY')
512
+ message_dict = {
513
+ 'type': 'READY',
514
+ 'num_gpu_blocks': self.vllm_config.cache_config.num_gpu_blocks,
515
+ }
516
+ message = json.dumps(message_dict).encode('utf-8')
517
+ socket.send(message)
518
519
while True:
520
# (RequestType, RequestData)
vllm/v1/engine/core_client.py
+10
-3
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
import asyncio
3
import contextlib
4
+import json
5
import queue
6
import uuid
7
import weakref
@@ -362,6 +363,7 @@ class MPClient(EngineCoreClient):
363
executor_class: type[Executor],
364
log_stats: bool,
365
):
366
+ self.vllm_config = vllm_config
367
# Serialization setup.
368
self.encoder = MsgpackEncoder()
369
self.decoder = MsgpackDecoder(EngineCoreOutputs)
@@ -430,14 +432,19 @@ class MPClient(EngineCoreClient):
432
raise RuntimeError("Engine core initialization failed. "
433
"See root cause above.")
434
433
- eng_id_bytes, msg = sync_input_socket.recv_multipart()
435
+ eng_id_bytes, data = sync_input_socket.recv_multipart()
436
eng_id = int.from_bytes(eng_id_bytes, byteorder="little")
437
if eng_id not in identities:
438
raise RuntimeError(f"Unexpected or duplicate engine: {eng_id}")
437
- if msg != b'READY':
438
- raise RuntimeError(f"Engine {eng_id} failed: {msg.decode()}")
439
+ message_dict = json.loads(data.decode('utf-8'))
440
+ if message_dict['type'] != 'READY':
441
+ raise RuntimeError(f"Engine {eng_id} failed: {data.decode()}")
442
logger.info("Core engine process %d ready.", eng_id)
443
identities.discard(eng_id)
444
+ # Setup KV cache config with initialization state from
445
+ # engine core process.
446
+ self.vllm_config.cache_config.num_gpu_blocks = message_dict[
447
+ 'num_gpu_blocks']
448
449
def _init_core_engines(
450
self,
vllm/v1/metrics/loggers.py
+17
-6
@@ -39,6 +39,10 @@ class StatLoggerBase(ABC):
39
iteration_stats: Optional[IterationStats]):
40
...
41
42
+ @abstractmethod
43
+ def log_engine_initialized(self):
44
+ ...
45
+
46
def log(self): # noqa
47
pass
48
@@ -47,6 +51,7 @@ class LoggingStatLogger(StatLoggerBase):
51
52
def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
53
self.engine_index = engine_index
54
+ self.vllm_config = vllm_config
55
self._reset(time.monotonic())
56
self.last_scheduler_stats = SchedulerStats()
57
# Prefix cache metrics. This cannot be reset.
@@ -127,12 +132,19 @@ class LoggingStatLogger(StatLoggerBase):
132
if scheduler_stats.spec_decoding_stats is not None:
133
self.spec_decoding_logging.log(log_fn=log_fn)
134
135
+ def log_engine_initialized(self):
136
+ logger.info(
137
+ "vllm cache_config_info with initialization " \
138
+ "after num_gpu_blocks is: %d",
139
+ self.vllm_config.cache_config.num_gpu_blocks)
140
+
141
142
class PrometheusStatLogger(StatLoggerBase):
143
144
def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
145
self._unregister_vllm_metrics()
135
-
146
+ self.vllm_config = vllm_config
147
+ self.engine_index = engine_index
148
# Use this flag to hide metrics that were deprecated in
149
# a previous release and which will be removed future
150
self.show_hidden_metrics = \
@@ -342,13 +354,9 @@ class PrometheusStatLogger(StatLoggerBase):
354
self.labelname_running_lora_adapters,
355
])
356
345
- #
346
- # Cache config info metric
347
- #
348
- self.log_metrics_info("cache_config", vllm_config.cache_config)
349
-
357
def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):
358
metrics_info = config_obj.metrics_info()
359
+ metrics_info["engine"] = self.engine_index
360
361
name, documentation = None, None
362
if type == "cache_config":
@@ -442,6 +450,9 @@ class PrometheusStatLogger(StatLoggerBase):
450
if hasattr(collector, "_name") and "vllm" in collector._name:
451
prometheus_client.REGISTRY.unregister(collector)
452
453
+ def log_engine_initialized(self):
454
+ self.log_metrics_info("cache_config", self.vllm_config.cache_config)
455
+
456
457
def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:
458
"""