A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 20:30 UTC
[BugFix] Fix multiple/duplicate stdout prefixes (#36822)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Mar 11, 2026 at 21:23 UTC
36735fd77224467e6580f3bd48eb32d4fca8c72e
4 files changed
+21
-37
vllm/entrypoints/cli/serve.py
-2
@@ -21,7 +21,6 @@ from vllm.usage.usage_lib import UsageContext
21
from vllm.utils.argparse_utils import FlexibleArgumentParser
22
from vllm.utils.network_utils import get_tcp_uri
23
from vllm.utils.system_utils import decorate_logs, set_process_title
24
-from vllm.v1.engine.core import EngineCoreProc
24
from vllm.v1.engine.utils import CoreEngineProcManager, launch_core_engines
25
from vllm.v1.executor import Executor
26
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
@@ -210,7 +209,6 @@ def run_headless(args: argparse.Namespace):
209
210
# Create the engines.
211
engine_manager = CoreEngineProcManager(
213
- target_fn=EngineCoreProc.run_engine_core,
212
local_engine_count=local_engine_count,
213
start_index=vllm_config.parallel_config.data_parallel_rank,
214
local_start_index=0,
vllm/utils/system_utils.py
+3
-1
@@ -204,7 +204,8 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
204
prefix = f"({worker_name} pid={pid}) "
205
else:
206
prefix = f"{CYAN}({worker_name} pid={pid}){RESET} "
207
- file_write = file.write
207
+ # Use the original write to avoid nesting prefixes on repeated calls.
208
+ file_write = getattr(file, "_original_write", file.write)
209
210
def write_with_prefix(s: str):
211
if not s:
@@ -224,6 +225,7 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
225
file.start_new_line = False # type: ignore[attr-defined]
226
227
file.start_new_line = True # type: ignore[attr-defined]
228
+ file._original_write = file_write # type: ignore[attr-defined]
229
file.write = write_with_prefix # type: ignore[method-assign]
230
231
vllm/v1/engine/core.py
+4
-12
@@ -1045,19 +1045,11 @@ class EngineCoreProc(EngineCore):
1045
data_parallel = parallel_config.data_parallel_size > 1 or dp_rank > 0
1046
if data_parallel:
1047
parallel_config.data_parallel_rank_local = local_dp_rank
1048
- maybe_init_worker_tracer(
1049
- instrumenting_module_name="vllm.engine_core",
1050
- process_kind="engine_core",
1051
- process_name=f"EngineCore_DP{dp_rank}",
1052
- )
1053
- set_process_title("EngineCore", f"DP{dp_rank}")
1048
+ process_title = f"EngineCore_DP{dp_rank}"
1049
else:
1055
- maybe_init_worker_tracer(
1056
- instrumenting_module_name="vllm.engine_core",
1057
- process_kind="engine_core",
1058
- process_name="EngineCore",
1059
- )
1060
- set_process_title("EngineCore")
1050
+ process_title = "EngineCore"
1051
+ set_process_title(process_title)
1052
+ maybe_init_worker_tracer("vllm.engine_core", "engine_core", process_title)
1053
decorate_logs()
1054
1055
if data_parallel and vllm_config.kv_transfer_config is not None:
vllm/v1/engine/utils.py
+14
-22
@@ -4,7 +4,7 @@
4
import contextlib
5
import os
6
import weakref
7
-from collections.abc import Callable, Iterator
7
+from collections.abc import Iterator
8
from dataclasses import dataclass
9
from enum import Enum, auto
10
from multiprocessing import Process, connection
@@ -85,7 +85,6 @@ class CoreEngineProcManager:
85
86
def __init__(
87
self,
88
- target_fn: Callable,
88
local_engine_count: int,
89
start_index: int,
90
local_start_index: int,
@@ -108,6 +107,10 @@ class CoreEngineProcManager:
107
if client_handshake_address:
108
common_kwargs["client_handshake_address"] = client_handshake_address
109
110
+ is_dp = vllm_config.parallel_config.data_parallel_size > 1
111
+
112
+ from vllm.v1.engine.core import EngineCoreProc
113
+
114
self.processes: list[BaseProcess] = []
115
local_dp_ranks = []
116
for index in range(local_engine_count):
@@ -118,35 +121,27 @@ class CoreEngineProcManager:
121
local_dp_ranks.append(local_index)
122
self.processes.append(
123
context.Process(
121
- target=target_fn,
122
- name=f"EngineCore_DP{global_index}",
124
+ target=EngineCoreProc.run_engine_core,
125
+ name=f"EngineCore_DP{global_index}" if is_dp else "EngineCore",
126
kwargs=common_kwargs
124
- | {
125
- "dp_rank": global_index,
126
- "local_dp_rank": local_index,
127
- },
127
+ | {"dp_rank": global_index, "local_dp_rank": local_index},
128
)
129
)
130
131
self._finalizer = weakref.finalize(self, shutdown, self.processes)
132
133
- data_parallel = vllm_config.parallel_config.data_parallel_size > 1
133
try:
134
for proc, local_dp_rank in zip(self.processes, local_dp_ranks):
135
# Adjust device control in DP for non-CUDA platforms
136
# as well as external and ray launchers
137
# For CUDA platforms, we use torch.cuda.set_device()
139
- with (
140
- set_device_control_env_var(vllm_config, local_dp_rank)
141
- if (
142
- data_parallel
143
- and (
144
- not current_platform.is_cuda_alike()
145
- or vllm_config.parallel_config.use_ray
146
- )
147
- )
148
- else contextlib.nullcontext()
138
+ if is_dp and (
139
+ not current_platform.is_cuda_alike()
140
+ or vllm_config.parallel_config.use_ray
141
):
142
+ with set_device_control_env_var(vllm_config, local_dp_rank):
143
+ proc.start()
144
+ else:
145
proc.start()
146
finally:
147
# Kill other procs if not all are running.
@@ -926,12 +921,9 @@ def launch_core_engines(
921
with zmq_socket_ctx(
922
local_handshake_address, zmq.ROUTER, bind=True
923
) as handshake_socket:
929
- from vllm.v1.engine.core import EngineCoreProc
930
-
924
# Start local engines.
925
if local_engine_count:
926
local_engine_manager = CoreEngineProcManager(
934
- EngineCoreProc.run_engine_core,
927
vllm_config=vllm_config,
928
executor_class=executor_class,
929
log_stats=log_stats,