@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e6680f9e25

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 22:35 UTC

[Bugfix] Add log prefix in non-dp mode engine core (#21889)

Signed-off-by: wuhang <wuhang6@huawei.com>

wuhang committed Aug 1, 2025 at 17:04 UTC e6680f9e25a433bcd754181705e72034ce6c470c

6 files changed +75 -81

vllm/entrypoints/cli/serve.py

+2 -9

@@ -2,9 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 import argparse

5 -import os

5 import signal

7 -import sys

6 from typing import Optional

8 import uvloop

@@ -18,10 +16,9 @@ from vllm.entrypoints.openai.cli_args import (make_arg_parser,

16 validate_parsed_serve_args)

17 from vllm.entrypoints.utils import (VLLM_SUBCMD_PARSER_EPILOG,

18 show_filtered_argument_or_group_from_help)

21 -from vllm.executor.multiproc_worker_utils import _add_prefix

19 from vllm.logger import init_logger

20 from vllm.usage.usage_lib import UsageContext

24 -from vllm.utils import FlexibleArgumentParser, get_tcp_uri

21 +from vllm.utils import FlexibleArgumentParser, decorate_logs, get_tcp_uri

22 from vllm.v1.engine.core import EngineCoreProc

23 from vllm.v1.engine.utils import CoreEngineProcManager, launch_core_engines

24 from vllm.v1.executor.abstract import Executor

@@ -229,11 +226,7 @@ def run_api_server_worker_proc(listen_address,

226 """Entrypoint for individual API server worker processes."""

227

228 # Add process-specific prefix to stdout and stderr.

232 - from multiprocessing import current_process

233 - process_name = current_process().name

234 - pid = os.getpid()

235 - _add_prefix(sys.stdout, process_name, pid)

236 - _add_prefix(sys.stderr, process_name, pid)

229 + decorate_logs()

230

231 uvloop.run(

232 run_server_worker(listen_address, sock, args, client_config,

vllm/entrypoints/openai/api_server.py

+4 -8

@@ -11,7 +11,6 @@ import multiprocessing

11 import os

12 import signal

13 import socket

14 -import sys

14 import tempfile

15 import uuid

16 from argparse import Namespace

@@ -95,15 +94,15 @@ from vllm.entrypoints.openai.serving_transcription import (

94 from vllm.entrypoints.openai.tool_parsers import ToolParserManager

95 from vllm.entrypoints.utils import (cli_env_setup, load_aware_call,

96 log_non_default_args, with_cancellation)

98 -from vllm.executor.multiproc_worker_utils import _add_prefix

97 from vllm.logger import init_logger

98 from vllm.reasoning import ReasoningParserManager

99 from vllm.transformers_utils.config import (

100 maybe_register_config_serialize_by_value)

101 from vllm.transformers_utils.tokenizer import MistralTokenizer

102 from vllm.usage.usage_lib import UsageContext

105 -from vllm.utils import (Device, FlexibleArgumentParser, get_open_zmq_ipc_path,

106 - is_valid_ipv6_address, set_process_title, set_ulimit)

103 +from vllm.utils import (Device, FlexibleArgumentParser, decorate_logs,

104 + get_open_zmq_ipc_path, is_valid_ipv6_address,

105 + set_process_title, set_ulimit)

106 from vllm.v1.metrics.prometheus import get_prometheus_registry

107 from vllm.version import __version__ as VLLM_VERSION

108

@@ -1808,10 +1807,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:

1807 """Run a single-worker API server."""

1808

1809 # Add process-specific prefix to stdout and stderr.

1811 - process_name = "APIServer"

1812 - pid = os.getpid()

1813 - _add_prefix(sys.stdout, process_name, pid)

1814 - _add_prefix(sys.stderr, process_name, pid)

1810 + decorate_logs("APIServer")

1811

1812 listen_address, sock = setup_server(args)

1813 await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)

vllm/executor/multiproc_worker_utils.py

+4 -38

@@ -3,21 +3,20 @@

4 import asyncio

5 import os

6 -import sys

6 import threading

7 import uuid

8 from dataclasses import dataclass

9 from multiprocessing import Queue

10 from multiprocessing.connection import wait

11 from multiprocessing.process import BaseProcess

13 -from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,

14 - TypeVar, Union)

12 +from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union

14 import torch

16 from vllm.config import VllmConfig

17 from vllm.logger import init_logger

20 -from vllm.utils import _maybe_force_spawn, get_mp_context, run_method

18 +from vllm.utils import (_maybe_force_spawn, decorate_logs, get_mp_context,

19 + run_method)

21 logger = init_logger(__name__)

@@ -25,10 +24,6 @@ T = TypeVar('T')

25 _TERMINATE = "TERMINATE" # sentinel

28 -# ANSI color codes

29 -CYAN = '\033[1;36m'

30 -RESET = '\033[0;0m'

31 -

27 JOIN_TIMEOUT_S = 2

@@ -213,9 +208,7 @@ def _run_worker_process(

208

209 # Add process-specific prefix to stdout and stderr

210 process_name = get_mp_context().current_process().name

216 - pid = os.getpid()

217 - _add_prefix(sys.stdout, process_name, pid)

218 - _add_prefix(sys.stderr, process_name, pid)

211 + decorate_logs(process_name)

212

213 # Initialize worker

214 worker = worker_factory(vllm_config, rank)

@@ -260,33 +253,6 @@ def _run_worker_process(

253 logger.info("Worker exiting")

254

255

263 -def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:

264 - """Prepend each output line with process-specific prefix"""

265 -

266 - prefix = f"{CYAN}({worker_name} pid={pid}){RESET} "

267 - file_write = file.write

268 -

269 - def write_with_prefix(s: str):

270 - if not s:

271 - return

272 - if file.start_new_line: # type: ignore[attr-defined]

273 - file_write(prefix)

274 - idx = 0

275 - while (next_idx := s.find('\n', idx)) != -1:

276 - next_idx += 1

277 - file_write(s[idx:next_idx])

278 - if next_idx == len(s):

279 - file.start_new_line = True # type: ignore[attr-defined]

280 - return

281 - file_write(prefix)

282 - idx = next_idx

283 - file_write(s[idx:])

284 - file.start_new_line = False # type: ignore[attr-defined]

285 -

286 - file.start_new_line = True # type: ignore[attr-defined]

287 - file.write = write_with_prefix # type: ignore[method-assign]

288 -

289 -

256 def set_multiprocessing_worker_envs(parallel_config):

257 """ Set up environment variables that should be used when there are workers

258 in a multiprocessing environment. This should be called by the parent

vllm/utils/__init__.py

+54 -1

@@ -47,7 +47,7 @@ from dataclasses import dataclass, field

47 from functools import cache, lru_cache, partial, wraps

48 from types import MappingProxyType

49 from typing import (TYPE_CHECKING, Any, Callable, Generic, Literal, NamedTuple,

50 - Optional, Tuple, TypeVar, Union, cast, overload)

50 + Optional, TextIO, Tuple, TypeVar, Union, cast, overload)

51 from urllib.parse import urlparse

52 from uuid import uuid4

@@ -167,6 +167,10 @@ GB_bytes = 1_000_000_000

167 GiB_bytes = 1 << 30

168 """The number of bytes in one gibibyte (GiB)."""

169

170 +# ANSI color codes

171 +CYAN = '\033[1;36m'

172 +RESET = '\033[0;0m'

173 +

174 STR_DTYPE_TO_TORCH_DTYPE = {

175 "half": torch.half,

176 "bfloat16": torch.bfloat16,

@@ -3258,3 +3262,52 @@ def set_process_title(name: str,

3262 else:

3263 name = f"{envs.VLLM_PROCESS_NAME_PREFIX}::{name}"

3264 setproctitle.setproctitle(name)

3265 +

3266 +

3267 +def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:

3268 + """Prepend each output line with process-specific prefix"""

3269 +

3270 + prefix = f"{CYAN}({worker_name} pid={pid}){RESET} "

3271 + file_write = file.write

3272 +

3273 + def write_with_prefix(s: str):

3274 + if not s:

3275 + return

3276 + if file.start_new_line: # type: ignore[attr-defined]

3277 + file_write(prefix)

3278 + idx = 0

3279 + while (next_idx := s.find('\n', idx)) != -1:

3280 + next_idx += 1

3281 + file_write(s[idx:next_idx])

3282 + if next_idx == len(s):

3283 + file.start_new_line = True # type: ignore[attr-defined]

3284 + return

3285 + file_write(prefix)

3286 + idx = next_idx

3287 + file_write(s[idx:])

3288 + file.start_new_line = False # type: ignore[attr-defined]

3289 +

3290 + file.start_new_line = True # type: ignore[attr-defined]

3291 + file.write = write_with_prefix # type: ignore[method-assign]

3292 +

3293 +

3294 +def decorate_logs(process_name: Optional[str] = None) -> None:

3295 + """

3296 + Adds a process-specific prefix to each line of output written to stdout and

3297 + stderr.

3298 +

3299 + This function is intended to be called before initializing the api_server,

3300 + engine_core, or worker classes, so that all subsequent output from the

3301 + process is prefixed with the process name and PID. This helps distinguish

3302 + log output from different processes in multi-process environments.

3303 +

3304 + Args:

3305 + process_name: Optional; the name of the process to use in the prefix.

3306 + If not provided, the current process name from the multiprocessing

3307 + context is used.

3308 + """

3309 + if process_name is None:

3310 + process_name = get_mp_context().current_process().name

3311 + pid = os.getpid()

3312 + _add_prefix(sys.stdout, process_name, pid)

3313 + _add_prefix(sys.stderr, process_name, pid)

vllm/v1/engine/core.py

+4 -18

@@ -3,7 +3,6 @@

3 import os

4 import queue

5 import signal

6 -import sys

6 import threading

7 import time

8 from collections import deque

@@ -19,15 +18,14 @@ import zmq

19 from vllm.config import ParallelConfig, VllmConfig

20 from vllm.distributed import stateless_destroy_torch_distributed_process_group

22 -from vllm.executor.multiproc_worker_utils import _add_prefix

21 from vllm.logger import init_logger

22 from vllm.logging_utils.dump_input import dump_engine_exception

23 from vllm.lora.request import LoRARequest

24 from vllm.tasks import POOLING_TASKS, SupportedTask

25 from vllm.transformers_utils.config import (

26 maybe_register_config_serialize_by_value)

29 -from vllm.utils import (make_zmq_socket, resolve_obj_by_qualname,

30 - set_process_title)

27 +from vllm.utils import (decorate_logs, make_zmq_socket,

28 + resolve_obj_by_qualname, set_process_title)

29 from vllm.v1.core.kv_cache_utils import (get_kv_cache_config,

30 unify_kv_cache_configs)

31 from vllm.v1.core.sched.interface import SchedulerInterface

@@ -649,12 +647,14 @@ class EngineCoreProc(EngineCore):

647 "vllm_config"].parallel_config

648 if parallel_config.data_parallel_size > 1 or dp_rank > 0:

649 set_process_title("DPEngineCore", str(dp_rank))

650 + decorate_logs()

651 # Set data parallel rank for this engine process.

652 parallel_config.data_parallel_rank = dp_rank

653 parallel_config.data_parallel_rank_local = local_dp_rank

654 engine_core = DPEngineCoreProc(*args, **kwargs)

655 else:

656 set_process_title("EngineCore")

657 + decorate_logs()

658 engine_core = EngineCoreProc(*args, **kwargs)

659

660 engine_core.run_busy_loop()

@@ -905,8 +905,6 @@ class DPEngineCoreProc(EngineCoreProc):

905 log_stats: bool,

906 client_handshake_address: Optional[str] = None,

907 ):

908 - self._decorate_logs()

909 -

908 # Counts forward-passes of the model so that we can synchronize

909 # finished with DP peers every N steps.

910 self.counter = 0

@@ -919,15 +917,6 @@ class DPEngineCoreProc(EngineCoreProc):

917 executor_class, log_stats, client_handshake_address,

918 dp_rank)

919

922 - def _decorate_logs(self):

923 - # Add process-specific prefix to stdout and stderr before

924 - # we initialize the engine.

925 - from multiprocessing import current_process

926 - process_name = current_process().name

927 - pid = os.getpid()

928 - _add_prefix(sys.stdout, process_name, pid)

929 - _add_prefix(sys.stderr, process_name, pid)

930 -

920 def _init_data_parallel(self, vllm_config: VllmConfig):

921

922 # Configure GPUs and stateless process group for data parallel.

@@ -1149,9 +1138,6 @@ class DPEngineCoreActor(DPEngineCoreProc):

1138 f"{(local_dp_rank + 1) * world_size}) "

1139 f"base value: \"{os.getenv(device_control_env_var)}\"") from e

1140

1152 - def _decorate_logs(self):

1153 - pass

1154 -

1141 @contextmanager

1142 def _perform_handshakes(self, handshake_address: str, identity: bytes,

1143 local_client: bool, vllm_config: VllmConfig,

vllm/v1/executor/multiproc_executor.py

+7 -7

@@ -4,7 +4,6 @@ import multiprocessing

4 import os

5 import pickle

6 import signal

7 -import sys

7 import threading

8 import time

9 import traceback

@@ -28,10 +27,11 @@ from vllm.distributed.device_communicators.shm_broadcast import (Handle,

27 MessageQueue)

28 from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator

29 from vllm.executor.multiproc_worker_utils import (

31 - _add_prefix, set_multiprocessing_worker_envs)

30 + set_multiprocessing_worker_envs)

31 from vllm.logger import init_logger

33 -from vllm.utils import (get_distributed_init_method, get_loopback_ip,

34 - get_mp_context, get_open_port, set_process_title)

32 +from vllm.utils import (decorate_logs, get_distributed_init_method,

33 + get_loopback_ip, get_mp_context, get_open_port,

34 + set_process_title)

35 from vllm.v1.executor.abstract import Executor, FailureCallback

36 from vllm.v1.outputs import ModelRunnerOutput

37 from vllm.worker.worker_base import WorkerWrapperBase

@@ -382,11 +382,11 @@ class WorkerProc:

382 pp_str = f"PP{rank // tp_size}" if pp_size > 1 else ""

383 tp_str = f"TP{rank % tp_size}" if tp_size > 1 else ""

384 suffix = f"{pp_str}{'_' if pp_str and tp_str else ''}{tp_str}"

385 + process_name = "VllmWorker"

386 if suffix:

387 set_process_title(suffix, append=True)

387 - pid = os.getpid()

388 - _add_prefix(sys.stdout, f"VllmWorker rank={rank}", pid)

389 - _add_prefix(sys.stderr, f"VllmWorker rank={rank}", pid)

388 + process_name = f"{process_name} {suffix}"

389 + decorate_logs(process_name)

390

391 # Initialize MessageQueue for receiving SchedulerOutput

392 self.rpc_broadcast_mq = MessageQueue.create_from_handle(