@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1543914c04
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] Improve TP>1 Error Handling + Stack Trace (#11721)

Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>

Robert Shaw committed Jan 3, 2025 at 16:29 UTC 1543914c04697fb252e4468b7c9d14be512b050a
4 files changed +40 -21
vllm/v1/engine/async_llm.py
-16
@@ -1,6 +1,5 @@
1 import asyncio
2 import os
3 -import signal
3 from typing import AsyncGenerator, Dict, List, Mapping, Optional, Type, Union
4
5 from vllm.config import ModelConfig, VllmConfig
@@ -42,21 +41,6 @@ class AsyncLLM(EngineClient):
41 start_engine_loop: bool = True,
42 ) -> None:
43
45 - # The child processes will send SIGQUIT when unrecoverable
46 - # errors happen. We kill the process tree here so that the
47 - # stack trace is very evident.
48 - # TODO: rather than killing the main process, we should
49 - # figure out how to raise an AsyncEngineDeadError and
50 - # handle at the API server level so we can return a better
51 - # error code to the clients calling VLLM.
52 - def sigquit_handler(signum, frame):
53 - logger.fatal(
54 - "AsyncLLM got SIGQUIT from worker processes, shutting "
55 - "down. See stack trace above for root cause issue.")
56 - kill_process_tree(os.getpid())
57 -
58 - signal.signal(signal.SIGQUIT, sigquit_handler)
59 -
44 assert start_engine_loop
45
46 self.log_requests = log_requests
vllm/v1/engine/core.py
+1 -1
@@ -198,7 +198,7 @@ class EngineCoreProc(EngineCore):
198 except Exception:
199 traceback = get_exception_traceback()
200 logger.error("EngineCore hit an exception: %s", traceback)
201 - parent_process.send_signal(signal.SIGQUIT)
201 + parent_process.send_signal(signal.SIGUSR1)
202
203 finally:
204 if engine_core is not None:
vllm/v1/engine/core_client.py
+18 -1
@@ -1,3 +1,5 @@
1 +import os
2 +import signal
3 import weakref
4 from abc import ABC, abstractmethod
5 from typing import List, Type
@@ -8,7 +10,8 @@ import zmq.asyncio
10
11 from vllm.config import VllmConfig
12 from vllm.logger import init_logger
11 -from vllm.utils import get_open_zmq_ipc_path, make_zmq_socket
13 +from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
14 + make_zmq_socket)
15 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
16 EngineCoreProfile, EngineCoreRequest,
17 EngineCoreRequestType, EngineCoreRequestUnion)
@@ -134,6 +137,20 @@ class MPClient(EngineCoreClient):
137 executor_class: Type[Executor],
138 log_stats: bool = False,
139 ):
140 + # The child processes will send SIGUSR1 when unrecoverable
141 + # errors happen. We kill the process tree here so that the
142 + # stack trace is very evident.
143 + # TODO(rob): rather than killing the main process, we should
144 + # figure out how to raise an AsyncEngineDeadError and
145 + # handle at the API server level so we can return a better
146 + # error code to the clients calling VLLM.
147 + def sigusr1_handler(signum, frame):
148 + logger.fatal("Got fatal signal from worker processes, shutting "
149 + "down. See stack trace above for root cause issue.")
150 + kill_process_tree(os.getpid())
151 +
152 + signal.signal(signal.SIGUSR1, sigusr1_handler)
153 +
154 # Serialization setup.
155 self.encoder = PickleEncoder()
156 self.decoder = msgspec.msgpack.Decoder(EngineCoreOutputs)
vllm/v1/executor/multiproc_executor.py
+21 -3
@@ -9,6 +9,7 @@ from enum import Enum, auto
9 from multiprocessing.process import BaseProcess
10 from typing import Any, Dict, List, Optional, Tuple
11
12 +import psutil
13 import zmq
14
15 from vllm.config import VllmConfig
@@ -38,6 +39,19 @@ class MultiprocExecutor(Executor):
39 # and ensure workers will be terminated.
40 self._finalizer = weakref.finalize(self, self.shutdown)
41
42 + # The child processes will send SIGUSR1 when unrecoverable
43 + # errors happen.
44 + def sigusr1_handler(signum, frame):
45 + logger.fatal(
46 + "MulitprocExecutor got fatal signal from worker processes, "
47 + "shutting down. See stack trace above for root cause issue.")
48 + # Propagate error up to parent process.
49 + parent_process = psutil.Process().parent()
50 + parent_process.send_signal(signal.SIGUSR1)
51 + self.shutdown()
52 +
53 + signal.signal(signal.SIGUSR1, sigusr1_handler)
54 +
55 self.vllm_config = vllm_config
56 self.parallel_config = vllm_config.parallel_config
57
@@ -335,8 +349,11 @@ class WorkerProc:
349 except SystemExit:
350 logger.debug("Worker interrupted.")
351
338 - except BaseException as e:
339 - logger.exception(e)
352 + except Exception:
353 + # worker_busy_loop sends exceptions exceptons to Executor
354 + # for shutdown, but if there is an error in startup or an
355 + # error with IPC itself, we need to alert the parent.
356 + psutil.Process().parent().send_signal(signal.SIGUSR1)
357 raise
358
359 finally:
@@ -377,9 +394,10 @@ class WorkerProc:
394
395 try:
396 output = getattr(self.worker, method)(*args, **kwargs)
380 - except BaseException as e:
397 + except Exception as e:
398 self.worker_response_mq.enqueue(
399 (WorkerProc.ResponseStatus.FAILURE, e))
400 + logger.exception("WorkerProc hit an exception: %s", exc_info=e)
401 continue
402
403 self.worker_response_mq.enqueue(