A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] Improve TP>1 Error Handling + Stack Trace (#11721)
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Robert Shaw committed
Jan 3, 2025 at 16:29 UTC
1543914c04697fb252e4468b7c9d14be512b050a
4 files changed
+40
-21
vllm/v1/engine/async_llm.py
-16
@@ -1,6 +1,5 @@
1
import asyncio
2
import os
3
-import signal
3
from typing import AsyncGenerator, Dict, List, Mapping, Optional, Type, Union
4
5
from vllm.config import ModelConfig, VllmConfig
@@ -42,21 +41,6 @@ class AsyncLLM(EngineClient):
41
start_engine_loop: bool = True,
42
) -> None:
43
45
- # The child processes will send SIGQUIT when unrecoverable
46
- # errors happen. We kill the process tree here so that the
47
- # stack trace is very evident.
48
- # TODO: rather than killing the main process, we should
49
- # figure out how to raise an AsyncEngineDeadError and
50
- # handle at the API server level so we can return a better
51
- # error code to the clients calling VLLM.
52
- def sigquit_handler(signum, frame):
53
- logger.fatal(
54
- "AsyncLLM got SIGQUIT from worker processes, shutting "
55
- "down. See stack trace above for root cause issue.")
56
- kill_process_tree(os.getpid())
57
-
58
- signal.signal(signal.SIGQUIT, sigquit_handler)
59
-
44
assert start_engine_loop
45
46
self.log_requests = log_requests
vllm/v1/engine/core.py
+1
-1
@@ -198,7 +198,7 @@ class EngineCoreProc(EngineCore):
198
except Exception:
199
traceback = get_exception_traceback()
200
logger.error("EngineCore hit an exception: %s", traceback)
201
- parent_process.send_signal(signal.SIGQUIT)
201
+ parent_process.send_signal(signal.SIGUSR1)
202
203
finally:
204
if engine_core is not None:
vllm/v1/engine/core_client.py
+18
-1
@@ -1,3 +1,5 @@
1
+import os
2
+import signal
3
import weakref
4
from abc import ABC, abstractmethod
5
from typing import List, Type
@@ -8,7 +10,8 @@ import zmq.asyncio
10
11
from vllm.config import VllmConfig
12
from vllm.logger import init_logger
11
-from vllm.utils import get_open_zmq_ipc_path, make_zmq_socket
13
+from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
14
+ make_zmq_socket)
15
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
16
EngineCoreProfile, EngineCoreRequest,
17
EngineCoreRequestType, EngineCoreRequestUnion)
@@ -134,6 +137,20 @@ class MPClient(EngineCoreClient):
137
executor_class: Type[Executor],
138
log_stats: bool = False,
139
):
140
+ # The child processes will send SIGUSR1 when unrecoverable
141
+ # errors happen. We kill the process tree here so that the
142
+ # stack trace is very evident.
143
+ # TODO(rob): rather than killing the main process, we should
144
+ # figure out how to raise an AsyncEngineDeadError and
145
+ # handle at the API server level so we can return a better
146
+ # error code to the clients calling VLLM.
147
+ def sigusr1_handler(signum, frame):
148
+ logger.fatal("Got fatal signal from worker processes, shutting "
149
+ "down. See stack trace above for root cause issue.")
150
+ kill_process_tree(os.getpid())
151
+
152
+ signal.signal(signal.SIGUSR1, sigusr1_handler)
153
+
154
# Serialization setup.
155
self.encoder = PickleEncoder()
156
self.decoder = msgspec.msgpack.Decoder(EngineCoreOutputs)
vllm/v1/executor/multiproc_executor.py
+21
-3
@@ -9,6 +9,7 @@ from enum import Enum, auto
9
from multiprocessing.process import BaseProcess
10
from typing import Any, Dict, List, Optional, Tuple
11
12
+import psutil
13
import zmq
14
15
from vllm.config import VllmConfig
@@ -38,6 +39,19 @@ class MultiprocExecutor(Executor):
39
# and ensure workers will be terminated.
40
self._finalizer = weakref.finalize(self, self.shutdown)
41
42
+ # The child processes will send SIGUSR1 when unrecoverable
43
+ # errors happen.
44
+ def sigusr1_handler(signum, frame):
45
+ logger.fatal(
46
+ "MulitprocExecutor got fatal signal from worker processes, "
47
+ "shutting down. See stack trace above for root cause issue.")
48
+ # Propagate error up to parent process.
49
+ parent_process = psutil.Process().parent()
50
+ parent_process.send_signal(signal.SIGUSR1)
51
+ self.shutdown()
52
+
53
+ signal.signal(signal.SIGUSR1, sigusr1_handler)
54
+
55
self.vllm_config = vllm_config
56
self.parallel_config = vllm_config.parallel_config
57
@@ -335,8 +349,11 @@ class WorkerProc:
349
except SystemExit:
350
logger.debug("Worker interrupted.")
351
338
- except BaseException as e:
339
- logger.exception(e)
352
+ except Exception:
353
+ # worker_busy_loop sends exceptions exceptons to Executor
354
+ # for shutdown, but if there is an error in startup or an
355
+ # error with IPC itself, we need to alert the parent.
356
+ psutil.Process().parent().send_signal(signal.SIGUSR1)
357
raise
358
359
finally:
@@ -377,9 +394,10 @@ class WorkerProc:
394
395
try:
396
output = getattr(self.worker, method)(*args, **kwargs)
380
- except BaseException as e:
397
+ except Exception as e:
398
self.worker_response_mq.enqueue(
399
(WorkerProc.ResponseStatus.FAILURE, e))
400
+ logger.exception("WorkerProc hit an exception: %s", exc_info=e)
401
continue
402
403
self.worker_response_mq.enqueue(