A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Add `multiproc_worker_utils` for multiprocessing-based workers (#4357)
Nick Hill committed
May 1, 2024 at 11:41 UTC
a657bfc48a11d87de146629a7b6c03e9ccfbc3fc
2 files changed
+440
tests/engine/test_multiproc_workers.py
new
+176
@@ -0,0 +1,176 @@
1
+import asyncio
2
+from concurrent.futures import ThreadPoolExecutor
3
+from functools import partial
4
+from time import sleep
5
+from typing import Any, List, Tuple
6
+
7
+import pytest
8
+
9
+from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
10
+ ResultHandler, WorkerMonitor)
11
+
12
+
13
+class DummyWorker:
14
+ """Dummy version of vllm.worker.worker.Worker"""
15
+
16
+ def __init__(self, rank: int):
17
+ self.rank = rank
18
+
19
+ def worker_method(self, worker_input: Any) -> Tuple[int, Any]:
20
+ sleep(0.05)
21
+
22
+ if isinstance(worker_input, Exception):
23
+ # simulate error case
24
+ raise worker_input
25
+
26
+ return self.rank, input
27
+
28
+
29
+def _start_workers() -> Tuple[List[ProcessWorkerWrapper], WorkerMonitor]:
30
+ result_handler = ResultHandler()
31
+ workers = [
32
+ ProcessWorkerWrapper(result_handler, partial(DummyWorker, rank=rank))
33
+ for rank in range(8)
34
+ ]
35
+
36
+ worker_monitor = WorkerMonitor(workers, result_handler)
37
+ assert not worker_monitor.is_alive()
38
+
39
+ result_handler.start()
40
+ worker_monitor.start()
41
+ assert worker_monitor.is_alive()
42
+
43
+ return workers, worker_monitor
44
+
45
+
46
+def test_local_workers() -> None:
47
+ """Test workers with sync task submission"""
48
+
49
+ workers, worker_monitor = _start_workers()
50
+
51
+ def execute_workers(worker_input: str) -> None:
52
+ worker_outputs = [
53
+ worker.execute_method("worker_method", worker_input)
54
+ for worker in workers
55
+ ]
56
+
57
+ for rank, output in enumerate(worker_outputs):
58
+ assert output.get() == (rank, input)
59
+
60
+ executor = ThreadPoolExecutor(max_workers=4)
61
+
62
+ # Test concurrent submission from different threads
63
+ futures = [
64
+ executor.submit(partial(execute_workers, f"thread {thread_num}"))
65
+ for thread_num in range(4)
66
+ ]
67
+
68
+ for future in futures:
69
+ future.result()
70
+
71
+ # Test error case
72
+ exception = ValueError("fake error")
73
+ result = workers[0].execute_method("worker_method", exception)
74
+ try:
75
+ result.get()
76
+ pytest.fail("task should have failed")
77
+ except Exception as e:
78
+ assert isinstance(e, ValueError)
79
+ assert str(e) == "fake error"
80
+
81
+ # Test cleanup when a worker fails
82
+ assert worker_monitor.is_alive()
83
+ workers[3].process.kill()
84
+
85
+ # Other workers should get shut down here
86
+ worker_monitor.join(2)
87
+
88
+ # Ensure everything is stopped
89
+ assert not worker_monitor.is_alive()
90
+ assert all(not worker.process.is_alive() for worker in workers)
91
+
92
+ # Further attempts to submit tasks should fail
93
+ try:
94
+ _result = workers[0].execute_method("worker_method", "test")
95
+ pytest.fail("task should fail once workers have been shut down")
96
+ except Exception as e:
97
+ assert isinstance(e, ChildProcessError)
98
+
99
+
100
+def test_local_workers_clean_shutdown() -> None:
101
+ """Test clean shutdown"""
102
+
103
+ workers, worker_monitor = _start_workers()
104
+
105
+ assert worker_monitor.is_alive()
106
+ assert all(worker.process.is_alive() for worker in workers)
107
+
108
+ # Clean shutdown
109
+ worker_monitor.close()
110
+
111
+ worker_monitor.join(5)
112
+
113
+ # Ensure everything is stopped
114
+ assert not worker_monitor.is_alive()
115
+ assert all(not worker.process.is_alive() for worker in workers)
116
+
117
+ # Further attempts to submit tasks should fail
118
+ try:
119
+ _result = workers[0].execute_method("worker_method", "test")
120
+ pytest.fail("task should fail once workers have been shut down")
121
+ except Exception as e:
122
+ assert isinstance(e, ChildProcessError)
123
+
124
+
125
+@pytest.mark.asyncio
126
+async def test_local_workers_async() -> None:
127
+ """Test local workers with async task submission"""
128
+
129
+ workers, worker_monitor = _start_workers()
130
+
131
+ async def execute_workers(worker_input: str) -> None:
132
+ worker_coros = [
133
+ worker.execute_method_async("worker_method", worker_input)
134
+ for worker in workers
135
+ ]
136
+
137
+ results = await asyncio.gather(*worker_coros)
138
+ for rank, result in enumerate(results):
139
+ assert result == (rank, input)
140
+
141
+ tasks = [
142
+ asyncio.create_task(execute_workers(f"task {task_num}"))
143
+ for task_num in range(4)
144
+ ]
145
+
146
+ for task in tasks:
147
+ await task
148
+
149
+ # Test error case
150
+ exception = ValueError("fake error")
151
+ try:
152
+ _result = await workers[0].execute_method_async(
153
+ "worker_method", exception)
154
+ pytest.fail("task should have failed")
155
+ except Exception as e:
156
+ assert isinstance(e, ValueError)
157
+ assert str(e) == "fake error"
158
+
159
+ # Test cleanup when a worker fails
160
+ assert worker_monitor.is_alive()
161
+ workers[3].process.kill()
162
+
163
+ # Other workers should get shut down here
164
+ worker_monitor.join(2)
165
+
166
+ # Ensure everything is stopped
167
+ assert not worker_monitor.is_alive()
168
+ assert all(not worker.process.is_alive() for worker in workers)
169
+
170
+ # Further attempts to submit tasks should fail
171
+ try:
172
+ _result = await workers[0].execute_method_async(
173
+ "worker_method", "test")
174
+ pytest.fail("task should fail once workers have been shut down")
175
+ except Exception as e:
176
+ assert isinstance(e, ChildProcessError)
vllm/executor/multiproc_worker_utils.py
new
+264
@@ -0,0 +1,264 @@
1
+import asyncio
2
+import multiprocessing
3
+import os
4
+import sys
5
+import threading
6
+import traceback
7
+import uuid
8
+from dataclasses import dataclass
9
+from multiprocessing import Queue
10
+from multiprocessing.connection import wait
11
+from multiprocessing.process import BaseProcess
12
+from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
13
+ TypeVar, Union)
14
+
15
+from vllm.logger import init_logger
16
+
17
+logger = init_logger(__name__)
18
+
19
+T = TypeVar('T')
20
+
21
+_TERMINATE = "TERMINATE" # sentinel
22
+
23
+# ANSI color codes
24
+CYAN = '\033[1;36m'
25
+RESET = '\033[0;0m'
26
+
27
+JOIN_TIMEOUT_S = 2
28
+
29
+# Use dedicated multiprocess context for workers.
30
+# Both spawn and fork work
31
+mp_method = os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
32
+mp = multiprocessing.get_context(mp_method)
33
+
34
+
35
+@dataclass
36
+class Result(Generic[T]):
37
+ """Result of task dispatched to worker"""
38
+
39
+ task_id: uuid.UUID
40
+ value: Optional[T] = None
41
+ exception: Optional[BaseException] = None
42
+
43
+
44
+class ResultFuture(threading.Event, Generic[T]):
45
+ """Synchronous future for non-async case"""
46
+
47
+ def __init__(self):
48
+ super().__init__()
49
+ self.result: Optional[Result[T]] = None
50
+
51
+ def set_result(self, result: Result[T]):
52
+ self.result = result
53
+ self.set()
54
+
55
+ def get(self) -> T:
56
+ self.wait()
57
+ assert self.result is not None
58
+ if self.result.exception is not None:
59
+ raise self.result.exception
60
+ return self.result.value # type: ignore[return-value]
61
+
62
+
63
+def _set_future_result(future: Union[ResultFuture, asyncio.Future],
64
+ result: Result):
65
+ if isinstance(future, ResultFuture):
66
+ future.set_result(result)
67
+ return
68
+ loop = future.get_loop()
69
+ if result.exception is not None:
70
+ loop.call_soon_threadsafe(future.set_exception, result.exception)
71
+ else:
72
+ loop.call_soon_threadsafe(future.set_result, result.value)
73
+
74
+
75
+class ResultHandler(threading.Thread):
76
+ """Handle results from all workers (in background thread)"""
77
+
78
+ def __init__(self) -> None:
79
+ super().__init__(daemon=True)
80
+ self.result_queue = mp.Queue()
81
+ self.tasks: Dict[uuid.UUID, Union[ResultFuture, asyncio.Future]] = {}
82
+
83
+ def run(self):
84
+ for result in iter(self.result_queue.get, _TERMINATE):
85
+ future = self.tasks.pop(result.task_id)
86
+ _set_future_result(future, result)
87
+ # Ensure that all waiters will receive an exception
88
+ for task_id, future in self.tasks.items():
89
+ _set_future_result(
90
+ future,
91
+ Result(task_id=task_id,
92
+ exception=ChildProcessError("worker died")))
93
+
94
+ def close(self):
95
+ self.result_queue.put(_TERMINATE)
96
+
97
+
98
+class WorkerMonitor(threading.Thread):
99
+ """Monitor worker status (in background thread)"""
100
+
101
+ def __init__(self, workers: List['ProcessWorkerWrapper'],
102
+ result_handler: ResultHandler):
103
+ super().__init__(daemon=True)
104
+ self.workers = workers
105
+ self.result_handler = result_handler
106
+ self._close = False
107
+
108
+ def run(self) -> None:
109
+ # Blocks until any worker exits
110
+ dead_sentinels = wait([w.process.sentinel for w in self.workers])
111
+ if not self._close:
112
+ self._close = True
113
+
114
+ # Kill / cleanup all workers
115
+ for worker in self.workers:
116
+ process = worker.process
117
+ if process.sentinel in dead_sentinels:
118
+ process.join(JOIN_TIMEOUT_S)
119
+ if process.exitcode is not None and process.exitcode != 0:
120
+ logger.error("Worker %s pid %s died, exit code: %s",
121
+ process.name, process.pid, process.exitcode)
122
+ # Cleanup any remaining workers
123
+ logger.info("Killing local vLLM worker processes")
124
+ for worker in self.workers:
125
+ worker.kill_worker()
126
+ # Must be done after worker task queues are all closed
127
+ self.result_handler.close()
128
+
129
+ for worker in self.workers:
130
+ worker.process.join(JOIN_TIMEOUT_S)
131
+
132
+ def close(self):
133
+ if self._close:
134
+ return
135
+ self._close = True
136
+ logger.info("Terminating local vLLM worker processes")
137
+ for worker in self.workers:
138
+ worker.terminate_worker()
139
+ # Must be done after worker task queues are all closed
140
+ self.result_handler.close()
141
+
142
+
143
+class ProcessWorkerWrapper:
144
+ """Local process wrapper for vllm.worker.Worker,
145
+ for handling single-node multi-GPU tensor parallel."""
146
+
147
+ def __init__(self, result_handler: ResultHandler,
148
+ worker_factory: Callable[[], Any]) -> None:
149
+ self._task_queue = mp.Queue()
150
+ self.result_queue = result_handler.result_queue
151
+ self.tasks = result_handler.tasks
152
+ self.process: BaseProcess = mp.Process( # type: ignore[attr-defined]
153
+ target=_run_worker_process,
154
+ name="VllmWorkerProcess",
155
+ kwargs=dict(
156
+ worker_factory=worker_factory,
157
+ task_queue=self._task_queue,
158
+ result_queue=self.result_queue,
159
+ ),
160
+ daemon=True)
161
+
162
+ self.process.start()
163
+
164
+ def _enqueue_task(self, future: Union[ResultFuture, asyncio.Future],
165
+ method: str, args, kwargs):
166
+ task_id = uuid.uuid4()
167
+ self.tasks[task_id] = future
168
+ try:
169
+ self._task_queue.put((task_id, method, args, kwargs))
170
+ except BaseException as e:
171
+ del self.tasks[task_id]
172
+ raise ChildProcessError("worker died") from e
173
+
174
+ def execute_method(self, method: str, *args, **kwargs):
175
+ future: ResultFuture = ResultFuture()
176
+ self._enqueue_task(future, method, args, kwargs)
177
+ return future
178
+
179
+ async def execute_method_async(self, method: str, *args, **kwargs):
180
+ future = asyncio.get_running_loop().create_future()
181
+ self._enqueue_task(future, method, args, kwargs)
182
+ return await future
183
+
184
+ def terminate_worker(self):
185
+ try:
186
+ self._task_queue.put(_TERMINATE)
187
+ except ValueError:
188
+ self.process.kill()
189
+ self._task_queue.close()
190
+
191
+ def kill_worker(self):
192
+ self._task_queue.close()
193
+ self.process.kill()
194
+
195
+
196
+def _run_worker_process(
197
+ worker_factory: Callable[[], Any],
198
+ task_queue: Queue,
199
+ result_queue: Queue,
200
+) -> None:
201
+ """Worker process event loop"""
202
+
203
+ # Add process-specific prefix to stdout and stderr
204
+ process_name = mp.current_process().name
205
+ pid = os.getpid()
206
+ _add_prefix(sys.stdout, process_name, pid)
207
+ _add_prefix(sys.stderr, process_name, pid)
208
+
209
+ # Initialize worker
210
+ worker = worker_factory()
211
+ del worker_factory
212
+
213
+ # Accept tasks from the engine in task_queue
214
+ # and return task output in result_queue
215
+ logger.info("Worker ready; awaiting tasks")
216
+ try:
217
+ for items in iter(task_queue.get, _TERMINATE):
218
+ output = None
219
+ exception = None
220
+ task_id, method, args, kwargs = items
221
+ try:
222
+ executor = getattr(worker, method)
223
+ output = executor(*args, **kwargs)
224
+ except BaseException as e:
225
+ tb = traceback.format_exc()
226
+ logger.error(
227
+ "Exception in worker %s while processing method %s: %s, %s",
228
+ process_name, method, e, tb)
229
+ exception = e
230
+ result_queue.put(
231
+ Result(task_id=task_id, value=output, exception=exception))
232
+ except KeyboardInterrupt:
233
+ pass
234
+ except Exception:
235
+ logger.exception("Worker failed")
236
+
237
+ logger.info("Worker exiting")
238
+
239
+
240
+def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
241
+ """Prepend each output line with process-specific prefix"""
242
+
243
+ prefix = f"{CYAN}({worker_name} pid={pid}){RESET} "
244
+ file_write = file.write
245
+
246
+ def write_with_prefix(s: str):
247
+ if not s:
248
+ return
249
+ if file.start_new_line: # type: ignore[attr-defined]
250
+ file_write(prefix)
251
+ idx = 0
252
+ while (next_idx := s.find('\n', idx)) != -1:
253
+ next_idx += 1
254
+ file_write(s[idx:next_idx])
255
+ if next_idx == len(s):
256
+ file.start_new_line = True # type: ignore[attr-defined]
257
+ return
258
+ file_write(prefix)
259
+ idx = next_idx
260
+ file_write(s[idx:])
261
+ file.start_new_line = False # type: ignore[attr-defined]
262
+
263
+ file.start_new_line = True # type: ignore[attr-defined]
264
+ file.write = write_with_prefix # type: ignore[method-assign]