@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a657bfc48a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Add `multiproc_worker_utils` for multiprocessing-based workers (#4357)

Nick Hill committed May 1, 2024 at 11:41 UTC a657bfc48a11d87de146629a7b6c03e9ccfbc3fc
2 files changed +440
tests/engine/test_multiproc_workers.py new
+176
@@ -0,0 +1,176 @@
1 +import asyncio
2 +from concurrent.futures import ThreadPoolExecutor
3 +from functools import partial
4 +from time import sleep
5 +from typing import Any, List, Tuple
6 +
7 +import pytest
8 +
9 +from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
10 + ResultHandler, WorkerMonitor)
11 +
12 +
13 +class DummyWorker:
14 + """Dummy version of vllm.worker.worker.Worker"""
15 +
16 + def __init__(self, rank: int):
17 + self.rank = rank
18 +
19 + def worker_method(self, worker_input: Any) -> Tuple[int, Any]:
20 + sleep(0.05)
21 +
22 + if isinstance(worker_input, Exception):
23 + # simulate error case
24 + raise worker_input
25 +
26 + return self.rank, input
27 +
28 +
29 +def _start_workers() -> Tuple[List[ProcessWorkerWrapper], WorkerMonitor]:
30 + result_handler = ResultHandler()
31 + workers = [
32 + ProcessWorkerWrapper(result_handler, partial(DummyWorker, rank=rank))
33 + for rank in range(8)
34 + ]
35 +
36 + worker_monitor = WorkerMonitor(workers, result_handler)
37 + assert not worker_monitor.is_alive()
38 +
39 + result_handler.start()
40 + worker_monitor.start()
41 + assert worker_monitor.is_alive()
42 +
43 + return workers, worker_monitor
44 +
45 +
46 +def test_local_workers() -> None:
47 + """Test workers with sync task submission"""
48 +
49 + workers, worker_monitor = _start_workers()
50 +
51 + def execute_workers(worker_input: str) -> None:
52 + worker_outputs = [
53 + worker.execute_method("worker_method", worker_input)
54 + for worker in workers
55 + ]
56 +
57 + for rank, output in enumerate(worker_outputs):
58 + assert output.get() == (rank, input)
59 +
60 + executor = ThreadPoolExecutor(max_workers=4)
61 +
62 + # Test concurrent submission from different threads
63 + futures = [
64 + executor.submit(partial(execute_workers, f"thread {thread_num}"))
65 + for thread_num in range(4)
66 + ]
67 +
68 + for future in futures:
69 + future.result()
70 +
71 + # Test error case
72 + exception = ValueError("fake error")
73 + result = workers[0].execute_method("worker_method", exception)
74 + try:
75 + result.get()
76 + pytest.fail("task should have failed")
77 + except Exception as e:
78 + assert isinstance(e, ValueError)
79 + assert str(e) == "fake error"
80 +
81 + # Test cleanup when a worker fails
82 + assert worker_monitor.is_alive()
83 + workers[3].process.kill()
84 +
85 + # Other workers should get shut down here
86 + worker_monitor.join(2)
87 +
88 + # Ensure everything is stopped
89 + assert not worker_monitor.is_alive()
90 + assert all(not worker.process.is_alive() for worker in workers)
91 +
92 + # Further attempts to submit tasks should fail
93 + try:
94 + _result = workers[0].execute_method("worker_method", "test")
95 + pytest.fail("task should fail once workers have been shut down")
96 + except Exception as e:
97 + assert isinstance(e, ChildProcessError)
98 +
99 +
100 +def test_local_workers_clean_shutdown() -> None:
101 + """Test clean shutdown"""
102 +
103 + workers, worker_monitor = _start_workers()
104 +
105 + assert worker_monitor.is_alive()
106 + assert all(worker.process.is_alive() for worker in workers)
107 +
108 + # Clean shutdown
109 + worker_monitor.close()
110 +
111 + worker_monitor.join(5)
112 +
113 + # Ensure everything is stopped
114 + assert not worker_monitor.is_alive()
115 + assert all(not worker.process.is_alive() for worker in workers)
116 +
117 + # Further attempts to submit tasks should fail
118 + try:
119 + _result = workers[0].execute_method("worker_method", "test")
120 + pytest.fail("task should fail once workers have been shut down")
121 + except Exception as e:
122 + assert isinstance(e, ChildProcessError)
123 +
124 +
125 +@pytest.mark.asyncio
126 +async def test_local_workers_async() -> None:
127 + """Test local workers with async task submission"""
128 +
129 + workers, worker_monitor = _start_workers()
130 +
131 + async def execute_workers(worker_input: str) -> None:
132 + worker_coros = [
133 + worker.execute_method_async("worker_method", worker_input)
134 + for worker in workers
135 + ]
136 +
137 + results = await asyncio.gather(*worker_coros)
138 + for rank, result in enumerate(results):
139 + assert result == (rank, input)
140 +
141 + tasks = [
142 + asyncio.create_task(execute_workers(f"task {task_num}"))
143 + for task_num in range(4)
144 + ]
145 +
146 + for task in tasks:
147 + await task
148 +
149 + # Test error case
150 + exception = ValueError("fake error")
151 + try:
152 + _result = await workers[0].execute_method_async(
153 + "worker_method", exception)
154 + pytest.fail("task should have failed")
155 + except Exception as e:
156 + assert isinstance(e, ValueError)
157 + assert str(e) == "fake error"
158 +
159 + # Test cleanup when a worker fails
160 + assert worker_monitor.is_alive()
161 + workers[3].process.kill()
162 +
163 + # Other workers should get shut down here
164 + worker_monitor.join(2)
165 +
166 + # Ensure everything is stopped
167 + assert not worker_monitor.is_alive()
168 + assert all(not worker.process.is_alive() for worker in workers)
169 +
170 + # Further attempts to submit tasks should fail
171 + try:
172 + _result = await workers[0].execute_method_async(
173 + "worker_method", "test")
174 + pytest.fail("task should fail once workers have been shut down")
175 + except Exception as e:
176 + assert isinstance(e, ChildProcessError)
vllm/executor/multiproc_worker_utils.py new
+264
@@ -0,0 +1,264 @@
1 +import asyncio
2 +import multiprocessing
3 +import os
4 +import sys
5 +import threading
6 +import traceback
7 +import uuid
8 +from dataclasses import dataclass
9 +from multiprocessing import Queue
10 +from multiprocessing.connection import wait
11 +from multiprocessing.process import BaseProcess
12 +from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
13 + TypeVar, Union)
14 +
15 +from vllm.logger import init_logger
16 +
17 +logger = init_logger(__name__)
18 +
19 +T = TypeVar('T')
20 +
21 +_TERMINATE = "TERMINATE" # sentinel
22 +
23 +# ANSI color codes
24 +CYAN = '\033[1;36m'
25 +RESET = '\033[0;0m'
26 +
27 +JOIN_TIMEOUT_S = 2
28 +
29 +# Use dedicated multiprocess context for workers.
30 +# Both spawn and fork work
31 +mp_method = os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
32 +mp = multiprocessing.get_context(mp_method)
33 +
34 +
35 +@dataclass
36 +class Result(Generic[T]):
37 + """Result of task dispatched to worker"""
38 +
39 + task_id: uuid.UUID
40 + value: Optional[T] = None
41 + exception: Optional[BaseException] = None
42 +
43 +
44 +class ResultFuture(threading.Event, Generic[T]):
45 + """Synchronous future for non-async case"""
46 +
47 + def __init__(self):
48 + super().__init__()
49 + self.result: Optional[Result[T]] = None
50 +
51 + def set_result(self, result: Result[T]):
52 + self.result = result
53 + self.set()
54 +
55 + def get(self) -> T:
56 + self.wait()
57 + assert self.result is not None
58 + if self.result.exception is not None:
59 + raise self.result.exception
60 + return self.result.value # type: ignore[return-value]
61 +
62 +
63 +def _set_future_result(future: Union[ResultFuture, asyncio.Future],
64 + result: Result):
65 + if isinstance(future, ResultFuture):
66 + future.set_result(result)
67 + return
68 + loop = future.get_loop()
69 + if result.exception is not None:
70 + loop.call_soon_threadsafe(future.set_exception, result.exception)
71 + else:
72 + loop.call_soon_threadsafe(future.set_result, result.value)
73 +
74 +
75 +class ResultHandler(threading.Thread):
76 + """Handle results from all workers (in background thread)"""
77 +
78 + def __init__(self) -> None:
79 + super().__init__(daemon=True)
80 + self.result_queue = mp.Queue()
81 + self.tasks: Dict[uuid.UUID, Union[ResultFuture, asyncio.Future]] = {}
82 +
83 + def run(self):
84 + for result in iter(self.result_queue.get, _TERMINATE):
85 + future = self.tasks.pop(result.task_id)
86 + _set_future_result(future, result)
87 + # Ensure that all waiters will receive an exception
88 + for task_id, future in self.tasks.items():
89 + _set_future_result(
90 + future,
91 + Result(task_id=task_id,
92 + exception=ChildProcessError("worker died")))
93 +
94 + def close(self):
95 + self.result_queue.put(_TERMINATE)
96 +
97 +
98 +class WorkerMonitor(threading.Thread):
99 + """Monitor worker status (in background thread)"""
100 +
101 + def __init__(self, workers: List['ProcessWorkerWrapper'],
102 + result_handler: ResultHandler):
103 + super().__init__(daemon=True)
104 + self.workers = workers
105 + self.result_handler = result_handler
106 + self._close = False
107 +
108 + def run(self) -> None:
109 + # Blocks until any worker exits
110 + dead_sentinels = wait([w.process.sentinel for w in self.workers])
111 + if not self._close:
112 + self._close = True
113 +
114 + # Kill / cleanup all workers
115 + for worker in self.workers:
116 + process = worker.process
117 + if process.sentinel in dead_sentinels:
118 + process.join(JOIN_TIMEOUT_S)
119 + if process.exitcode is not None and process.exitcode != 0:
120 + logger.error("Worker %s pid %s died, exit code: %s",
121 + process.name, process.pid, process.exitcode)
122 + # Cleanup any remaining workers
123 + logger.info("Killing local vLLM worker processes")
124 + for worker in self.workers:
125 + worker.kill_worker()
126 + # Must be done after worker task queues are all closed
127 + self.result_handler.close()
128 +
129 + for worker in self.workers:
130 + worker.process.join(JOIN_TIMEOUT_S)
131 +
132 + def close(self):
133 + if self._close:
134 + return
135 + self._close = True
136 + logger.info("Terminating local vLLM worker processes")
137 + for worker in self.workers:
138 + worker.terminate_worker()
139 + # Must be done after worker task queues are all closed
140 + self.result_handler.close()
141 +
142 +
143 +class ProcessWorkerWrapper:
144 + """Local process wrapper for vllm.worker.Worker,
145 + for handling single-node multi-GPU tensor parallel."""
146 +
147 + def __init__(self, result_handler: ResultHandler,
148 + worker_factory: Callable[[], Any]) -> None:
149 + self._task_queue = mp.Queue()
150 + self.result_queue = result_handler.result_queue
151 + self.tasks = result_handler.tasks
152 + self.process: BaseProcess = mp.Process( # type: ignore[attr-defined]
153 + target=_run_worker_process,
154 + name="VllmWorkerProcess",
155 + kwargs=dict(
156 + worker_factory=worker_factory,
157 + task_queue=self._task_queue,
158 + result_queue=self.result_queue,
159 + ),
160 + daemon=True)
161 +
162 + self.process.start()
163 +
164 + def _enqueue_task(self, future: Union[ResultFuture, asyncio.Future],
165 + method: str, args, kwargs):
166 + task_id = uuid.uuid4()
167 + self.tasks[task_id] = future
168 + try:
169 + self._task_queue.put((task_id, method, args, kwargs))
170 + except BaseException as e:
171 + del self.tasks[task_id]
172 + raise ChildProcessError("worker died") from e
173 +
174 + def execute_method(self, method: str, *args, **kwargs):
175 + future: ResultFuture = ResultFuture()
176 + self._enqueue_task(future, method, args, kwargs)
177 + return future
178 +
179 + async def execute_method_async(self, method: str, *args, **kwargs):
180 + future = asyncio.get_running_loop().create_future()
181 + self._enqueue_task(future, method, args, kwargs)
182 + return await future
183 +
184 + def terminate_worker(self):
185 + try:
186 + self._task_queue.put(_TERMINATE)
187 + except ValueError:
188 + self.process.kill()
189 + self._task_queue.close()
190 +
191 + def kill_worker(self):
192 + self._task_queue.close()
193 + self.process.kill()
194 +
195 +
196 +def _run_worker_process(
197 + worker_factory: Callable[[], Any],
198 + task_queue: Queue,
199 + result_queue: Queue,
200 +) -> None:
201 + """Worker process event loop"""
202 +
203 + # Add process-specific prefix to stdout and stderr
204 + process_name = mp.current_process().name
205 + pid = os.getpid()
206 + _add_prefix(sys.stdout, process_name, pid)
207 + _add_prefix(sys.stderr, process_name, pid)
208 +
209 + # Initialize worker
210 + worker = worker_factory()
211 + del worker_factory
212 +
213 + # Accept tasks from the engine in task_queue
214 + # and return task output in result_queue
215 + logger.info("Worker ready; awaiting tasks")
216 + try:
217 + for items in iter(task_queue.get, _TERMINATE):
218 + output = None
219 + exception = None
220 + task_id, method, args, kwargs = items
221 + try:
222 + executor = getattr(worker, method)
223 + output = executor(*args, **kwargs)
224 + except BaseException as e:
225 + tb = traceback.format_exc()
226 + logger.error(
227 + "Exception in worker %s while processing method %s: %s, %s",
228 + process_name, method, e, tb)
229 + exception = e
230 + result_queue.put(
231 + Result(task_id=task_id, value=output, exception=exception))
232 + except KeyboardInterrupt:
233 + pass
234 + except Exception:
235 + logger.exception("Worker failed")
236 +
237 + logger.info("Worker exiting")
238 +
239 +
240 +def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
241 + """Prepend each output line with process-specific prefix"""
242 +
243 + prefix = f"{CYAN}({worker_name} pid={pid}){RESET} "
244 + file_write = file.write
245 +
246 + def write_with_prefix(s: str):
247 + if not s:
248 + return
249 + if file.start_new_line: # type: ignore[attr-defined]
250 + file_write(prefix)
251 + idx = 0
252 + while (next_idx := s.find('\n', idx)) != -1:
253 + next_idx += 1
254 + file_write(s[idx:next_idx])
255 + if next_idx == len(s):
256 + file.start_new_line = True # type: ignore[attr-defined]
257 + return
258 + file_write(prefix)
259 + idx = next_idx
260 + file_write(s[idx:])
261 + file.start_new_line = False # type: ignore[attr-defined]
262 +
263 + file.start_new_line = True # type: ignore[attr-defined]
264 + file.write = write_with_prefix # type: ignore[method-assign]