@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7ed82d1974

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[V0 Deprecation] Remove V0 MP executor (#25329)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Sep 20, 2025 at 21:26 UTC 7ed82d1974837957a3bfb6d576b9cffba24d31ae

3 files changed +33 -530

vllm/executor/mp_distributed_executor.py deleted

-244

@@ -1,244 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import asyncio

5 -import os

6 -from typing import Any, Callable, List, Optional, Union

7 -

8 -import cloudpickle

9 -

10 -from vllm.executor.executor_base import DistributedExecutorBase

11 -from vllm.executor.multiproc_worker_utils import (

12 - ProcessWorkerWrapper, ResultHandler, WorkerMonitor,

13 - set_multiprocessing_worker_envs)

14 -from vllm.logger import init_logger

15 -from vllm.model_executor.layers.sampler import SamplerOutput

16 -from vllm.sequence import ExecuteModelRequest

17 -from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,

18 - get_distributed_init_method, get_ip, get_open_port,

19 - make_async, run_method, update_environment_variables)

20 -from vllm.worker.worker_base import WorkerWrapperBase

21 -

22 -logger = init_logger(__name__)

23 -

24 -

25 -class MultiprocessingDistributedExecutor(DistributedExecutorBase):

26 - """Python multiprocessing-based distributed executor"""

27 -

28 - uses_ray: bool = False

29 -

30 - def _check_cuda(self) -> None:

31 - """Check that the number of GPUs is sufficient for the parallel

32 - configuration. Separate from _init_executor to reduce the number of

33 - indented blocks.

34 - """

35 - parallel_config = self.parallel_config

36 - world_size = parallel_config.world_size

37 - tensor_parallel_size = parallel_config.tensor_parallel_size

38 -

39 - cuda_device_count = cuda_device_count_stateless()

40 - # Use confusing message for more common TP-only case.

41 - if tensor_parallel_size > cuda_device_count:

42 - raise RuntimeError(

43 - f"please set tensor_parallel_size ({tensor_parallel_size}) "

44 - f"to less than max local gpu count ({cuda_device_count})")

45 -

46 - if world_size > cuda_device_count:

47 - raise RuntimeError(

48 - f"please ensure that world_size ({world_size}) "

49 - f"is less than than max local gpu count ({cuda_device_count})")

50 -

51 - # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers

52 - if "CUDA_VISIBLE_DEVICES" not in os.environ:

53 - update_environment_variables({

54 - "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))

55 - })

56 -

57 - def _init_executor(self) -> None:

58 -

59 - from vllm.platforms import current_platform

60 - if current_platform.is_cuda_alike():

61 - self._check_cuda()

62 -

63 - # Create the parallel GPU workers.

64 - world_size = self.parallel_config.world_size

65 - tensor_parallel_size = self.parallel_config.tensor_parallel_size

66 -

67 - # Set multiprocessing envs that are common to V0 and V1

68 - set_multiprocessing_worker_envs(self.parallel_config)

69 -

70 - # Multiprocessing-based executor does not support multi-node setting.

71 - # Since it only works for single node, we can use the loopback address

72 - # 127.0.0.1 for communication.

73 - distributed_init_method = get_distributed_init_method(

74 - "127.0.0.1", get_open_port())

75 -

76 - self.workers: List[ProcessWorkerWrapper] = []

77 - # This is the list of workers that are rank 0 of each TP group EXCEPT

78 - # global rank 0. These are the workers that will broadcast to the

79 - # rest of the workers.

80 - self.tp_driver_workers: List[ProcessWorkerWrapper] = []

81 - # This is the list of workers that are not drivers and not the first

82 - # worker in a TP group. These are the workers that will be

83 - # broadcasted to.

84 - self.non_driver_workers: List[ProcessWorkerWrapper] = []

85 -

86 - if world_size == 1:

87 - self.worker_monitor = None

88 - else:

89 - result_handler = ResultHandler()

90 - for rank in range(1, world_size):

91 - worker = ProcessWorkerWrapper(result_handler,

92 - WorkerWrapperBase,

93 - self.vllm_config, rank)

94 - self.workers.append(worker)

95 - if rank % tensor_parallel_size == 0:

96 - self.tp_driver_workers.append(worker)

97 - else:

98 - self.non_driver_workers.append(worker)

99 -

100 - self.worker_monitor = WorkerMonitor(self.workers, result_handler)

101 - result_handler.start()

102 - self.worker_monitor.start()

103 -

104 - # Set up signal handlers to shut down the executor cleanly

105 - # sometimes gc does not work well

106 -

107 - self.driver_worker = WorkerWrapperBase(self.vllm_config, 0)

108 -

109 - all_kwargs = []

110 - distributed_init_method = get_distributed_init_method(

111 - get_ip(), get_open_port())

112 - for i in range(world_size):

113 - local_rank = i

114 - rank = i

115 - kwargs = dict(

116 - vllm_config=self.vllm_config,

117 - local_rank=local_rank,

118 - rank=rank,

119 - distributed_init_method=distributed_init_method,

120 - is_driver_worker=(not self.parallel_config)

121 - or (rank % self.parallel_config.tensor_parallel_size == 0),

122 - )

123 - all_kwargs.append(kwargs)

124 - self._run_workers("init_worker", all_kwargs)

125 - self._run_workers("init_device")

126 - self._run_workers("load_model",

127 - max_concurrent_workers=self.parallel_config.

128 - max_parallel_loading_workers)

129 - self.driver_exec_model = make_async(self.driver_worker.execute_model)

130 - self.pp_locks: Optional[List[asyncio.Lock]] = None

131 -

132 - def shutdown(self):

133 - if (worker_monitor := getattr(self, "worker_monitor",

134 - None)) is not None:

135 - worker_monitor.close()

136 -

137 - def _driver_execute_model(

138 - self, execute_model_req: Optional[ExecuteModelRequest]

139 - ) -> Optional[List[SamplerOutput]]:

140 - """Run execute_model in the driver worker.

141 -

142 - Passing None will cause the driver to stop the model execution

143 - loop running in each of the remote workers.

144 - """

145 - return self.driver_worker.execute_model(execute_model_req)

146 -

147 - def _run_workers(

148 - self,

149 - method: Union[str, Callable],

150 - *args,

151 - async_run_tensor_parallel_workers_only: bool = False,

152 - max_concurrent_workers: Optional[int] = None,

153 - **kwargs,

154 - ) -> List[Any]:

155 - """Runs the given method on all workers.

156 -

157 - Args:

158 - async_run_tensor_parallel_workers_only: If True the method will be

159 - run only in the remote TP workers, not the driver worker.

160 - It will also be run asynchronously and return a list of futures

161 - rather than blocking on the results.

162 - """

163 - if isinstance(method, str):

164 - sent_method = method

165 - else:

166 - sent_method = cloudpickle.dumps(method)

167 - del method

168 -

169 - if max_concurrent_workers:

170 - raise NotImplementedError(

171 - "max_concurrent_workers is not supported yet.")

172 -

173 - if async_run_tensor_parallel_workers_only:

174 - # Run only non-driver workers and just return futures.

175 - return [

176 - worker.execute_method(sent_method, *args, **kwargs)

177 - for worker in self.non_driver_workers

178 - ]

179 -

180 - # Start all remote workers first.

181 - worker_outputs = [

182 - worker.execute_method(sent_method, *args, **kwargs)

183 - for worker in self.workers

184 - ]

185 -

186 - driver_worker_output = run_method(self.driver_worker, sent_method,

187 - args, kwargs)

188 -

189 - # Get the results of the workers.

190 - return [driver_worker_output

191 - ] + [output.get() for output in worker_outputs]

192 -

193 - def check_health(self) -> None:

194 - """Raises an error if engine is unhealthy."""

195 - if self.worker_monitor is not None and not self.worker_monitor.is_alive(

196 - ):

197 - raise RuntimeError("Worker processes are not running")

198 -

199 - def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:

200 - """Wait for futures returned from _run_workers() with

201 - async_run_remote_workers_only to complete."""

202 - for result in parallel_worker_tasks:

203 - result.get()

204 -

205 - async def _driver_execute_model_async(

206 - self,

207 - execute_model_req: Optional[ExecuteModelRequest] = None

208 - ) -> List[SamplerOutput]:

209 - if not self.tp_driver_workers:

210 - return await self.driver_exec_model(execute_model_req)

211 -

212 - if self.pp_locks is None:

213 - # This locks each pipeline parallel stage so multiple virtual

214 - # engines can't execute on the same stage at the same time

215 - # We create the locks here to avoid creating them in the constructor

216 - # which uses a different asyncio loop.

217 - self.pp_locks = [

218 - asyncio.Lock()

219 - for _ in range(self.parallel_config.pipeline_parallel_size)

220 - ]

221 -

222 - tasks = [

223 - asyncio.create_task(

224 - _run_task_with_lock(self.driver_exec_model, self.pp_locks[0],

225 - execute_model_req))

226 - ]

227 - for pp_rank, driver_worker in enumerate(self.tp_driver_workers,

228 - start=1):

229 - tasks.append(

230 - asyncio.create_task(

231 - _run_task_with_lock(driver_worker.execute_method_async,

232 - self.pp_locks[pp_rank],

233 - "execute_model", execute_model_req)))

234 - results = await asyncio.gather(*tasks)

235 -

236 - # Only the last PP stage has the final results.

237 - return results[-1]

238 -

239 - async def _start_worker_execution_loop(self):

240 - coros = [

241 - worker.execute_method_async("start_worker_execution_loop")

242 - for worker in self.non_driver_workers

243 - ]

244 - return await asyncio.gather(*coros)

vllm/executor/multiproc_worker_utils.py deleted

-279

@@ -1,279 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import asyncio

5 -import os

6 -import threading

7 -import uuid

8 -from dataclasses import dataclass

9 -from multiprocessing import Queue

10 -from multiprocessing.connection import wait

11 -from multiprocessing.process import BaseProcess

12 -from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union

13 -

14 -import torch

15 -

16 -from vllm.config import VllmConfig

17 -from vllm.logger import init_logger

18 -from vllm.utils import (_maybe_force_spawn, decorate_logs, get_mp_context,

19 - run_method)

20 -

21 -logger = init_logger(__name__)

22 -

23 -T = TypeVar('T')

24 -

25 -_TERMINATE = "TERMINATE" # sentinel

26 -

27 -JOIN_TIMEOUT_S = 2

28 -

29 -

30 -@dataclass

31 -class Result(Generic[T]):

32 - """Result of task dispatched to worker"""

33 -

34 - task_id: uuid.UUID

35 - value: Optional[T] = None

36 - exception: Optional[BaseException] = None

37 -

38 -

39 -class ResultFuture(threading.Event, Generic[T]):

40 - """Synchronous future for non-async case"""

41 -

42 - def __init__(self):

43 - super().__init__()

44 - self.result: Optional[Result[T]] = None

45 -

46 - def set_result(self, result: Result[T]):

47 - self.result = result

48 - self.set()

49 -

50 - def get(self) -> T:

51 - self.wait()

52 - assert self.result is not None

53 - if self.result.exception is not None:

54 - raise self.result.exception

55 - return self.result.value # type: ignore[return-value]

56 -

57 -

58 -def _set_future_result(future: Union[ResultFuture, asyncio.Future],

59 - result: Result):

60 - if isinstance(future, ResultFuture):

61 - future.set_result(result)

62 - return

63 - loop = future.get_loop()

64 - if not loop.is_closed():

65 - if result.exception is not None:

66 - loop.call_soon_threadsafe(future.set_exception, result.exception)

67 - else:

68 - loop.call_soon_threadsafe(future.set_result, result.value)

69 -

70 -

71 -class ResultHandler(threading.Thread):

72 - """Handle results from all workers (in background thread)"""

73 -

74 - def __init__(self) -> None:

75 - super().__init__(daemon=True)

76 - self.result_queue = get_mp_context().Queue()

77 - self.tasks: Dict[uuid.UUID, Union[ResultFuture, asyncio.Future]] = {}

78 -

79 - def run(self):

80 - for result in iter(self.result_queue.get, _TERMINATE):

81 - future = self.tasks.pop(result.task_id)

82 - _set_future_result(future, result)

83 - # Ensure that all waiters will receive an exception

84 - for task_id, future in self.tasks.items():

85 - _set_future_result(

86 - future,

87 - Result(task_id=task_id,

88 - exception=ChildProcessError("worker died")))

89 -

90 - def close(self):

91 - self.result_queue.put(_TERMINATE)

92 -

93 -

94 -class WorkerMonitor(threading.Thread):

95 - """Monitor worker status (in background thread)"""

96 -

97 - def __init__(self, workers: List['ProcessWorkerWrapper'],

98 - result_handler: ResultHandler):

99 - super().__init__(daemon=True)

100 - self.workers = workers

101 - self.result_handler = result_handler

102 - self._close = False

103 -

104 - def run(self) -> None:

105 - # Blocks until any worker exits

106 - dead_sentinels = wait([w.process.sentinel for w in self.workers])

107 - if not self._close:

108 - self._close = True

109 -

110 - # Kill / cleanup all workers

111 - for worker in self.workers:

112 - process = worker.process

113 - if process.sentinel in dead_sentinels:

114 - process.join(JOIN_TIMEOUT_S)

115 - if process.exitcode is not None and process.exitcode != 0:

116 - logger.error("Worker %s pid %s died, exit code: %s",

117 - process.name, process.pid, process.exitcode)

118 - # Cleanup any remaining workers

119 - if logger:

120 - logger.info("Killing local vLLM worker processes")

121 - for worker in self.workers:

122 - worker.kill_worker()

123 - # Must be done after worker task queues are all closed

124 - self.result_handler.close()

125 -

126 - for worker in self.workers:

127 - worker.process.join(JOIN_TIMEOUT_S)

128 -

129 - def close(self):

130 - if self._close:

131 - return

132 - self._close = True

133 - logger.info("Terminating local vLLM worker processes")

134 - for worker in self.workers:

135 - worker.terminate_worker()

136 - # Must be done after worker task queues are all closed

137 - self.result_handler.close()

138 -

139 -

140 -class ProcessWorkerWrapper:

141 - """Local process wrapper for vllm.worker.Worker,

142 - for handling single-node multi-GPU tensor parallel."""

143 -

144 - def __init__(self, result_handler: ResultHandler,

145 - worker_factory: Callable[[VllmConfig, int], Any],

146 - vllm_config: VllmConfig, rank: int) -> None:

147 - self.mp = get_mp_context()

148 - self._task_queue = self.mp.Queue()

149 - self.result_queue = result_handler.result_queue

150 - self.tasks = result_handler.tasks

151 - self.process: BaseProcess = self.mp.Process( # type: ignore[attr-defined]

152 - target=_run_worker_process,

153 - name="VllmWorkerProcess",

154 - kwargs=dict(

155 - worker_factory=worker_factory,

156 - task_queue=self._task_queue,

157 - result_queue=self.result_queue,

158 - vllm_config=vllm_config,

159 - rank=rank,

160 - ),

161 - daemon=True)

162 -

163 - self.process.start()

164 -

165 - def _enqueue_task(self, future: Union[ResultFuture, asyncio.Future],

166 - method: Union[str, bytes], args, kwargs):

167 - task_id = uuid.uuid4()

168 - self.tasks[task_id] = future

169 - try:

170 - self._task_queue.put((task_id, method, args, kwargs))

171 - except SystemExit:

172 - raise

173 - except BaseException as e:

174 - del self.tasks[task_id]

175 - raise ChildProcessError("worker died") from e

176 -

177 - def execute_method(self, method: Union[str, bytes], *args, **kwargs):

178 - future: ResultFuture = ResultFuture()

179 - self._enqueue_task(future, method, args, kwargs)

180 - return future

181 -

182 - async def execute_method_async(self, method: Union[str, bytes], *args,

183 - **kwargs):

184 - future = asyncio.get_running_loop().create_future()

185 - self._enqueue_task(future, method, args, kwargs)

186 - return await future

187 -

188 - def terminate_worker(self):

189 - try:

190 - self._task_queue.put(_TERMINATE)

191 - except ValueError:

192 - self.process.kill()

193 - self._task_queue.close()

194 -

195 - def kill_worker(self):

196 - self._task_queue.close()

197 - self.process.kill()

198 -

199 -

200 -def _run_worker_process(

201 - worker_factory: Callable[[VllmConfig, int], Any],

202 - task_queue: Queue,

203 - result_queue: Queue,

204 - vllm_config: VllmConfig,

205 - rank: int,

206 -) -> None:

207 - """Worker process event loop"""

208 -

209 - # Add process-specific prefix to stdout and stderr

210 - process_name = get_mp_context().current_process().name

211 - decorate_logs(process_name)

212 -

213 - # Initialize worker

214 - worker = worker_factory(vllm_config, rank)

215 - del worker_factory

216 -

217 - # Accept tasks from the engine in task_queue

218 - # and return task output in result_queue

219 - logger.info("Worker ready; awaiting tasks")

220 - try:

221 - for items in iter(task_queue.get, _TERMINATE):

222 - output = None

223 - exception = None

224 - task_id, method, args, kwargs = items

225 - try:

226 - output = run_method(worker, method, args, kwargs)

227 - except SystemExit:

228 - raise

229 - except KeyboardInterrupt:

230 - break

231 - except BaseException as e:

232 - logger.exception(

233 - "Exception in worker %s while processing method %s.",

234 - process_name, method)

235 - exception = e

236 - result_queue.put(

237 - Result(task_id=task_id, value=output, exception=exception))

238 - except KeyboardInterrupt:

239 - pass

240 - except Exception:

241 - logger.exception("Worker failed")

242 -

243 - # Flush TunableOp results when TunableOp is enabled and

244 - # online (in situ) tuning is enabled.

245 - # Offline tuning API (record_untuned_is_enabled()) only

246 - # available in PyTorch 2.6 or later.

247 - if torch.cuda.is_available():

248 - import torch.cuda.tunable as tunable

249 - if (tunable.is_enabled() and tunable.tuning_is_enabled()

250 - and not tunable.record_untuned_is_enabled()):

251 - tunable.write_file()

252 -

253 - logger.info("Worker exiting")

254 -

255 -

256 -def set_multiprocessing_worker_envs(parallel_config):

257 - """ Set up environment variables that should be used when there are workers

258 - in a multiprocessing environment. This should be called by the parent

259 - process before worker processes are created"""

260 -

261 - _maybe_force_spawn()

262 -

263 - # Configure thread parallelism if OMP_NUM_THREADS isn't set

264 - #

265 - # Helps to avoid CPU contention. The default of spawning a thread per

266 - # core combined with multiprocessing for each GPU can have a negative

267 - # impact on performance. The contention is amplified when running in a

268 - # container where CPU limits can cause throttling.

269 - default_omp_num_threads = 1

270 - if "OMP_NUM_THREADS" not in os.environ and (

271 - current_parallelism :=

272 - torch.get_num_threads()) > default_omp_num_threads:

273 - logger.warning(

274 - "Reducing Torch parallelism from %d threads to %d to avoid "

275 - "unnecessary CPU contention. Set OMP_NUM_THREADS in the "

276 - "external environment to tune this value as needed.",

277 - current_parallelism, default_omp_num_threads)

278 - os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)

279 - torch.set_num_threads(default_omp_num_threads)

vllm/v1/executor/multiproc_executor.py

+33 -7

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import multiprocessing

4 +import os

5 import pickle

6 import queue

7 import signal

@@ -19,6 +20,7 @@ from threading import Thread

20 from typing import Any, Callable, Optional, Union, cast

22 import cloudpickle

23 +import torch

25 import vllm.envs as envs

26 from vllm.config import VllmConfig

@@ -28,14 +30,12 @@ from vllm.distributed.device_communicators.shm_broadcast import (Handle,

30 MessageQueue)

31 from vllm.distributed.parallel_state import (get_dp_group, get_ep_group,

32 get_pp_group, get_tp_group)

31 -from vllm.executor.multiproc_worker_utils import (

32 - set_multiprocessing_worker_envs)

33 from vllm.logger import init_logger

34 from vllm.multimodal import MULTIMODAL_REGISTRY

35 from vllm.multimodal.cache import worker_receiver_cache_from_config

36 -from vllm.utils import (decorate_logs, get_distributed_init_method,

37 - get_loopback_ip, get_mp_context, get_open_port,

38 - set_process_title)

36 +from vllm.utils import (_maybe_force_spawn, decorate_logs,

37 + get_distributed_init_method, get_loopback_ip,

38 + get_mp_context, get_open_port, set_process_title)

39 from vllm.v1.core.sched.output import SchedulerOutput

40 from vllm.v1.executor.abstract import Executor, FailureCallback

41 from vllm.v1.executor.utils import get_and_update_mm_cache

@@ -67,8 +67,8 @@ class MultiprocExecutor(Executor):

67 f"tensor_parallel_size ({tensor_parallel_size}) x pipeline"

68 f"_parallel_size ({pp_parallel_size}). ")

70 - # Set multiprocessing envs that are common to V0 and V1

71 - set_multiprocessing_worker_envs(self.parallel_config)

70 + # Set multiprocessing envs

71 + set_multiprocessing_worker_envs()

73 # Multiprocessing-based executor does not support multi-node setting.

74 # Since it only works for single node, we can use the loopback address

@@ -698,3 +698,29 @@ class WorkerProc:

698 process_name += f"_EP{ep_rank}"

699 set_process_title(name=process_name)

700 decorate_logs(process_name)

701 +

702 +

703 +def set_multiprocessing_worker_envs():

704 + """ Set up environment variables that should be used when there are workers

705 + in a multiprocessing environment. This should be called by the parent

706 + process before worker processes are created"""

707 +

708 + _maybe_force_spawn()

709 +

710 + # Configure thread parallelism if OMP_NUM_THREADS isn't set

711 + #

712 + # Helps to avoid CPU contention. The default of spawning a thread per

713 + # core combined with multiprocessing for each GPU can have a negative

714 + # impact on performance. The contention is amplified when running in a

715 + # container where CPU limits can cause throttling.

716 + default_omp_num_threads = 1

717 + if "OMP_NUM_THREADS" not in os.environ and (

718 + current_parallelism :=

719 + torch.get_num_threads()) > default_omp_num_threads:

720 + logger.warning(

721 + "Reducing Torch parallelism from %d threads to %d to avoid "

722 + "unnecessary CPU contention. Set OMP_NUM_THREADS in the "

723 + "external environment to tune this value as needed.",

724 + current_parallelism, default_omp_num_threads)

725 + os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)

726 + torch.set_num_threads(default_omp_num_threads)