@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 87a0c076af
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[core] allow callable in collective_rpc (#12151)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 17, 2025 at 20:47 UTC 87a0c076afafb93dd082ff3876bea08adca56c56
13 files changed +147 -50
.buildkite/test-pipeline.yaml
+3 -1
@@ -107,7 +107,7 @@ steps:
107 source_file_dependencies:
108 - vllm/
109 commands:
110 - - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_generate_multiple_loras.py --ignore=entrypoints/llm/test_guided_generate.py
110 + - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_generate_multiple_loras.py --ignore=entrypoints/llm/test_guided_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
111 - pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
112 - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
113 - pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
@@ -466,7 +466,9 @@ steps:
466 - vllm/worker/worker_base.py
467 - vllm/worker/worker.py
468 - vllm/worker/model_runner.py
469 + - entrypoints/llm/test_collective_rpc.py
470 commands:
471 + - pytest -v -s entrypoints/llm/test_collective_rpc.py
472 - torchrun --nproc-per-node=2 distributed/test_torchrun_example.py
473 - pytest -v -s ./compile/test_basic_correctness.py
474 - pytest -v -s ./compile/test_wrapper.py
tests/engine/test_custom_executor.py
+2 -2
@@ -1,6 +1,6 @@
1 import asyncio
2 import os
3 -from typing import Any, Dict, List, Optional, Tuple
3 +from typing import Any, Callable, Dict, List, Optional, Tuple, Union
4
5 import pytest
6
@@ -18,7 +18,7 @@ class Mock:
18 class CustomUniExecutor(UniProcExecutor):
19
20 def collective_rpc(self,
21 - method: str,
21 + method: Union[str, Callable],
22 timeout: Optional[float] = None,
23 args: Tuple = (),
24 kwargs: Optional[Dict] = None) -> List[Any]:
tests/entrypoints/llm/test_collective_rpc.py new
+36
@@ -0,0 +1,36 @@
1 +import pytest
2 +
3 +from vllm import LLM
4 +
5 +from ...utils import fork_new_process_for_each_test
6 +
7 +
8 +@pytest.mark.parametrize("tp_size", [1, 2])
9 +@pytest.mark.parametrize("backend", ["mp", "ray"])
10 +@fork_new_process_for_each_test
11 +def test_collective_rpc(tp_size, backend):
12 + if tp_size == 1 and backend == "ray":
13 + pytest.skip("Skip duplicate test case")
14 + if tp_size == 1:
15 + backend = None
16 +
17 + # intentionally define the method and class in the test function,
18 + # to test if they can be serialized and sent to the workers
19 + def echo_rank(self):
20 + return self.rank
21 +
22 + from vllm.worker.worker import Worker
23 +
24 + class MyWorker(Worker):
25 +
26 + def echo_rank(self):
27 + return self.rank
28 +
29 + llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct",
30 + enforce_eager=True,
31 + load_format="dummy",
32 + tensor_parallel_size=tp_size,
33 + distributed_executor_backend=backend,
34 + worker_cls=MyWorker)
35 + for method in ["echo_rank", echo_rank]:
36 + assert llm.collective_rpc(method) == list(range(tp_size))
vllm/engine/llm_engine.py
+14 -3
@@ -5,10 +5,10 @@ from collections import deque
5 from contextlib import contextmanager
6 from dataclasses import dataclass
7 from functools import partial
8 -from typing import (TYPE_CHECKING, Callable, ClassVar, Deque, Dict, Iterable,
9 - List, Mapping, NamedTuple, Optional)
8 +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
9 + Iterable, List, Mapping, NamedTuple, Optional)
10 from typing import Sequence as GenericSequence
11 -from typing import Set, Type, Union, cast, overload
11 +from typing import Set, Tuple, Type, Union, cast, overload
12
13 import torch
14 from typing_extensions import TypeVar, deprecated
@@ -1816,6 +1816,17 @@ class LLMEngine:
1816 def stop_profile(self) -> None:
1817 self.model_executor.stop_profile()
1818
1819 + def collective_rpc(self,
1820 + method: Union[str, Callable],
1821 + timeout: Optional[float] = None,
1822 + args: Tuple = (),
1823 + kwargs: Optional[Dict] = None) -> List[Any]:
1824 + """
1825 + See LLM.collective_rpc for more details.
1826 + """
1827 + return self.model_executor.collective_rpc(method, timeout, args,
1828 + kwargs)
1829 +
1830 def check_health(self) -> None:
1831 if self.tokenizer:
1832 self.tokenizer.check_health()
vllm/entrypoints/llm.py
+9 -5
@@ -1,8 +1,8 @@
1 import itertools
2 import warnings
3 from contextlib import contextmanager
4 -from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
5 - Union, cast, overload)
4 +from typing import (Any, Callable, ClassVar, Dict, List, Optional, Sequence,
5 + Tuple, Type, Union, cast, overload)
6
7 import cloudpickle
8 from tqdm import tqdm
@@ -464,7 +464,7 @@ class LLM:
464 return self.engine_class.validate_outputs(outputs, RequestOutput)
465
466 def collective_rpc(self,
467 - method: str,
467 + method: Union[str, Callable],
468 timeout: Optional[float] = None,
469 args: Tuple = (),
470 kwargs: Optional[Dict] = None) -> List[Any]:
@@ -476,9 +476,13 @@ class LLM:
476 Then, users can call the new methods through this API.
477 It is recommended to use this API to only pass control messages,
478 and set up data-plane communication to pass data.
479 + The method can also be a callable, which will be serialized
480 + and sent to all workers to execute.
481 + If the method is a callable, it should accept an additional
482 + `self` argument, in addition to the arguments passed in `args`
483 + and `kwargs`. The `self` argument will be the worker object.
484 """
480 - return self.llm_engine.model_executor.collective_rpc(
481 - method, timeout, args, kwargs)
485 + return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
486
487 def beam_search(
488 self,
vllm/executor/executor_base.py
+5 -4
@@ -1,6 +1,7 @@
1 import asyncio
2 from abc import ABC, abstractmethod
3 -from typing import Any, Awaitable, Dict, List, Optional, Set, Tuple, Union
3 +from typing import (Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple,
4 + Union)
5
6 from vllm.config import VllmConfig
7 from vllm.logger import init_logger
@@ -47,7 +48,7 @@ class ExecutorBase(ABC):
48
49 @abstractmethod
50 def collective_rpc(self,
50 - method: str,
51 + method: Union[str, Callable],
52 timeout: Optional[float] = None,
53 args: Tuple = (),
54 kwargs: Optional[Dict] = None) -> List[Any]:
@@ -260,7 +261,7 @@ class DistributedExecutorBase(ExecutorBase):
261 raise NotImplementedError
262
263 def collective_rpc(self,
263 - method: str,
264 + method: Union[str, Callable],
265 timeout: Optional[float] = None,
266 args: Tuple = (),
267 kwargs: Optional[Dict] = None) -> List[Any]:
@@ -269,7 +270,7 @@ class DistributedExecutorBase(ExecutorBase):
270 @abstractmethod
271 def _run_workers(
272 self,
272 - method: str,
273 + method: Union[str, Callable],
274 *args,
275 async_run_tensor_parallel_workers_only: bool = False,
276 max_concurrent_workers: Optional[int] = None,
vllm/executor/mp_distributed_executor.py
+14 -7
@@ -1,5 +1,7 @@
1 import asyncio
2 -from typing import Any, List, Optional
2 +from typing import Any, Callable, List, Optional, Union
3 +
4 +import cloudpickle
5
6 from vllm.executor.executor_base import DistributedExecutorBase
7 from vllm.executor.multiproc_worker_utils import (
@@ -9,7 +11,7 @@ from vllm.logger import init_logger
11 from vllm.model_executor.layers.sampler import SamplerOutput
12 from vllm.sequence import ExecuteModelRequest
13 from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
12 - get_ip, get_open_port, make_async)
14 + get_ip, get_open_port, make_async, run_method)
15 from vllm.worker.worker_base import WorkerWrapperBase
16
17 logger = init_logger(__name__)
@@ -107,7 +109,7 @@ class MultiprocessingDistributedExecutor(DistributedExecutorBase):
109
110 def _run_workers(
111 self,
110 - method: str,
112 + method: Union[str, Callable],
113 *args,
114 async_run_tensor_parallel_workers_only: bool = False,
115 max_concurrent_workers: Optional[int] = None,
@@ -121,6 +123,11 @@ class MultiprocessingDistributedExecutor(DistributedExecutorBase):
123 It will also be run asynchronously and return a list of futures
124 rather than blocking on the results.
125 """
126 + if isinstance(method, str):
127 + sent_method = method
128 + else:
129 + sent_method = cloudpickle.dumps(method)
130 + del method
131
132 if max_concurrent_workers:
133 raise NotImplementedError(
@@ -129,18 +136,18 @@ class MultiprocessingDistributedExecutor(DistributedExecutorBase):
136 if async_run_tensor_parallel_workers_only:
137 # Run only non-driver workers and just return futures.
138 return [
132 - worker.execute_method(method, *args, **kwargs)
139 + worker.execute_method(sent_method, *args, **kwargs)
140 for worker in self.non_driver_workers
141 ]
142
143 # Start all remote workers first.
144 worker_outputs = [
138 - worker.execute_method(method, *args, **kwargs)
145 + worker.execute_method(sent_method, *args, **kwargs)
146 for worker in self.workers
147 ]
148
142 - driver_worker_method = getattr(self.driver_worker, method)
143 - driver_worker_output = driver_worker_method(*args, **kwargs)
149 + driver_worker_output = run_method(self.driver_worker, sent_method,
150 + args, kwargs)
151
152 # Get the results of the workers.
153 return [driver_worker_output
vllm/executor/multiproc_worker_utils.py
+6 -6
@@ -15,7 +15,7 @@ import torch
15 from vllm.config import VllmConfig
16 from vllm.logger import init_logger
17 from vllm.triton_utils.importing import HAS_TRITON
18 -from vllm.utils import _check_multiproc_method, get_mp_context
18 +from vllm.utils import _check_multiproc_method, get_mp_context, run_method
19
20 if HAS_TRITON:
21 from vllm.triton_utils import maybe_set_triton_cache_manager
@@ -169,7 +169,7 @@ class ProcessWorkerWrapper:
169 self.process.start()
170
171 def _enqueue_task(self, future: Union[ResultFuture, asyncio.Future],
172 - method: str, args, kwargs):
172 + method: Union[str, bytes], args, kwargs):
173 task_id = uuid.uuid4()
174 self.tasks[task_id] = future
175 try:
@@ -180,12 +180,13 @@ class ProcessWorkerWrapper:
180 del self.tasks[task_id]
181 raise ChildProcessError("worker died") from e
182
183 - def execute_method(self, method: str, *args, **kwargs):
183 + def execute_method(self, method: Union[str, bytes], *args, **kwargs):
184 future: ResultFuture = ResultFuture()
185 self._enqueue_task(future, method, args, kwargs)
186 return future
187
188 - async def execute_method_async(self, method: str, *args, **kwargs):
188 + async def execute_method_async(self, method: Union[str, bytes], *args,
189 + **kwargs):
190 future = asyncio.get_running_loop().create_future()
191 self._enqueue_task(future, method, args, kwargs)
192 return await future
@@ -230,8 +231,7 @@ def _run_worker_process(
231 exception = None
232 task_id, method, args, kwargs = items
233 try:
233 - executor = getattr(worker, method)
234 - output = executor(*args, **kwargs)
234 + output = run_method(worker, method, args, kwargs)
235 except SystemExit:
236 raise
237 except KeyboardInterrupt:
vllm/executor/ray_distributed_executor.py
+10 -4
@@ -2,8 +2,9 @@ import asyncio
2 import os
3 from collections import defaultdict
4 from dataclasses import dataclass
5 -from typing import TYPE_CHECKING, Any, Dict, List, Optional
5 +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
6
7 +import cloudpickle
8 import msgspec
9
10 import vllm.envs as envs
@@ -410,7 +411,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
411
412 def _run_workers(
413 self,
413 - method: str,
414 + method: Union[str, Callable],
415 *args,
416 async_run_tensor_parallel_workers_only: bool = False,
417 max_concurrent_workers: Optional[int] = None,
@@ -426,6 +427,11 @@ class RayDistributedExecutor(DistributedExecutorBase):
427 rather than blocking on the results.
428 - args/kwargs: All workers share the same args/kwargs
429 """
430 + if isinstance(method, str):
431 + sent_method = method
432 + else:
433 + sent_method = cloudpickle.dumps(method)
434 + del method
435 if self.use_ray_spmd_worker:
436 assert not async_run_tensor_parallel_workers_only, (
437 "async_run_tensor_parallel_workers_only is not supported for "
@@ -440,7 +446,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
446 if async_run_tensor_parallel_workers_only:
447 ray_workers = self.non_driver_workers
448 ray_worker_outputs = [
443 - worker.execute_method.remote(method, *args, **kwargs)
449 + worker.execute_method.remote(sent_method, *args, **kwargs)
450 for worker in ray_workers
451 ]
452
@@ -455,7 +461,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
461 if not self.use_ray_spmd_worker:
462 # Start the driver worker after all the ray workers.
463 driver_worker_output = [
458 - self.driver_worker.execute_method(method, *args, **kwargs)
464 + self.driver_worker.execute_method(sent_method, *args, **kwargs)
465 ]
466
467 # Get the results of the ray workers.
vllm/executor/uniproc_executor.py
+5 -9
@@ -1,5 +1,5 @@
1 import os
2 -from typing import Any, Dict, List, Optional, Tuple
2 +from typing import Any, Callable, Dict, List, Optional, Tuple, Union
3
4 import torch
5 import torch.distributed as dist
@@ -7,7 +7,8 @@ import torch.distributed as dist
7 import vllm.envs as envs
8 from vllm.executor.executor_base import ExecutorBase
9 from vllm.logger import init_logger
10 -from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10 +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
11 + run_method)
12 from vllm.worker.worker_base import WorkerWrapperBase
13
14 logger = init_logger(__name__)
@@ -39,18 +40,13 @@ class UniProcExecutor(ExecutorBase):
40 self.collective_rpc("load_model")
41
42 def collective_rpc(self,
42 - method: str,
43 + method: Union[str, Callable],
44 timeout: Optional[float] = None,
45 args: Tuple = (),
46 kwargs: Optional[Dict] = None) -> List[Any]:
47 if kwargs is None:
48 kwargs = {}
48 - try:
49 - func = getattr(self.driver_worker, method)
50 - except AttributeError:
51 - raise NotImplementedError(f"Method {method} is not implemented.") \
52 - from None
53 - answer = func(*args, **kwargs)
49 + answer = run_method(self.driver_worker, method, args, kwargs)
50 return [answer]
51
52 def check_health(self) -> None:
vllm/utils.py
+23
@@ -36,6 +36,7 @@ from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
36 overload)
37 from uuid import uuid4
38
39 +import cloudpickle
40 import numpy as np
41 import numpy.typing as npt
42 import psutil
@@ -2166,3 +2167,25 @@ def bind_kv_cache(
2167 assert len(forward_ctx.kv_cache) == len(kv_cache)
2168 for ve, ve_kv_cache in enumerate(kv_cache):
2169 forward_ctx.kv_cache[ve] = ve_kv_cache[kv_cache_idx]
2170 +
2171 +
2172 +def run_method(obj: Any, method: Union[str, bytes, Callable], args: Tuple[Any],
2173 + kwargs: Dict[str, Any]) -> Any:
2174 + """
2175 + Run a method of an object with the given arguments and keyword arguments.
2176 + If the method is string, it will be converted to a method using getattr.
2177 + If the method is serialized bytes and will be deserialized using
2178 + cloudpickle.
2179 + If the method is a callable, it will be called directly.
2180 + """
2181 + if isinstance(method, bytes):
2182 + func = partial(cloudpickle.loads(method), obj)
2183 + elif isinstance(method, str):
2184 + try:
2185 + func = getattr(obj, method)
2186 + except AttributeError:
2187 + raise NotImplementedError(f"Method {method!r} is not"
2188 + " implemented.") from None
2189 + else:
2190 + func = partial(method, obj) # type: ignore
2191 + return func(*args, **kwargs)
vllm/v1/executor/multiproc_executor.py
+15 -4
@@ -6,9 +6,11 @@ import time
6 import weakref
7 from dataclasses import dataclass
8 from enum import Enum, auto
9 +from functools import partial
10 from multiprocessing.process import BaseProcess
10 -from typing import Any, Dict, List, Optional, Tuple
11 +from typing import Any, Callable, Dict, List, Optional, Tuple, Union
12
13 +import cloudpickle
14 import psutil
15 import zmq
16
@@ -120,7 +122,7 @@ class MultiprocExecutor(Executor):
122 return kv_cache_specs[0]
123
124 def collective_rpc(self,
123 - method: str,
125 + method: Union[str, Callable],
126 timeout: Optional[float] = None,
127 args: Tuple = (),
128 kwargs: Optional[Dict] = None) -> List[Any]:
@@ -141,7 +143,12 @@ class MultiprocExecutor(Executor):
143 kwargs = kwargs or {}
144
145 try:
144 - self.rpc_broadcast_mq.enqueue((method, args, kwargs))
146 + if isinstance(method, str):
147 + send_method = method
148 + else:
149 + send_method = cloudpickle.dumps(
150 + method, protocol=pickle.HIGHEST_PROTOCOL)
151 + self.rpc_broadcast_mq.enqueue((send_method, args, kwargs))
152
153 responses = [None] * self.world_size
154 for w in self.workers:
@@ -408,7 +415,11 @@ class WorkerProc:
415 method, args, kwargs = self.rpc_broadcast_mq.dequeue()
416
417 try:
411 - output = getattr(self.worker, method)(*args, **kwargs)
418 + if isinstance(method, str):
419 + func = getattr(self.worker, method)
420 + elif isinstance(method, bytes):
421 + func = partial(cloudpickle.loads(method), self.worker)
422 + output = func(*args, **kwargs)
423 except Exception as e:
424 self.worker_response_mq.enqueue(
425 (WorkerProc.ResponseStatus.FAILURE, e))
vllm/worker/worker_base.py
+5 -5
@@ -14,7 +14,8 @@ from vllm.lora.request import LoRARequest
14 from vllm.model_executor.layers.sampler import SamplerOutput
15 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
16 from vllm.utils import (enable_trace_function_call_for_thread,
17 - resolve_obj_by_qualname, update_environment_variables)
17 + resolve_obj_by_qualname, run_method,
18 + update_environment_variables)
19 from vllm.worker.model_runner_base import (BroadcastableModelInput,
20 ModelRunnerBase,
21 ModelRunnerInputBase)
@@ -539,17 +540,16 @@ class WorkerWrapperBase:
540 self.worker = worker_class(**kwargs)
541 assert self.worker is not None
542
542 - def execute_method(self, method: str, *args, **kwargs):
543 + def execute_method(self, method: Union[str, bytes], *args, **kwargs):
544 try:
545 target = self if self.worker is None else self.worker
545 - executor = getattr(target, method)
546 - return executor(*args, **kwargs)
546 + return run_method(target, method, args, kwargs)
547 except Exception as e:
548 # if the driver worker also execute methods,
549 # exceptions in the rest worker may cause deadlock in rpc like ray
550 # see https://github.com/vllm-project/vllm/issues/3455
551 # print the error and inform the user to solve the error
552 - msg = (f"Error executing method {method}. "
552 + msg = (f"Error executing method {method!r}. "
553 "This might cause deadlock in distributed execution.")
554 logger.exception(msg)
555 raise e