A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[core] allow callable in collective_rpc (#12151)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 17, 2025 at 20:47 UTC
87a0c076afafb93dd082ff3876bea08adca56c56
13 files changed
+147
-50
.buildkite/test-pipeline.yaml
+3
-1
@@ -107,7 +107,7 @@ steps:
107
source_file_dependencies:
108
- vllm/
109
commands:
110
- - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_generate_multiple_loras.py --ignore=entrypoints/llm/test_guided_generate.py
110
+ - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_generate_multiple_loras.py --ignore=entrypoints/llm/test_guided_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
111
- pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
112
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
113
- pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
@@ -466,7 +466,9 @@ steps:
466
- vllm/worker/worker_base.py
467
- vllm/worker/worker.py
468
- vllm/worker/model_runner.py
469
+ - entrypoints/llm/test_collective_rpc.py
470
commands:
471
+ - pytest -v -s entrypoints/llm/test_collective_rpc.py
472
- torchrun --nproc-per-node=2 distributed/test_torchrun_example.py
473
- pytest -v -s ./compile/test_basic_correctness.py
474
- pytest -v -s ./compile/test_wrapper.py
tests/engine/test_custom_executor.py
+2
-2
@@ -1,6 +1,6 @@
1
import asyncio
2
import os
3
-from typing import Any, Dict, List, Optional, Tuple
3
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
4
5
import pytest
6
@@ -18,7 +18,7 @@ class Mock:
18
class CustomUniExecutor(UniProcExecutor):
19
20
def collective_rpc(self,
21
- method: str,
21
+ method: Union[str, Callable],
22
timeout: Optional[float] = None,
23
args: Tuple = (),
24
kwargs: Optional[Dict] = None) -> List[Any]:
tests/entrypoints/llm/test_collective_rpc.py
new
+36
@@ -0,0 +1,36 @@
1
+import pytest
2
+
3
+from vllm import LLM
4
+
5
+from ...utils import fork_new_process_for_each_test
6
+
7
+
8
+@pytest.mark.parametrize("tp_size", [1, 2])
9
+@pytest.mark.parametrize("backend", ["mp", "ray"])
10
+@fork_new_process_for_each_test
11
+def test_collective_rpc(tp_size, backend):
12
+ if tp_size == 1 and backend == "ray":
13
+ pytest.skip("Skip duplicate test case")
14
+ if tp_size == 1:
15
+ backend = None
16
+
17
+ # intentionally define the method and class in the test function,
18
+ # to test if they can be serialized and sent to the workers
19
+ def echo_rank(self):
20
+ return self.rank
21
+
22
+ from vllm.worker.worker import Worker
23
+
24
+ class MyWorker(Worker):
25
+
26
+ def echo_rank(self):
27
+ return self.rank
28
+
29
+ llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct",
30
+ enforce_eager=True,
31
+ load_format="dummy",
32
+ tensor_parallel_size=tp_size,
33
+ distributed_executor_backend=backend,
34
+ worker_cls=MyWorker)
35
+ for method in ["echo_rank", echo_rank]:
36
+ assert llm.collective_rpc(method) == list(range(tp_size))
vllm/engine/llm_engine.py
+14
-3
@@ -5,10 +5,10 @@ from collections import deque
5
from contextlib import contextmanager
6
from dataclasses import dataclass
7
from functools import partial
8
-from typing import (TYPE_CHECKING, Callable, ClassVar, Deque, Dict, Iterable,
9
- List, Mapping, NamedTuple, Optional)
8
+from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
9
+ Iterable, List, Mapping, NamedTuple, Optional)
10
from typing import Sequence as GenericSequence
11
-from typing import Set, Type, Union, cast, overload
11
+from typing import Set, Tuple, Type, Union, cast, overload
12
13
import torch
14
from typing_extensions import TypeVar, deprecated
@@ -1816,6 +1816,17 @@ class LLMEngine:
1816
def stop_profile(self) -> None:
1817
self.model_executor.stop_profile()
1818
1819
+ def collective_rpc(self,
1820
+ method: Union[str, Callable],
1821
+ timeout: Optional[float] = None,
1822
+ args: Tuple = (),
1823
+ kwargs: Optional[Dict] = None) -> List[Any]:
1824
+ """
1825
+ See LLM.collective_rpc for more details.
1826
+ """
1827
+ return self.model_executor.collective_rpc(method, timeout, args,
1828
+ kwargs)
1829
+
1830
def check_health(self) -> None:
1831
if self.tokenizer:
1832
self.tokenizer.check_health()
vllm/entrypoints/llm.py
+9
-5
@@ -1,8 +1,8 @@
1
import itertools
2
import warnings
3
from contextlib import contextmanager
4
-from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
5
- Union, cast, overload)
4
+from typing import (Any, Callable, ClassVar, Dict, List, Optional, Sequence,
5
+ Tuple, Type, Union, cast, overload)
6
7
import cloudpickle
8
from tqdm import tqdm
@@ -464,7 +464,7 @@ class LLM:
464
return self.engine_class.validate_outputs(outputs, RequestOutput)
465
466
def collective_rpc(self,
467
- method: str,
467
+ method: Union[str, Callable],
468
timeout: Optional[float] = None,
469
args: Tuple = (),
470
kwargs: Optional[Dict] = None) -> List[Any]:
@@ -476,9 +476,13 @@ class LLM:
476
Then, users can call the new methods through this API.
477
It is recommended to use this API to only pass control messages,
478
and set up data-plane communication to pass data.
479
+ The method can also be a callable, which will be serialized
480
+ and sent to all workers to execute.
481
+ If the method is a callable, it should accept an additional
482
+ `self` argument, in addition to the arguments passed in `args`
483
+ and `kwargs`. The `self` argument will be the worker object.
484
"""
480
- return self.llm_engine.model_executor.collective_rpc(
481
- method, timeout, args, kwargs)
485
+ return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
486
487
def beam_search(
488
self,
vllm/executor/executor_base.py
+5
-4
@@ -1,6 +1,7 @@
1
import asyncio
2
from abc import ABC, abstractmethod
3
-from typing import Any, Awaitable, Dict, List, Optional, Set, Tuple, Union
3
+from typing import (Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple,
4
+ Union)
5
6
from vllm.config import VllmConfig
7
from vllm.logger import init_logger
@@ -47,7 +48,7 @@ class ExecutorBase(ABC):
48
49
@abstractmethod
50
def collective_rpc(self,
50
- method: str,
51
+ method: Union[str, Callable],
52
timeout: Optional[float] = None,
53
args: Tuple = (),
54
kwargs: Optional[Dict] = None) -> List[Any]:
@@ -260,7 +261,7 @@ class DistributedExecutorBase(ExecutorBase):
261
raise NotImplementedError
262
263
def collective_rpc(self,
263
- method: str,
264
+ method: Union[str, Callable],
265
timeout: Optional[float] = None,
266
args: Tuple = (),
267
kwargs: Optional[Dict] = None) -> List[Any]:
@@ -269,7 +270,7 @@ class DistributedExecutorBase(ExecutorBase):
270
@abstractmethod
271
def _run_workers(
272
self,
272
- method: str,
273
+ method: Union[str, Callable],
274
*args,
275
async_run_tensor_parallel_workers_only: bool = False,
276
max_concurrent_workers: Optional[int] = None,
vllm/executor/mp_distributed_executor.py
+14
-7
@@ -1,5 +1,7 @@
1
import asyncio
2
-from typing import Any, List, Optional
2
+from typing import Any, Callable, List, Optional, Union
3
+
4
+import cloudpickle
5
6
from vllm.executor.executor_base import DistributedExecutorBase
7
from vllm.executor.multiproc_worker_utils import (
@@ -9,7 +11,7 @@ from vllm.logger import init_logger
11
from vllm.model_executor.layers.sampler import SamplerOutput
12
from vllm.sequence import ExecuteModelRequest
13
from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
12
- get_ip, get_open_port, make_async)
14
+ get_ip, get_open_port, make_async, run_method)
15
from vllm.worker.worker_base import WorkerWrapperBase
16
17
logger = init_logger(__name__)
@@ -107,7 +109,7 @@ class MultiprocessingDistributedExecutor(DistributedExecutorBase):
109
110
def _run_workers(
111
self,
110
- method: str,
112
+ method: Union[str, Callable],
113
*args,
114
async_run_tensor_parallel_workers_only: bool = False,
115
max_concurrent_workers: Optional[int] = None,
@@ -121,6 +123,11 @@ class MultiprocessingDistributedExecutor(DistributedExecutorBase):
123
It will also be run asynchronously and return a list of futures
124
rather than blocking on the results.
125
"""
126
+ if isinstance(method, str):
127
+ sent_method = method
128
+ else:
129
+ sent_method = cloudpickle.dumps(method)
130
+ del method
131
132
if max_concurrent_workers:
133
raise NotImplementedError(
@@ -129,18 +136,18 @@ class MultiprocessingDistributedExecutor(DistributedExecutorBase):
136
if async_run_tensor_parallel_workers_only:
137
# Run only non-driver workers and just return futures.
138
return [
132
- worker.execute_method(method, *args, **kwargs)
139
+ worker.execute_method(sent_method, *args, **kwargs)
140
for worker in self.non_driver_workers
141
]
142
143
# Start all remote workers first.
144
worker_outputs = [
138
- worker.execute_method(method, *args, **kwargs)
145
+ worker.execute_method(sent_method, *args, **kwargs)
146
for worker in self.workers
147
]
148
142
- driver_worker_method = getattr(self.driver_worker, method)
143
- driver_worker_output = driver_worker_method(*args, **kwargs)
149
+ driver_worker_output = run_method(self.driver_worker, sent_method,
150
+ args, kwargs)
151
152
# Get the results of the workers.
153
return [driver_worker_output
vllm/executor/multiproc_worker_utils.py
+6
-6
@@ -15,7 +15,7 @@ import torch
15
from vllm.config import VllmConfig
16
from vllm.logger import init_logger
17
from vllm.triton_utils.importing import HAS_TRITON
18
-from vllm.utils import _check_multiproc_method, get_mp_context
18
+from vllm.utils import _check_multiproc_method, get_mp_context, run_method
19
20
if HAS_TRITON:
21
from vllm.triton_utils import maybe_set_triton_cache_manager
@@ -169,7 +169,7 @@ class ProcessWorkerWrapper:
169
self.process.start()
170
171
def _enqueue_task(self, future: Union[ResultFuture, asyncio.Future],
172
- method: str, args, kwargs):
172
+ method: Union[str, bytes], args, kwargs):
173
task_id = uuid.uuid4()
174
self.tasks[task_id] = future
175
try:
@@ -180,12 +180,13 @@ class ProcessWorkerWrapper:
180
del self.tasks[task_id]
181
raise ChildProcessError("worker died") from e
182
183
- def execute_method(self, method: str, *args, **kwargs):
183
+ def execute_method(self, method: Union[str, bytes], *args, **kwargs):
184
future: ResultFuture = ResultFuture()
185
self._enqueue_task(future, method, args, kwargs)
186
return future
187
188
- async def execute_method_async(self, method: str, *args, **kwargs):
188
+ async def execute_method_async(self, method: Union[str, bytes], *args,
189
+ **kwargs):
190
future = asyncio.get_running_loop().create_future()
191
self._enqueue_task(future, method, args, kwargs)
192
return await future
@@ -230,8 +231,7 @@ def _run_worker_process(
231
exception = None
232
task_id, method, args, kwargs = items
233
try:
233
- executor = getattr(worker, method)
234
- output = executor(*args, **kwargs)
234
+ output = run_method(worker, method, args, kwargs)
235
except SystemExit:
236
raise
237
except KeyboardInterrupt:
vllm/executor/ray_distributed_executor.py
+10
-4
@@ -2,8 +2,9 @@ import asyncio
2
import os
3
from collections import defaultdict
4
from dataclasses import dataclass
5
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
5
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
6
7
+import cloudpickle
8
import msgspec
9
10
import vllm.envs as envs
@@ -410,7 +411,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
411
412
def _run_workers(
413
self,
413
- method: str,
414
+ method: Union[str, Callable],
415
*args,
416
async_run_tensor_parallel_workers_only: bool = False,
417
max_concurrent_workers: Optional[int] = None,
@@ -426,6 +427,11 @@ class RayDistributedExecutor(DistributedExecutorBase):
427
rather than blocking on the results.
428
- args/kwargs: All workers share the same args/kwargs
429
"""
430
+ if isinstance(method, str):
431
+ sent_method = method
432
+ else:
433
+ sent_method = cloudpickle.dumps(method)
434
+ del method
435
if self.use_ray_spmd_worker:
436
assert not async_run_tensor_parallel_workers_only, (
437
"async_run_tensor_parallel_workers_only is not supported for "
@@ -440,7 +446,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
446
if async_run_tensor_parallel_workers_only:
447
ray_workers = self.non_driver_workers
448
ray_worker_outputs = [
443
- worker.execute_method.remote(method, *args, **kwargs)
449
+ worker.execute_method.remote(sent_method, *args, **kwargs)
450
for worker in ray_workers
451
]
452
@@ -455,7 +461,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
461
if not self.use_ray_spmd_worker:
462
# Start the driver worker after all the ray workers.
463
driver_worker_output = [
458
- self.driver_worker.execute_method(method, *args, **kwargs)
464
+ self.driver_worker.execute_method(sent_method, *args, **kwargs)
465
]
466
467
# Get the results of the ray workers.
vllm/executor/uniproc_executor.py
+5
-9
@@ -1,5 +1,5 @@
1
import os
2
-from typing import Any, Dict, List, Optional, Tuple
2
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
3
4
import torch
5
import torch.distributed as dist
@@ -7,7 +7,8 @@ import torch.distributed as dist
7
import vllm.envs as envs
8
from vllm.executor.executor_base import ExecutorBase
9
from vllm.logger import init_logger
10
-from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
11
+ run_method)
12
from vllm.worker.worker_base import WorkerWrapperBase
13
14
logger = init_logger(__name__)
@@ -39,18 +40,13 @@ class UniProcExecutor(ExecutorBase):
40
self.collective_rpc("load_model")
41
42
def collective_rpc(self,
42
- method: str,
43
+ method: Union[str, Callable],
44
timeout: Optional[float] = None,
45
args: Tuple = (),
46
kwargs: Optional[Dict] = None) -> List[Any]:
47
if kwargs is None:
48
kwargs = {}
48
- try:
49
- func = getattr(self.driver_worker, method)
50
- except AttributeError:
51
- raise NotImplementedError(f"Method {method} is not implemented.") \
52
- from None
53
- answer = func(*args, **kwargs)
49
+ answer = run_method(self.driver_worker, method, args, kwargs)
50
return [answer]
51
52
def check_health(self) -> None:
vllm/utils.py
+23
@@ -36,6 +36,7 @@ from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
36
overload)
37
from uuid import uuid4
38
39
+import cloudpickle
40
import numpy as np
41
import numpy.typing as npt
42
import psutil
@@ -2166,3 +2167,25 @@ def bind_kv_cache(
2167
assert len(forward_ctx.kv_cache) == len(kv_cache)
2168
for ve, ve_kv_cache in enumerate(kv_cache):
2169
forward_ctx.kv_cache[ve] = ve_kv_cache[kv_cache_idx]
2170
+
2171
+
2172
+def run_method(obj: Any, method: Union[str, bytes, Callable], args: Tuple[Any],
2173
+ kwargs: Dict[str, Any]) -> Any:
2174
+ """
2175
+ Run a method of an object with the given arguments and keyword arguments.
2176
+ If the method is string, it will be converted to a method using getattr.
2177
+ If the method is serialized bytes and will be deserialized using
2178
+ cloudpickle.
2179
+ If the method is a callable, it will be called directly.
2180
+ """
2181
+ if isinstance(method, bytes):
2182
+ func = partial(cloudpickle.loads(method), obj)
2183
+ elif isinstance(method, str):
2184
+ try:
2185
+ func = getattr(obj, method)
2186
+ except AttributeError:
2187
+ raise NotImplementedError(f"Method {method!r} is not"
2188
+ " implemented.") from None
2189
+ else:
2190
+ func = partial(method, obj) # type: ignore
2191
+ return func(*args, **kwargs)
vllm/v1/executor/multiproc_executor.py
+15
-4
@@ -6,9 +6,11 @@ import time
6
import weakref
7
from dataclasses import dataclass
8
from enum import Enum, auto
9
+from functools import partial
10
from multiprocessing.process import BaseProcess
10
-from typing import Any, Dict, List, Optional, Tuple
11
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
12
13
+import cloudpickle
14
import psutil
15
import zmq
16
@@ -120,7 +122,7 @@ class MultiprocExecutor(Executor):
122
return kv_cache_specs[0]
123
124
def collective_rpc(self,
123
- method: str,
125
+ method: Union[str, Callable],
126
timeout: Optional[float] = None,
127
args: Tuple = (),
128
kwargs: Optional[Dict] = None) -> List[Any]:
@@ -141,7 +143,12 @@ class MultiprocExecutor(Executor):
143
kwargs = kwargs or {}
144
145
try:
144
- self.rpc_broadcast_mq.enqueue((method, args, kwargs))
146
+ if isinstance(method, str):
147
+ send_method = method
148
+ else:
149
+ send_method = cloudpickle.dumps(
150
+ method, protocol=pickle.HIGHEST_PROTOCOL)
151
+ self.rpc_broadcast_mq.enqueue((send_method, args, kwargs))
152
153
responses = [None] * self.world_size
154
for w in self.workers:
@@ -408,7 +415,11 @@ class WorkerProc:
415
method, args, kwargs = self.rpc_broadcast_mq.dequeue()
416
417
try:
411
- output = getattr(self.worker, method)(*args, **kwargs)
418
+ if isinstance(method, str):
419
+ func = getattr(self.worker, method)
420
+ elif isinstance(method, bytes):
421
+ func = partial(cloudpickle.loads(method), self.worker)
422
+ output = func(*args, **kwargs)
423
except Exception as e:
424
self.worker_response_mq.enqueue(
425
(WorkerProc.ResponseStatus.FAILURE, e))
vllm/worker/worker_base.py
+5
-5
@@ -14,7 +14,8 @@ from vllm.lora.request import LoRARequest
14
from vllm.model_executor.layers.sampler import SamplerOutput
15
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
16
from vllm.utils import (enable_trace_function_call_for_thread,
17
- resolve_obj_by_qualname, update_environment_variables)
17
+ resolve_obj_by_qualname, run_method,
18
+ update_environment_variables)
19
from vllm.worker.model_runner_base import (BroadcastableModelInput,
20
ModelRunnerBase,
21
ModelRunnerInputBase)
@@ -539,17 +540,16 @@ class WorkerWrapperBase:
540
self.worker = worker_class(**kwargs)
541
assert self.worker is not None
542
542
- def execute_method(self, method: str, *args, **kwargs):
543
+ def execute_method(self, method: Union[str, bytes], *args, **kwargs):
544
try:
545
target = self if self.worker is None else self.worker
545
- executor = getattr(target, method)
546
- return executor(*args, **kwargs)
546
+ return run_method(target, method, args, kwargs)
547
except Exception as e:
548
# if the driver worker also execute methods,
549
# exceptions in the rest worker may cause deadlock in rpc like ray
550
# see https://github.com/vllm-project/vllm/issues/3455
551
# print the error and inform the user to solve the error
552
- msg = (f"Error executing method {method}. "
552
+ msg = (f"Error executing method {method!r}. "
553
"This might cause deadlock in distributed execution.")
554
logger.exception(msg)
555
raise e