A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Add MultiprocessingGPUExecutor (#4539)
Co-authored-by: SAHIL SUNEJA <suneja@us.ibm.com>
Nick Hill committed
May 14, 2024 at 10:38 UTC
676a99982fe9aabe72fd52a91e08988a653a7359
11 files changed
+225
-39
.buildkite/test-pipeline.yaml
+8
-4
@@ -34,10 +34,14 @@ steps:
34
mirror_hardwares: [amd]
35
commands:
36
- pytest -v -s distributed/test_pynccl_library.py
37
- - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s distributed/test_basic_distributed_correctness.py
38
- - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s distributed/test_basic_distributed_correctness.py
39
- - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s distributed/test_chunked_prefill_distributed.py
40
- - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s distributed/test_chunked_prefill_distributed.py
37
+ - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
38
+ - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
39
+ - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
40
+ - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
41
+ - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
42
+ - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
43
+ - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
44
+ - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
45
46
- label: Distributed Tests (Multiple Groups)
47
working_dir: "/vllm-workspace/tests"
tests/distributed/test_basic_distributed_correctness.py
+10
-7
@@ -20,6 +20,7 @@ import torch
20
MODELS = [
21
os.environ["TEST_DIST_MODEL"],
22
]
23
+DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
24
VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
25
26
@@ -36,19 +37,21 @@ def test_models(
37
dtype: str,
38
max_tokens: int,
39
) -> None:
39
- enforce_eager = False
40
+ distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
41
+
42
backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
41
- if backend_by_env_var == "FLASHINFER":
42
- enforce_eager = True
43
+ enforce_eager = backend_by_env_var == "FLASHINFER"
44
45
hf_model = hf_runner(model, dtype=dtype)
46
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47
del hf_model
48
48
- vllm_model = vllm_runner(model,
49
- dtype=dtype,
50
- tensor_parallel_size=2,
51
- enforce_eager=enforce_eager)
49
+ vllm_model = vllm_runner(
50
+ model,
51
+ dtype=dtype,
52
+ tensor_parallel_size=2,
53
+ enforce_eager=enforce_eager,
54
+ distributed_executor_backend=distributed_executor_backend)
55
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
56
del vllm_model
57
tests/distributed/test_chunked_prefill_distributed.py
+4
@@ -19,6 +19,7 @@ import torch
19
MODELS = [
20
os.environ["TEST_DIST_MODEL"],
21
]
22
+DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
23
24
25
@pytest.mark.skipif(torch.cuda.device_count() < 2,
@@ -36,6 +37,8 @@ def test_models(
37
max_tokens: int,
38
chunked_prefill_token_size: int,
39
) -> None:
40
+ distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
41
+
42
# Add a chunked prefill config.
43
max_num_seqs = min(chunked_prefill_token_size, 256)
44
assert chunked_prefill_token_size != -1
@@ -53,6 +56,7 @@ def test_models(
56
max_num_seqs=max_num_seqs,
57
enable_chunked_prefill=enable_chunked_prefill,
58
max_num_batched_tokens=max_num_batched_tokens,
59
+ distributed_executor_backend=distributed_executor_backend,
60
)
61
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
62
del vllm_model
tests/lora/test_mixtral.py
+1
-2
@@ -38,8 +38,7 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
38
enable_lora=True,
39
max_num_seqs=16,
40
max_loras=4,
41
- tensor_parallel_size=tp_size,
42
- worker_use_ray=True)
41
+ tensor_parallel_size=tp_size)
42
43
expected_lora_output = [
44
"give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
vllm/config.py
+29
-9
@@ -521,9 +521,7 @@ class ParallelConfig:
521
Args:
522
pipeline_parallel_size: Number of pipeline parallel groups.
523
tensor_parallel_size: Number of tensor parallel groups.
524
- worker_use_ray: Whether to use Ray for model workers. Will be set to
525
- True if either pipeline_parallel_size or tensor_parallel_size is
526
- greater than 1.
524
+ worker_use_ray: Deprecated, use distributed_executor_backend instead.
525
max_parallel_loading_workers: Maximum number of multiple batches
526
when load model sequentially. To avoid RAM OOM when using tensor
527
parallel and large models.
@@ -533,22 +531,27 @@ class ParallelConfig:
531
If None, will use synchronous tokenization.
532
ray_workers_use_nsight: Whether to profile Ray workers with nsight, see
533
https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
534
+ distributed_executor_backend: Backend to use for distributed model
535
+ workers, either "ray" or "mp" (multiprocessing). If either
536
+ pipeline_parallel_size or tensor_parallel_size is greater than 1,
537
+ will default to "ray" if Ray is installed or "mp" otherwise.
538
"""
539
540
def __init__(
541
self,
542
pipeline_parallel_size: int,
543
tensor_parallel_size: int,
542
- worker_use_ray: bool,
544
+ worker_use_ray: Optional[bool] = None,
545
max_parallel_loading_workers: Optional[int] = None,
546
disable_custom_all_reduce: bool = False,
547
tokenizer_pool_config: Optional[TokenizerPoolConfig] = None,
548
ray_workers_use_nsight: bool = False,
549
placement_group: Optional["PlacementGroup"] = None,
550
+ distributed_executor_backend: Optional[str] = None,
551
) -> None:
552
self.pipeline_parallel_size = pipeline_parallel_size
553
self.tensor_parallel_size = tensor_parallel_size
551
- self.worker_use_ray = worker_use_ray
554
+ self.distributed_executor_backend = distributed_executor_backend
555
self.max_parallel_loading_workers = max_parallel_loading_workers
556
self.disable_custom_all_reduce = disable_custom_all_reduce
557
self.tokenizer_pool_config = tokenizer_pool_config
@@ -556,14 +559,29 @@ class ParallelConfig:
559
self.placement_group = placement_group
560
561
self.world_size = pipeline_parallel_size * self.tensor_parallel_size
559
- if self.world_size > 1:
560
- self.worker_use_ray = True
562
+ if worker_use_ray:
563
+ if self.distributed_executor_backend is None:
564
+ self.distributed_executor_backend = "ray"
565
+ elif self.distributed_executor_backend != "ray":
566
+ raise ValueError(f"worker-use-ray can't be used with "
567
+ f"distributed executor backend "
568
+ f"'{self.distributed_executor_backend}'.")
569
+
570
+ if self.distributed_executor_backend is None and self.world_size > 1:
571
+ from vllm.executor import ray_utils
572
+ ray_found = ray_utils.ray is not None
573
+ self.distributed_executor_backend = "ray" if ray_found else "mp"
574
+
575
self._verify_args()
576
577
def _verify_args(self) -> None:
578
if self.pipeline_parallel_size > 1:
579
raise NotImplementedError(
580
"Pipeline parallelism is not supported yet.")
581
+ if self.distributed_executor_backend not in ("ray", "mp", None):
582
+ raise ValueError(
583
+ "Unrecognized distributed executor backend. Supported values "
584
+ "are 'ray' or 'mp'.")
585
if not self.disable_custom_all_reduce and self.world_size > 1:
586
if is_hip():
587
self.disable_custom_all_reduce = True
@@ -575,7 +593,8 @@ class ParallelConfig:
593
logger.info(
594
"Disabled the custom all-reduce kernel because it is not "
595
"supported with pipeline parallelism.")
578
- if self.ray_workers_use_nsight and not self.worker_use_ray:
596
+ if self.ray_workers_use_nsight and (
597
+ not self.distributed_executor_backend == "ray"):
598
raise ValueError("Unable to use nsight profiling unless workers "
599
"run with Ray.")
600
@@ -887,7 +906,8 @@ class SpeculativeConfig:
906
pipeline_parallel_size=target_parallel_config.
907
pipeline_parallel_size,
908
tensor_parallel_size=target_parallel_config.tensor_parallel_size,
890
- worker_use_ray=target_parallel_config.worker_use_ray,
909
+ distributed_executor_backend=target_parallel_config.
910
+ distributed_executor_backend,
911
max_parallel_loading_workers=target_parallel_config.
912
max_parallel_loading_workers,
913
disable_custom_all_reduce=target_parallel_config.
vllm/engine/arg_utils.py
+12
-4
@@ -34,6 +34,7 @@ class EngineArgs:
34
seed: int = 0
35
max_model_len: Optional[int] = None
36
worker_use_ray: bool = False
37
+ distributed_executor_backend: Optional[str] = None
38
pipeline_parallel_size: int = 1
39
tensor_parallel_size: int = 1
40
max_parallel_loading_workers: Optional[int] = None
@@ -221,10 +222,17 @@ class EngineArgs:
222
' Can be overridden per request via guided_decoding_backend'
223
' parameter.')
224
# Parallel arguments
224
- parser.add_argument('--worker-use-ray',
225
- action='store_true',
226
- help='Use Ray for distributed serving, will be '
227
- 'automatically set when using more than 1 GPU.')
225
+ parser.add_argument(
226
+ '--distributed-executor-backend',
227
+ choices=['ray', 'mp'],
228
+ default=EngineArgs.distributed_executor_backend,
229
+ help='Backend to use for distributed serving. When more than 1 GPU '
230
+ 'is used, will be automatically set to "ray" if installed '
231
+ 'or "mp" (multiprocessing) otherwise.')
232
+ parser.add_argument(
233
+ '--worker-use-ray',
234
+ action='store_true',
235
+ help='Deprecated, use --distributed-executor-backend=ray.')
236
parser.add_argument('--pipeline-parallel-size',
237
'-pp',
238
type=int,
vllm/engine/async_llm_engine.py
+10
-6
@@ -348,27 +348,31 @@ class AsyncLLMEngine:
348
"""Creates an async LLM engine from the engine arguments."""
349
# Create the engine configs.
350
engine_config = engine_args.create_engine_config()
351
+ distributed_executor_backend = (
352
+ engine_config.parallel_config.distributed_executor_backend)
353
354
if engine_config.device_config.device_type == "neuron":
355
from vllm.executor.neuron_executor import NeuronExecutorAsync
356
executor_class = NeuronExecutorAsync
357
elif engine_config.device_config.device_type == "cpu":
356
- assert not engine_config.parallel_config.worker_use_ray, (
357
- "Ray is not supported with the CPU backend.")
358
+ assert distributed_executor_backend is None, (
359
+ "Distributed execution is not supported with the CPU backend.")
360
from vllm.executor.cpu_executor import CPUExecutorAsync
361
executor_class = CPUExecutorAsync
360
- elif engine_config.parallel_config.worker_use_ray:
362
+ elif distributed_executor_backend == "ray":
363
initialize_ray_cluster(engine_config.parallel_config)
364
from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
365
executor_class = RayGPUExecutorAsync
366
+ elif distributed_executor_backend == "mp":
367
+ from vllm.executor.multiproc_gpu_executor import (
368
+ MultiprocessingGPUExecutorAsync)
369
+ executor_class = MultiprocessingGPUExecutorAsync
370
else:
365
- assert engine_config.parallel_config.world_size == 1, (
366
- "Ray is required if parallel_config.world_size > 1.")
371
from vllm.executor.gpu_executor import GPUExecutorAsync
372
executor_class = GPUExecutorAsync
373
# Create the async LLM engine.
374
engine = cls(
371
- engine_config.parallel_config.worker_use_ray,
375
+ distributed_executor_backend == "ray",
376
engine_args.engine_use_ray,
377
**engine_config.to_dict(),
378
executor_class=executor_class,
vllm/engine/llm_engine.py
+7
-3
@@ -274,6 +274,8 @@ class LLMEngine:
274
"""Creates an LLM engine from the engine arguments."""
275
# Create the engine configs.
276
engine_config = engine_args.create_engine_config()
277
+ distributed_executor_backend = (
278
+ engine_config.parallel_config.distributed_executor_backend)
279
280
# Initialize the cluster and specify the executor class.
281
if engine_config.device_config.device_type == "neuron":
@@ -282,13 +284,15 @@ class LLMEngine:
284
elif engine_config.device_config.device_type == "cpu":
285
from vllm.executor.cpu_executor import CPUExecutor
286
executor_class = CPUExecutor
285
- elif engine_config.parallel_config.worker_use_ray:
287
+ elif distributed_executor_backend == "ray":
288
initialize_ray_cluster(engine_config.parallel_config)
289
from vllm.executor.ray_gpu_executor import RayGPUExecutor
290
executor_class = RayGPUExecutor
291
+ elif distributed_executor_backend == "mp":
292
+ from vllm.executor.multiproc_gpu_executor import (
293
+ MultiprocessingGPUExecutor)
294
+ executor_class = MultiprocessingGPUExecutor
295
else:
290
- assert engine_config.parallel_config.world_size == 1, (
291
- "Ray is required if parallel_config.world_size > 1.")
296
from vllm.executor.gpu_executor import GPUExecutor
297
executor_class = GPUExecutor
298
vllm/executor/multiproc_gpu_executor.py
new
+140
@@ -0,0 +1,140 @@
1
+import asyncio
2
+import os
3
+from functools import partial
4
+from typing import Any, Dict, Optional, Tuple
5
+
6
+from vllm.executor.distributed_gpu_executor import ( # yapf: disable
7
+ DistributedGPUExecutor, DistributedGPUExecutorAsync)
8
+from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
9
+ ResultHandler, WorkerMonitor)
10
+from vllm.logger import init_logger
11
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
12
+ get_vllm_instance_id, make_async)
13
+
14
+logger = init_logger(__name__)
15
+
16
+
17
+class MultiprocessingGPUExecutor(DistributedGPUExecutor):
18
+ """Python multiprocessing-based multi-GPU executor"""
19
+
20
+ def _init_executor(self) -> None:
21
+ assert (
22
+ not self.speculative_config
23
+ ), "Speculative decoding not yet supported for MultiProcGPU backend."
24
+
25
+ # Create the parallel GPU workers.
26
+ world_size = self.parallel_config.tensor_parallel_size
27
+
28
+ # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
29
+ if "CUDA_VISIBLE_DEVICES" not in os.environ:
30
+ os.environ["CUDA_VISIBLE_DEVICES"] = (",".join(
31
+ map(str, range(world_size))))
32
+
33
+ # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
34
+ os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
35
+
36
+ from torch.cuda import device_count
37
+ assert world_size <= device_count(), (
38
+ "please set tensor_parallel_size to less than max local gpu count")
39
+
40
+ distributed_init_method = get_distributed_init_method(
41
+ get_ip(), get_open_port())
42
+
43
+ if world_size == 1:
44
+ self.workers = []
45
+ else:
46
+ result_handler = ResultHandler()
47
+ self.workers = [
48
+ ProcessWorkerWrapper(
49
+ result_handler,
50
+ partial(
51
+ self._create_worker,
52
+ rank=rank,
53
+ local_rank=rank,
54
+ distributed_init_method=distributed_init_method,
55
+ )) for rank in range(1, world_size)
56
+ ]
57
+
58
+ self.worker_monitor = WorkerMonitor(self.workers, result_handler)
59
+ result_handler.start()
60
+ self.worker_monitor.start()
61
+
62
+ self.driver_worker = self._create_worker(
63
+ distributed_init_method=distributed_init_method)
64
+ self._run_workers("init_device")
65
+ self._run_workers("load_model",
66
+ max_concurrent_workers=self.parallel_config.
67
+ max_parallel_loading_workers)
68
+
69
+ def shutdown(self):
70
+ if (worker_monitor := getattr(self, "worker_monitor",
71
+ None)) is not None:
72
+ worker_monitor.close()
73
+
74
+ def _run_workers(
75
+ self,
76
+ method: str,
77
+ *args,
78
+ driver_args: Optional[Tuple[Any, ...]] = None,
79
+ driver_kwargs: Optional[Dict[str, Any]] = None,
80
+ max_concurrent_workers: Optional[int] = None,
81
+ **kwargs,
82
+ ) -> Any:
83
+ """Runs the given method on all workers."""
84
+
85
+ if max_concurrent_workers:
86
+ raise NotImplementedError(
87
+ "max_concurrent_workers is not supported yet.")
88
+
89
+ # Start the workers first.
90
+ worker_outputs = [
91
+ worker.execute_method(method, *args, **kwargs)
92
+ for worker in self.workers
93
+ ]
94
+
95
+ if driver_args is None:
96
+ driver_args = args
97
+ if driver_kwargs is None:
98
+ driver_kwargs = kwargs
99
+
100
+ # Start the driver worker after all the ray workers.
101
+ driver_worker_method = getattr(self.driver_worker, method)
102
+ driver_worker_output = driver_worker_method(*driver_args,
103
+ **driver_kwargs)
104
+
105
+ # Get the results of the workers.
106
+ return [driver_worker_output
107
+ ] + [output.get() for output in worker_outputs]
108
+
109
+ def check_health(self) -> None:
110
+ """Raises an error if engine is unhealthy."""
111
+ if not self.worker_monitor.is_alive():
112
+ raise RuntimeError("Worker processes are not running")
113
+
114
+
115
+class MultiprocessingGPUExecutorAsync(MultiprocessingGPUExecutor,
116
+ DistributedGPUExecutorAsync):
117
+
118
+ async def _run_workers_async(
119
+ self,
120
+ method: str,
121
+ *args,
122
+ driver_args: Optional[Tuple[Any, ...]] = None,
123
+ driver_kwargs: Optional[Dict[str, Any]] = None,
124
+ **kwargs,
125
+ ) -> Any:
126
+ """Runs the given method on all workers."""
127
+ if driver_args is None:
128
+ driver_args = args
129
+ if driver_kwargs is None:
130
+ driver_kwargs = kwargs
131
+
132
+ driver_executor = make_async(getattr(self.driver_worker, method))
133
+
134
+ # Run all the workers asynchronously.
135
+ coros = [driver_executor(*driver_args, **driver_kwargs)] + [
136
+ worker.execute_method_async(method, *args, **kwargs)
137
+ for worker in self.workers
138
+ ]
139
+
140
+ return await asyncio.gather(*coros)
vllm/executor/ray_gpu_executor.py
+2
-2
@@ -31,7 +31,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
31
assert (not self.speculative_config
32
), "Speculative decoding not yet supported for RayGPU backend."
33
34
- assert self.parallel_config.worker_use_ray
34
+ assert self.parallel_config.distributed_executor_backend == "ray"
35
placement_group = self.parallel_config.placement_group
36
37
# Disable Ray usage stats collection.
@@ -264,7 +264,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
264
f"required, but found {current_version}")
265
266
from ray.dag import InputNode, MultiOutputNode
267
- assert self.parallel_config.worker_use_ray
267
+ assert self.parallel_config.distributed_executor_backend == "ray"
268
269
# Right now, compiled DAG requires at least 1 arg. We send
270
# a dummy value for now. It will be fixed soon.
vllm/executor/ray_utils.py
+2
-2
@@ -44,7 +44,7 @@ try:
44
45
except ImportError as e:
46
logger.warning(
47
- "Failed to import Ray with %r. For distributed inference, "
47
+ "Failed to import Ray with %r. For multi-node inference, "
48
"please install Ray with `pip install ray`.", e)
49
ray = None # type: ignore
50
RayWorkerWrapper = None # type: ignore
@@ -67,7 +67,7 @@ def initialize_ray_cluster(
67
"""
68
if ray is None:
69
raise ImportError(
70
- "Ray is not installed. Please install Ray to use distributed "
70
+ "Ray is not installed. Please install Ray to use multi-node "
71
"serving.")
72
73
# Connect to a ray cluster.