@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 676a99982f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Add MultiprocessingGPUExecutor (#4539)

Co-authored-by: SAHIL SUNEJA <suneja@us.ibm.com>

Nick Hill committed May 14, 2024 at 10:38 UTC 676a99982fe9aabe72fd52a91e08988a653a7359
11 files changed +225 -39
.buildkite/test-pipeline.yaml
+8 -4
@@ -34,10 +34,14 @@ steps:
34 mirror_hardwares: [amd]
35 commands:
36 - pytest -v -s distributed/test_pynccl_library.py
37 - - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s distributed/test_basic_distributed_correctness.py
38 - - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s distributed/test_basic_distributed_correctness.py
39 - - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s distributed/test_chunked_prefill_distributed.py
40 - - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s distributed/test_chunked_prefill_distributed.py
37 + - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
38 + - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
39 + - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
40 + - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
41 + - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
42 + - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
43 + - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
44 + - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
45
46 - label: Distributed Tests (Multiple Groups)
47 working_dir: "/vllm-workspace/tests"
tests/distributed/test_basic_distributed_correctness.py
+10 -7
@@ -20,6 +20,7 @@ import torch
20 MODELS = [
21 os.environ["TEST_DIST_MODEL"],
22 ]
23 +DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
24 VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
25
26
@@ -36,19 +37,21 @@ def test_models(
37 dtype: str,
38 max_tokens: int,
39 ) -> None:
39 - enforce_eager = False
40 + distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
41 +
42 backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
41 - if backend_by_env_var == "FLASHINFER":
42 - enforce_eager = True
43 + enforce_eager = backend_by_env_var == "FLASHINFER"
44
45 hf_model = hf_runner(model, dtype=dtype)
46 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47 del hf_model
48
48 - vllm_model = vllm_runner(model,
49 - dtype=dtype,
50 - tensor_parallel_size=2,
51 - enforce_eager=enforce_eager)
49 + vllm_model = vllm_runner(
50 + model,
51 + dtype=dtype,
52 + tensor_parallel_size=2,
53 + enforce_eager=enforce_eager,
54 + distributed_executor_backend=distributed_executor_backend)
55 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
56 del vllm_model
57
tests/distributed/test_chunked_prefill_distributed.py
+4
@@ -19,6 +19,7 @@ import torch
19 MODELS = [
20 os.environ["TEST_DIST_MODEL"],
21 ]
22 +DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
23
24
25 @pytest.mark.skipif(torch.cuda.device_count() < 2,
@@ -36,6 +37,8 @@ def test_models(
37 max_tokens: int,
38 chunked_prefill_token_size: int,
39 ) -> None:
40 + distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
41 +
42 # Add a chunked prefill config.
43 max_num_seqs = min(chunked_prefill_token_size, 256)
44 assert chunked_prefill_token_size != -1
@@ -53,6 +56,7 @@ def test_models(
56 max_num_seqs=max_num_seqs,
57 enable_chunked_prefill=enable_chunked_prefill,
58 max_num_batched_tokens=max_num_batched_tokens,
59 + distributed_executor_backend=distributed_executor_backend,
60 )
61 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
62 del vllm_model
tests/lora/test_mixtral.py
+1 -2
@@ -38,8 +38,7 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
38 enable_lora=True,
39 max_num_seqs=16,
40 max_loras=4,
41 - tensor_parallel_size=tp_size,
42 - worker_use_ray=True)
41 + tensor_parallel_size=tp_size)
42
43 expected_lora_output = [
44 "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
vllm/config.py
+29 -9
@@ -521,9 +521,7 @@ class ParallelConfig:
521 Args:
522 pipeline_parallel_size: Number of pipeline parallel groups.
523 tensor_parallel_size: Number of tensor parallel groups.
524 - worker_use_ray: Whether to use Ray for model workers. Will be set to
525 - True if either pipeline_parallel_size or tensor_parallel_size is
526 - greater than 1.
524 + worker_use_ray: Deprecated, use distributed_executor_backend instead.
525 max_parallel_loading_workers: Maximum number of multiple batches
526 when load model sequentially. To avoid RAM OOM when using tensor
527 parallel and large models.
@@ -533,22 +531,27 @@ class ParallelConfig:
531 If None, will use synchronous tokenization.
532 ray_workers_use_nsight: Whether to profile Ray workers with nsight, see
533 https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
534 + distributed_executor_backend: Backend to use for distributed model
535 + workers, either "ray" or "mp" (multiprocessing). If either
536 + pipeline_parallel_size or tensor_parallel_size is greater than 1,
537 + will default to "ray" if Ray is installed or "mp" otherwise.
538 """
539
540 def __init__(
541 self,
542 pipeline_parallel_size: int,
543 tensor_parallel_size: int,
542 - worker_use_ray: bool,
544 + worker_use_ray: Optional[bool] = None,
545 max_parallel_loading_workers: Optional[int] = None,
546 disable_custom_all_reduce: bool = False,
547 tokenizer_pool_config: Optional[TokenizerPoolConfig] = None,
548 ray_workers_use_nsight: bool = False,
549 placement_group: Optional["PlacementGroup"] = None,
550 + distributed_executor_backend: Optional[str] = None,
551 ) -> None:
552 self.pipeline_parallel_size = pipeline_parallel_size
553 self.tensor_parallel_size = tensor_parallel_size
551 - self.worker_use_ray = worker_use_ray
554 + self.distributed_executor_backend = distributed_executor_backend
555 self.max_parallel_loading_workers = max_parallel_loading_workers
556 self.disable_custom_all_reduce = disable_custom_all_reduce
557 self.tokenizer_pool_config = tokenizer_pool_config
@@ -556,14 +559,29 @@ class ParallelConfig:
559 self.placement_group = placement_group
560
561 self.world_size = pipeline_parallel_size * self.tensor_parallel_size
559 - if self.world_size > 1:
560 - self.worker_use_ray = True
562 + if worker_use_ray:
563 + if self.distributed_executor_backend is None:
564 + self.distributed_executor_backend = "ray"
565 + elif self.distributed_executor_backend != "ray":
566 + raise ValueError(f"worker-use-ray can't be used with "
567 + f"distributed executor backend "
568 + f"'{self.distributed_executor_backend}'.")
569 +
570 + if self.distributed_executor_backend is None and self.world_size > 1:
571 + from vllm.executor import ray_utils
572 + ray_found = ray_utils.ray is not None
573 + self.distributed_executor_backend = "ray" if ray_found else "mp"
574 +
575 self._verify_args()
576
577 def _verify_args(self) -> None:
578 if self.pipeline_parallel_size > 1:
579 raise NotImplementedError(
580 "Pipeline parallelism is not supported yet.")
581 + if self.distributed_executor_backend not in ("ray", "mp", None):
582 + raise ValueError(
583 + "Unrecognized distributed executor backend. Supported values "
584 + "are 'ray' or 'mp'.")
585 if not self.disable_custom_all_reduce and self.world_size > 1:
586 if is_hip():
587 self.disable_custom_all_reduce = True
@@ -575,7 +593,8 @@ class ParallelConfig:
593 logger.info(
594 "Disabled the custom all-reduce kernel because it is not "
595 "supported with pipeline parallelism.")
578 - if self.ray_workers_use_nsight and not self.worker_use_ray:
596 + if self.ray_workers_use_nsight and (
597 + not self.distributed_executor_backend == "ray"):
598 raise ValueError("Unable to use nsight profiling unless workers "
599 "run with Ray.")
600
@@ -887,7 +906,8 @@ class SpeculativeConfig:
906 pipeline_parallel_size=target_parallel_config.
907 pipeline_parallel_size,
908 tensor_parallel_size=target_parallel_config.tensor_parallel_size,
890 - worker_use_ray=target_parallel_config.worker_use_ray,
909 + distributed_executor_backend=target_parallel_config.
910 + distributed_executor_backend,
911 max_parallel_loading_workers=target_parallel_config.
912 max_parallel_loading_workers,
913 disable_custom_all_reduce=target_parallel_config.
vllm/engine/arg_utils.py
+12 -4
@@ -34,6 +34,7 @@ class EngineArgs:
34 seed: int = 0
35 max_model_len: Optional[int] = None
36 worker_use_ray: bool = False
37 + distributed_executor_backend: Optional[str] = None
38 pipeline_parallel_size: int = 1
39 tensor_parallel_size: int = 1
40 max_parallel_loading_workers: Optional[int] = None
@@ -221,10 +222,17 @@ class EngineArgs:
222 ' Can be overridden per request via guided_decoding_backend'
223 ' parameter.')
224 # Parallel arguments
224 - parser.add_argument('--worker-use-ray',
225 - action='store_true',
226 - help='Use Ray for distributed serving, will be '
227 - 'automatically set when using more than 1 GPU.')
225 + parser.add_argument(
226 + '--distributed-executor-backend',
227 + choices=['ray', 'mp'],
228 + default=EngineArgs.distributed_executor_backend,
229 + help='Backend to use for distributed serving. When more than 1 GPU '
230 + 'is used, will be automatically set to "ray" if installed '
231 + 'or "mp" (multiprocessing) otherwise.')
232 + parser.add_argument(
233 + '--worker-use-ray',
234 + action='store_true',
235 + help='Deprecated, use --distributed-executor-backend=ray.')
236 parser.add_argument('--pipeline-parallel-size',
237 '-pp',
238 type=int,
vllm/engine/async_llm_engine.py
+10 -6
@@ -348,27 +348,31 @@ class AsyncLLMEngine:
348 """Creates an async LLM engine from the engine arguments."""
349 # Create the engine configs.
350 engine_config = engine_args.create_engine_config()
351 + distributed_executor_backend = (
352 + engine_config.parallel_config.distributed_executor_backend)
353
354 if engine_config.device_config.device_type == "neuron":
355 from vllm.executor.neuron_executor import NeuronExecutorAsync
356 executor_class = NeuronExecutorAsync
357 elif engine_config.device_config.device_type == "cpu":
356 - assert not engine_config.parallel_config.worker_use_ray, (
357 - "Ray is not supported with the CPU backend.")
358 + assert distributed_executor_backend is None, (
359 + "Distributed execution is not supported with the CPU backend.")
360 from vllm.executor.cpu_executor import CPUExecutorAsync
361 executor_class = CPUExecutorAsync
360 - elif engine_config.parallel_config.worker_use_ray:
362 + elif distributed_executor_backend == "ray":
363 initialize_ray_cluster(engine_config.parallel_config)
364 from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
365 executor_class = RayGPUExecutorAsync
366 + elif distributed_executor_backend == "mp":
367 + from vllm.executor.multiproc_gpu_executor import (
368 + MultiprocessingGPUExecutorAsync)
369 + executor_class = MultiprocessingGPUExecutorAsync
370 else:
365 - assert engine_config.parallel_config.world_size == 1, (
366 - "Ray is required if parallel_config.world_size > 1.")
371 from vllm.executor.gpu_executor import GPUExecutorAsync
372 executor_class = GPUExecutorAsync
373 # Create the async LLM engine.
374 engine = cls(
371 - engine_config.parallel_config.worker_use_ray,
375 + distributed_executor_backend == "ray",
376 engine_args.engine_use_ray,
377 **engine_config.to_dict(),
378 executor_class=executor_class,
vllm/engine/llm_engine.py
+7 -3
@@ -274,6 +274,8 @@ class LLMEngine:
274 """Creates an LLM engine from the engine arguments."""
275 # Create the engine configs.
276 engine_config = engine_args.create_engine_config()
277 + distributed_executor_backend = (
278 + engine_config.parallel_config.distributed_executor_backend)
279
280 # Initialize the cluster and specify the executor class.
281 if engine_config.device_config.device_type == "neuron":
@@ -282,13 +284,15 @@ class LLMEngine:
284 elif engine_config.device_config.device_type == "cpu":
285 from vllm.executor.cpu_executor import CPUExecutor
286 executor_class = CPUExecutor
285 - elif engine_config.parallel_config.worker_use_ray:
287 + elif distributed_executor_backend == "ray":
288 initialize_ray_cluster(engine_config.parallel_config)
289 from vllm.executor.ray_gpu_executor import RayGPUExecutor
290 executor_class = RayGPUExecutor
291 + elif distributed_executor_backend == "mp":
292 + from vllm.executor.multiproc_gpu_executor import (
293 + MultiprocessingGPUExecutor)
294 + executor_class = MultiprocessingGPUExecutor
295 else:
290 - assert engine_config.parallel_config.world_size == 1, (
291 - "Ray is required if parallel_config.world_size > 1.")
296 from vllm.executor.gpu_executor import GPUExecutor
297 executor_class = GPUExecutor
298
vllm/executor/multiproc_gpu_executor.py new
+140
@@ -0,0 +1,140 @@
1 +import asyncio
2 +import os
3 +from functools import partial
4 +from typing import Any, Dict, Optional, Tuple
5 +
6 +from vllm.executor.distributed_gpu_executor import ( # yapf: disable
7 + DistributedGPUExecutor, DistributedGPUExecutorAsync)
8 +from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
9 + ResultHandler, WorkerMonitor)
10 +from vllm.logger import init_logger
11 +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
12 + get_vllm_instance_id, make_async)
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +class MultiprocessingGPUExecutor(DistributedGPUExecutor):
18 + """Python multiprocessing-based multi-GPU executor"""
19 +
20 + def _init_executor(self) -> None:
21 + assert (
22 + not self.speculative_config
23 + ), "Speculative decoding not yet supported for MultiProcGPU backend."
24 +
25 + # Create the parallel GPU workers.
26 + world_size = self.parallel_config.tensor_parallel_size
27 +
28 + # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
29 + if "CUDA_VISIBLE_DEVICES" not in os.environ:
30 + os.environ["CUDA_VISIBLE_DEVICES"] = (",".join(
31 + map(str, range(world_size))))
32 +
33 + # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
34 + os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
35 +
36 + from torch.cuda import device_count
37 + assert world_size <= device_count(), (
38 + "please set tensor_parallel_size to less than max local gpu count")
39 +
40 + distributed_init_method = get_distributed_init_method(
41 + get_ip(), get_open_port())
42 +
43 + if world_size == 1:
44 + self.workers = []
45 + else:
46 + result_handler = ResultHandler()
47 + self.workers = [
48 + ProcessWorkerWrapper(
49 + result_handler,
50 + partial(
51 + self._create_worker,
52 + rank=rank,
53 + local_rank=rank,
54 + distributed_init_method=distributed_init_method,
55 + )) for rank in range(1, world_size)
56 + ]
57 +
58 + self.worker_monitor = WorkerMonitor(self.workers, result_handler)
59 + result_handler.start()
60 + self.worker_monitor.start()
61 +
62 + self.driver_worker = self._create_worker(
63 + distributed_init_method=distributed_init_method)
64 + self._run_workers("init_device")
65 + self._run_workers("load_model",
66 + max_concurrent_workers=self.parallel_config.
67 + max_parallel_loading_workers)
68 +
69 + def shutdown(self):
70 + if (worker_monitor := getattr(self, "worker_monitor",
71 + None)) is not None:
72 + worker_monitor.close()
73 +
74 + def _run_workers(
75 + self,
76 + method: str,
77 + *args,
78 + driver_args: Optional[Tuple[Any, ...]] = None,
79 + driver_kwargs: Optional[Dict[str, Any]] = None,
80 + max_concurrent_workers: Optional[int] = None,
81 + **kwargs,
82 + ) -> Any:
83 + """Runs the given method on all workers."""
84 +
85 + if max_concurrent_workers:
86 + raise NotImplementedError(
87 + "max_concurrent_workers is not supported yet.")
88 +
89 + # Start the workers first.
90 + worker_outputs = [
91 + worker.execute_method(method, *args, **kwargs)
92 + for worker in self.workers
93 + ]
94 +
95 + if driver_args is None:
96 + driver_args = args
97 + if driver_kwargs is None:
98 + driver_kwargs = kwargs
99 +
100 + # Start the driver worker after all the ray workers.
101 + driver_worker_method = getattr(self.driver_worker, method)
102 + driver_worker_output = driver_worker_method(*driver_args,
103 + **driver_kwargs)
104 +
105 + # Get the results of the workers.
106 + return [driver_worker_output
107 + ] + [output.get() for output in worker_outputs]
108 +
109 + def check_health(self) -> None:
110 + """Raises an error if engine is unhealthy."""
111 + if not self.worker_monitor.is_alive():
112 + raise RuntimeError("Worker processes are not running")
113 +
114 +
115 +class MultiprocessingGPUExecutorAsync(MultiprocessingGPUExecutor,
116 + DistributedGPUExecutorAsync):
117 +
118 + async def _run_workers_async(
119 + self,
120 + method: str,
121 + *args,
122 + driver_args: Optional[Tuple[Any, ...]] = None,
123 + driver_kwargs: Optional[Dict[str, Any]] = None,
124 + **kwargs,
125 + ) -> Any:
126 + """Runs the given method on all workers."""
127 + if driver_args is None:
128 + driver_args = args
129 + if driver_kwargs is None:
130 + driver_kwargs = kwargs
131 +
132 + driver_executor = make_async(getattr(self.driver_worker, method))
133 +
134 + # Run all the workers asynchronously.
135 + coros = [driver_executor(*driver_args, **driver_kwargs)] + [
136 + worker.execute_method_async(method, *args, **kwargs)
137 + for worker in self.workers
138 + ]
139 +
140 + return await asyncio.gather(*coros)
vllm/executor/ray_gpu_executor.py
+2 -2
@@ -31,7 +31,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
31 assert (not self.speculative_config
32 ), "Speculative decoding not yet supported for RayGPU backend."
33
34 - assert self.parallel_config.worker_use_ray
34 + assert self.parallel_config.distributed_executor_backend == "ray"
35 placement_group = self.parallel_config.placement_group
36
37 # Disable Ray usage stats collection.
@@ -264,7 +264,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
264 f"required, but found {current_version}")
265
266 from ray.dag import InputNode, MultiOutputNode
267 - assert self.parallel_config.worker_use_ray
267 + assert self.parallel_config.distributed_executor_backend == "ray"
268
269 # Right now, compiled DAG requires at least 1 arg. We send
270 # a dummy value for now. It will be fixed soon.
vllm/executor/ray_utils.py
+2 -2
@@ -44,7 +44,7 @@ try:
44
45 except ImportError as e:
46 logger.warning(
47 - "Failed to import Ray with %r. For distributed inference, "
47 + "Failed to import Ray with %r. For multi-node inference, "
48 "please install Ray with `pip install ray`.", e)
49 ray = None # type: ignore
50 RayWorkerWrapper = None # type: ignore
@@ -67,7 +67,7 @@ def initialize_ray_cluster(
67 """
68 if ray is None:
69 raise ImportError(
70 - "Ray is not installed. Please install Ray to use distributed "
70 + "Ray is not installed. Please install Ray to use multi-node "
71 "serving.")
72
73 # Connect to a ray cluster.