@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65b89d16ee
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Ray] Integration compiled DAG off by default (#2471)

SangBin Cho committed Feb 9, 2024 at 02:57 UTC 65b89d16ee54063d0737e3da1b15e2177916118d
2 files changed +73 -7
vllm/engine/llm_engine.py
+55 -7
@@ -2,6 +2,7 @@ import copy
2 from collections import defaultdict
3 import os
4 import time
5 +import pickle
6 from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
7 Union)
8
@@ -30,6 +31,11 @@ if TYPE_CHECKING:
31 logger = init_logger(__name__)
32 _LOCAL_LOGGING_INTERVAL_SEC = 5
33
34 +# If the env var is set, it uses the Ray's compiled DAG API
35 +# which optimizes the control plane overhead.
36 +# Run VLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
37 +USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
38 +
39
40 class LLMEngine:
41 """An LLM engine that receives requests and generates texts.
@@ -124,6 +130,10 @@ class LLMEngine:
130 self.stat_logger = StatLogger(
131 local_interval=_LOCAL_LOGGING_INTERVAL_SEC)
132
133 + self.forward_dag = None
134 + if USE_RAY_COMPILED_DAG:
135 + self.forward_dag = self._compiled_ray_dag()
136 +
137 def get_tokenizer_for_seq(self, sequence: Sequence):
138 return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
139
@@ -806,7 +816,8 @@ class LLMEngine:
816 "blocks_to_swap_in": scheduler_outputs.blocks_to_swap_in,
817 "blocks_to_swap_out": scheduler_outputs.blocks_to_swap_out,
818 "blocks_to_copy": scheduler_outputs.blocks_to_copy,
809 - })
819 + },
820 + use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
821
822 # Only the driver worker returns the sampling results.
823 output = all_outputs[0]
@@ -966,6 +977,7 @@ class LLMEngine:
977 driver_args: Optional[List[Any]] = None,
978 driver_kwargs: Optional[Dict[str, Any]] = None,
979 max_concurrent_workers: Optional[int] = None,
980 + use_ray_compiled_dag: bool = False,
981 **kwargs,
982 ) -> Any:
983 """Runs the given method on all workers."""
@@ -974,11 +986,16 @@ class LLMEngine:
986 raise NotImplementedError(
987 "max_concurrent_workers is not supported yet.")
988
977 - # Start the ray workers first.
978 - ray_worker_outputs = [
979 - worker.execute_method.remote(method, *args, **kwargs)
980 - for worker in self.workers
981 - ]
989 + if use_ray_compiled_dag:
990 + # Right now, compiled DAG can only accept a single
991 + # input. TODO(sang): Fix it.
992 + output_channels = self.forward_dag.execute(1)
993 + else:
994 + # Start the ray workers first.
995 + ray_worker_outputs = [
996 + worker.execute_method.remote(method, *args, **kwargs)
997 + for worker in self.workers
998 + ]
999
1000 if driver_args is None:
1001 driver_args = args
@@ -991,6 +1008,37 @@ class LLMEngine:
1008
1009 # Get the results of the ray workers.
1010 if self.workers:
994 - ray_worker_outputs = ray.get(ray_worker_outputs)
1011 + if use_ray_compiled_dag:
1012 + try:
1013 + ray_worker_outputs = [
1014 + pickle.loads(chan.begin_read())
1015 + for chan in output_channels
1016 + ]
1017 + finally:
1018 + # Has to call end_read in order to reuse the DAG.
1019 + for chan in output_channels:
1020 + chan.end_read()
1021 + else:
1022 + ray_worker_outputs = ray.get(ray_worker_outputs)
1023
1024 return [driver_worker_output] + ray_worker_outputs
1025 +
1026 + def _compiled_ray_dag(self):
1027 + import pkg_resources
1028 + required_version = "2.9"
1029 + current_version = pkg_resources.get_distribution("ray").version
1030 + if current_version < required_version:
1031 + raise ValueError(f"Ray version {required_version} or greater is "
1032 + f"required, but found {current_version}")
1033 +
1034 + from ray.dag import MultiOutputNode, InputNode
1035 + assert self.parallel_config.worker_use_ray
1036 +
1037 + # Right now, compiled DAG requires at least 1 arg. We send
1038 + # a dummy value for now. It will be fixed soon.
1039 + with InputNode() as input_data:
1040 + forward_dag = MultiOutputNode([
1041 + worker.execute_model_compiled_dag_remote.bind(input_data)
1042 + for worker in self.workers
1043 + ])
1044 + return forward_dag.experimental_compile()
vllm/engine/ray_utils.py
+18
@@ -1,3 +1,5 @@
1 +import pickle
2 +
3 from typing import Optional, List, Tuple, TYPE_CHECKING
4
5 from vllm.config import ParallelConfig
@@ -18,6 +20,11 @@ try:
20 from transformers.dynamic_module_utils import init_hf_modules
21 init_hf_modules()
22 self.worker = None
23 + # Since the compiled DAG runs a main execution
24 + # in a different thread that calls cuda.set_device.
25 + # The flag indicates is set_device is called on
26 + # that thread.
27 + self.compiled_dag_cuda_device_set = False
28
29 def init_worker(self, worker_init_fn):
30 self.worker = worker_init_fn()
@@ -40,6 +47,17 @@ try:
47 def set_cuda_visible_devices(self, device_ids) -> None:
48 set_cuda_visible_devices(device_ids)
49
50 + def execute_model_compiled_dag_remote(self, ignored):
51 + """Used only when compiled DAG is enabled."""
52 + import torch
53 + if not self.compiled_dag_cuda_device_set:
54 + torch.cuda.set_device(self.worker.device)
55 + self.compiled_dag_cuda_device_set = True
56 +
57 + output = self.worker.execute_model()
58 + output = pickle.dumps(output)
59 + return output
60 +
61 except ImportError as e:
62 logger.warning(f"Failed to import Ray with {e!r}. "
63 "For distributed inference, please install Ray with "