@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 92e793d91a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[core] LLM.collective_rpc interface and RLHF example (#12084)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 16, 2025 at 20:19 UTC 92e793d91a1a4e982662ecca0096e5edcafd21c6
6 files changed +270 -35
.buildkite/test-pipeline.yaml
+4
@@ -126,11 +126,15 @@ steps:
126 - tests/distributed
127 - tests/spec_decode/e2e/test_integration_dist_tp4
128 - tests/compile
129 + - examples/offline_inference/rlhf.py
130 commands:
131 - pytest -v -s distributed/test_utils.py
132 - pytest -v -s compile/test_basic_correctness.py
133 - pytest -v -s distributed/test_pynccl.py
134 - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
135 + # TODO: create a dedicated test section for multi-GPU example tests
136 + # when we have multiple distributed example tests
137 + - python3 ../examples/offline_inference/rlhf.py
138
139 - label: Metrics, Tracing Test # 10min
140 num_gpus: 2
examples/offline_inference/rlhf.py new
+191
@@ -0,0 +1,191 @@
1 +"""
2 +a simple demonstration of RLHF with vLLM, inspired by
3 +the OpenRLHF framework https://github.com/OpenRLHF/OpenRLHF .
4 +It follows the design that, training processes and inference processes
5 +are different, and they live on different GPUs.
6 +Training processes send prompts to inference processes to generate data,
7 +and also synchronize the weights of the model by broadcasting the weights
8 +from the training process to the inference process.
9 +Note that this is a simple demonstration of one training instance and one
10 +inference instance. In practice, there could be multiple training instances
11 +and multiple inference instances. For the full implementation, please refer
12 +to the OpenRLHF framework.
13 +"""
14 +import os
15 +
16 +import ray
17 +import torch
18 +from ray.util.placement_group import placement_group
19 +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
20 +from transformers import AutoModelForCausalLM
21 +
22 +from vllm import LLM, SamplingParams, configure_as_vllm_process
23 +from vllm.utils import get_ip, get_open_port
24 +from vllm.worker.worker import Worker
25 +
26 +
27 +def stateless_init_process_group(master_address, master_port, rank, world_size,
28 + device):
29 + """
30 + vLLM provides `StatelessProcessGroup` to create a process group
31 + without considering the global process group in torch.distributed.
32 + It is recommended to create `StatelessProcessGroup`, and then initialize
33 + the data-plane communication (NCCL) between external (train processes)
34 + and vLLM workers.
35 + """
36 + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
37 + from vllm.distributed.utils import StatelessProcessGroup
38 + pg = StatelessProcessGroup.create(host=master_address,
39 + port=master_port,
40 + rank=rank,
41 + world_size=world_size)
42 + pynccl = PyNcclCommunicator(pg, device=device)
43 + return pynccl
44 +
45 +
46 +class MyWorker(Worker):
47 + """
48 + The `MyWorker` class inherits from `Worker` to provide custom functions.
49 + For simplicity, we define the `MyWorker` class in this self-contained
50 + script. Normally, we should define the `MyWorker` class in a separate
51 + file and pass the qualified name of the class to the `worker_cls`
52 + parameter.
53 + """
54 +
55 + def init_weight_update_group(self, master_address, master_port,
56 + rank_offset, world_size):
57 + from vllm.distributed.parallel_state import get_world_group
58 + rank = get_world_group().rank + rank_offset
59 + self.model_update_group = stateless_init_process_group(
60 + master_address,
61 + master_port,
62 + rank,
63 + world_size,
64 + self.device,
65 + )
66 +
67 + def update_weight(self, name, dtype, shape):
68 + weight = torch.empty(shape, dtype=dtype, device="cuda")
69 + self.model_update_group.broadcast(weight,
70 + src=0,
71 + stream=torch.cuda.current_stream())
72 +
73 + self.model_runner.model.load_weights(weights=[(name, weight)])
74 +
75 + del weight
76 +
77 + def check_weights_changed(self):
78 + """
79 + Check if the weights are updated to 0.
80 + """
81 + weights_updated = True
82 + for name, p in self.model_runner.model.named_parameters():
83 + weights_updated = weights_updated and torch.allclose(
84 + p, torch.zeros_like(p))
85 + return weights_updated
86 +
87 +
88 +class MyLLM(LLM):
89 +
90 + def __init__(self, *args, **kwargs):
91 + # a hack to make the script work.
92 + # stop ray from manipulating CUDA_VISIBLE_DEVICES
93 + # at the top-level
94 + del os.environ["CUDA_VISIBLE_DEVICES"]
95 + super().__init__(*args, **kwargs)
96 +
97 +
98 +"""
99 +Start the training process, here we use huggingface transformers
100 +as an example to hold a model on GPU 0.
101 +
102 +It is important for all the processes outside of vLLM to call
103 +`configure_as_vllm_process` to set some common environment variables
104 +the same as vLLM workers.
105 +"""
106 +configure_as_vllm_process()
107 +
108 +train_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
109 +train_model.to("cuda:0")
110 +"""
111 +Start the inference process, here we use vLLM to hold a model on GPU 1 and
112 +GPU 2. For the details on how to use ray, please refer to the ray
113 +documentation https://docs.ray.io/en/latest/ .
114 +"""
115 +os.environ["CUDA_VISIBLE_DEVICES"] = "1,2"
116 +ray.init()
117 +
118 +pg_inference = placement_group([{"GPU": 1, "CPU": 0}] * 2)
119 +ray.get(pg_inference.ready())
120 +scheduling_inference = PlacementGroupSchedulingStrategy(
121 + placement_group=pg_inference,
122 + placement_group_capture_child_tasks=True,
123 + placement_group_bundle_index=0,
124 +)
125 +"""
126 +launch the vLLM inference engine.
127 +here we use `enforce_eager` to reduce the start time.
128 +"""
129 +llm = ray.remote(
130 + num_cpus=0,
131 + num_gpus=0,
132 + scheduling_strategy=scheduling_inference,
133 +)(MyLLM).remote(
134 + model="facebook/opt-125m",
135 + enforce_eager=True,
136 + worker_cls=MyWorker,
137 + tensor_parallel_size=2,
138 + distributed_executor_backend="ray",
139 +)
140 +
141 +# Generate texts from the prompts.
142 +prompts = [
143 + "Hello, my name is",
144 + "The president of the United States is",
145 + "The capital of France is",
146 + "The future of AI is",
147 +]
148 +
149 +sampling_params = SamplingParams(temperature=0)
150 +
151 +outputs = ray.get(llm.generate.remote(prompts, sampling_params))
152 +
153 +for output in outputs:
154 + prompt = output.prompt
155 + generated_text = output.outputs[0].text
156 + print(f"Prompt: {prompt!r}, "
157 + f"Generated text: {generated_text!r}")
158 +
159 +# set up the communication between the training process
160 +# and the inference engine.
161 +master_address = get_ip()
162 +master_port = get_open_port()
163 +
164 +handle = llm.collective_rpc.remote("init_weight_update_group",
165 + args=(master_address, master_port, 1, 3))
166 +model_update_group = stateless_init_process_group(master_address, master_port,
167 + 0, 3, torch.device("cuda:0"))
168 +ray.get(handle)
169 +
170 +# simulate training, modify the weights of the model.
171 +for name, p in train_model.named_parameters():
172 + p.data.zero_()
173 +
174 +# sync weight from the training process to the inference engine.
175 +for name, p in train_model.named_parameters():
176 + handle = llm.collective_rpc.remote("update_weight",
177 + args=(name, p.dtype, p.shape))
178 + model_update_group.broadcast(p, src=0, stream=torch.cuda.current_stream())
179 + ray.get(handle)
180 +
181 +# check if the weights are updated.
182 +assert all(ray.get(llm.collective_rpc.remote("check_weights_changed")))
183 +
184 +# use the updated model to generate texts, they will be nonsense
185 +# because the weights are all zeros.
186 +outputs_updated = ray.get(llm.generate.remote(prompts, sampling_params))
187 +for output in outputs_updated:
188 + prompt = output.prompt
189 + generated_text = output.outputs[0].text
190 + print(f"Prompt: {prompt!r}, "
191 + f"Generated text: {generated_text!r}")
vllm/__init__.py
+39
@@ -17,6 +17,44 @@ from vllm.sampling_params import SamplingParams
17
18 from .version import __version__, __version_tuple__
19
20 +
21 +def configure_as_vllm_process():
22 + """
23 + set some common config/environment variables that should be set
24 + for all processes created by vllm and all processes
25 + that interact with vllm workers.
26 + """
27 + import os
28 +
29 + import torch
30 +
31 + # see https://github.com/NVIDIA/nccl/issues/1234
32 + os.environ['NCCL_CUMEM_ENABLE'] = '0'
33 +
34 + # see https://github.com/vllm-project/vllm/issues/10480
35 + os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
36 + # see https://github.com/vllm-project/vllm/issues/10619
37 + torch._inductor.config.compile_threads = 1
38 +
39 + from vllm.platforms import current_platform
40 +
41 + if current_platform.is_xpu():
42 + # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158
43 + torch._dynamo.config.disable = True
44 + elif current_platform.is_hpu():
45 + # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1)
46 + # does not support torch.compile
47 + # Eager backend (PT_HPU_LAZY_MODE = 0) must be selected for
48 + # torch.compile support
49 + is_lazy = os.environ.get('PT_HPU_LAZY_MODE', '1') == '1'
50 + if is_lazy:
51 + torch._dynamo.config.disable = True
52 + # NOTE(kzawora) multi-HPU inference with HPUGraphs (lazy-only)
53 + # requires enabling lazy collectives
54 + # see https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html # noqa: E501
55 + os.environ['PT_HPU_ENABLE_LAZY_COLLECTIVES'] = 'true'
56 +
57 +
58 __all__ = [
59 "__version__",
60 "__version_tuple__",
@@ -42,4 +80,5 @@ __all__ = [
80 "AsyncEngineArgs",
81 "initialize_ray_cluster",
82 "PoolingParams",
83 + "configure_as_vllm_process",
84 ]
vllm/entrypoints/llm.py
+25
@@ -4,6 +4,7 @@ from contextlib import contextmanager
4 from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
5 Union, cast, overload)
6
7 +import cloudpickle
8 from tqdm import tqdm
9 from typing_extensions import deprecated
10
@@ -186,6 +187,13 @@ class LLM:
187 if "disable_log_stats" not in kwargs:
188 kwargs["disable_log_stats"] = True
189
190 + if "worker_cls" in kwargs:
191 + worker_cls = kwargs["worker_cls"]
192 + # if the worker_cls is not qualified string name,
193 + # we serialize it using cloudpickle to avoid pickling issues
194 + if isinstance(worker_cls, type):
195 + kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)
196 +
197 if compilation_config is not None:
198 if isinstance(compilation_config, (int, dict)):
199 compilation_config_instance = CompilationConfig.from_cli(
@@ -455,6 +463,23 @@ class LLM:
463 outputs = self._run_engine(use_tqdm=use_tqdm)
464 return self.engine_class.validate_outputs(outputs, RequestOutput)
465
466 + def collective_rpc(self,
467 + method: str,
468 + timeout: Optional[float] = None,
469 + args: Tuple = (),
470 + kwargs: Optional[Dict] = None) -> List[Any]:
471 + """
472 + Run a method on all workers, with homogeneous arguments.
473 + The main extension point for the LLM entrypoint.
474 + Users can provide custom worker class through `worker_cls`
475 + argument, and implement new methods in the worker class.
476 + Then, users can call the new methods through this API.
477 + It is recommended to use this API to only pass control messages,
478 + and set up data-plane communication to pass data.
479 + """
480 + return self.llm_engine.model_executor.collective_rpc(
481 + method, timeout, args, kwargs)
482 +
483 def beam_search(
484 self,
485 prompts: List[Union[TokensPrompt, TextPrompt]],
vllm/plugins/__init__.py
-31
@@ -1,9 +1,6 @@
1 import logging
2 -import os
2 from typing import Callable, Dict
3
5 -import torch
6 -
4 import vllm.envs as envs
5
6 logger = logging.getLogger(__name__)
@@ -50,34 +47,6 @@ def load_general_plugins():
47 processes. They should be designed in a way that they can be loaded
48 multiple times without causing issues.
49 """
53 -
54 - # all processes created by vllm will load plugins,
55 - # and here we can inject some common environment variables
56 - # for all processes.
57 -
58 - # see https://github.com/vllm-project/vllm/issues/10480
59 - os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
60 - # see https://github.com/vllm-project/vllm/issues/10619
61 - torch._inductor.config.compile_threads = 1
62 -
63 - from vllm.platforms import current_platform
64 -
65 - if current_platform.is_xpu():
66 - # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158
67 - torch._dynamo.config.disable = True
68 - if current_platform.is_hpu():
69 - # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1)
70 - # does not support torch.compile
71 - # Eager backend (PT_HPU_LAZY_MODE = 0) must be selected for
72 - # torch.compile support
73 - is_lazy = os.environ.get('PT_HPU_LAZY_MODE', '1') == '1'
74 - if is_lazy:
75 - torch._dynamo.config.disable = True
76 - # NOTE(kzawora) multi-HPU inference with HPUGraphs (lazy-only)
77 - # requires enabling lazy collectives
78 - # see https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html # noqa: E501
79 - os.environ['PT_HPU_ENABLE_LAZY_COLLECTIVES'] = 'true'
80 -
50 global plugins_loaded
51 if plugins_loaded:
52 return
vllm/worker/worker_base.py
+11 -4
@@ -4,6 +4,7 @@ import time
4 from abc import ABC, abstractmethod
5 from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
6
7 +import cloudpickle
8 import torch
9
10 from vllm.config import ObservabilityConfig, VllmConfig
@@ -521,14 +522,20 @@ class WorkerWrapperBase:
522 kwargs = all_kwargs[self.rpc_rank]
523 enable_trace_function_call_for_thread(self.vllm_config)
524
524 - # see https://github.com/NVIDIA/nccl/issues/1234
525 - os.environ['NCCL_CUMEM_ENABLE'] = '0'
525 + from vllm import configure_as_vllm_process
526 + configure_as_vllm_process()
527
528 from vllm.plugins import load_general_plugins
529 load_general_plugins()
530
530 - worker_class = resolve_obj_by_qualname(
531 - self.vllm_config.parallel_config.worker_cls)
531 + if isinstance(self.vllm_config.parallel_config.worker_cls, str):
532 + worker_class = resolve_obj_by_qualname(
533 + self.vllm_config.parallel_config.worker_cls)
534 + else:
535 + assert isinstance(self.vllm_config.parallel_config.worker_cls,
536 + bytes)
537 + worker_class = cloudpickle.loads(
538 + self.vllm_config.parallel_config.worker_cls)
539 self.worker = worker_class(**kwargs)
540 assert self.worker is not None
541