@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 99aa4eddaf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[torch.compile] register allreduce operations as custom ops (#8526)

youkaichao committed Sep 16, 2024 at 22:57 UTC 99aa4eddaf929f57dac405b00db3f5286624ee8b
9 files changed +137 -50
.buildkite/test-pipeline.yaml
+3 -7
@@ -163,13 +163,6 @@ steps:
163 - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
164 - python3 offline_inference_encoder_decoder.py
165
166 -- label: torch compile integration test
167 - source_file_dependencies:
168 - - vllm/
169 - commands:
170 - - pytest -v -s ./compile/test_full_graph.py
171 - - pytest -v -s ./compile/test_wrapper.py
172 -
166 - label: Prefix Caching Test # 7min
167 #mirror_hardwares: [amd]
168 source_file_dependencies:
@@ -348,7 +341,10 @@ steps:
341 - vllm/executor/
342 - vllm/model_executor/models/
343 - tests/distributed/
344 + - vllm/compilation
345 commands:
346 + - pytest -v -s ./compile/test_full_graph.py
347 + - pytest -v -s ./compile/test_wrapper.py
348 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
349 - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
350 # Avoid importing model tests that cause CUDA reinitialization error
csrc/custom_all_reduce.cu
-12
@@ -55,18 +55,6 @@ bool _is_weak_contiguous(torch::Tensor& t) {
55 t.numel() * t.element_size());
56 }
57
58 -bool should_custom_ar(torch::Tensor& inp, int64_t max_size, int64_t world_size,
59 - bool full_nvlink) {
60 - auto inp_size = inp.numel() * inp.element_size();
61 - // custom allreduce requires input byte size to be multiples of 16
62 - if (inp_size % 16 != 0) return false;
63 - if (!_is_weak_contiguous(inp)) return false;
64 - if (world_size == 2 || full_nvlink) return inp_size <= max_size;
65 - // for 4 or more non NVLink-capable GPUs, custom allreduce provides little
66 - // performance improvement over NCCL.
67 - return false;
68 -}
69 -
58 void _all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
59 cudaStream_t stream) {
60 auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
csrc/ops.h
-2
@@ -241,8 +241,6 @@ fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
241 const std::vector<std::string>& handles,
242 const std::vector<int64_t>& offsets, int64_t rank,
243 bool full_nvlink);
244 -bool should_custom_ar(torch::Tensor& inp, int64_t max_size, int64_t world_size,
245 - bool full_nvlink);
244 void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out);
245 void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
246 torch::Tensor& out);
csrc/torch_bindings.cpp
-5
@@ -411,11 +411,6 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _custom_ar), custom_ar) {
411 "bool full_nvlink) -> int");
412 custom_ar.impl("init_custom_ar", torch::kCUDA, &init_custom_ar);
413
414 - custom_ar.def(
415 - "should_custom_ar(Tensor inp, int max_size, int world_size, "
416 - "bool full_nvlink) -> bool");
417 - custom_ar.impl("should_custom_ar", torch::kCUDA, &should_custom_ar);
418 -
414 custom_ar.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()");
415 custom_ar.impl("all_reduce_reg", torch::kCUDA, &all_reduce_reg);
416
tests/compile/__init__.py
tests/compile/test_full_graph.py
+13 -2
@@ -2,9 +2,20 @@ import os
2
3 import pytest
4
5 +from vllm.utils import cuda_device_count_stateless
6 +
7 +from ..utils import fork_new_process_for_each_test
8 +
9
10 @pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
7 -def test_full_graph(model):
11 +@pytest.mark.parametrize("tp_size", [1, 2])
12 +@fork_new_process_for_each_test
13 +def test_full_graph(model, tp_size):
14 +
15 + # Skip the test if there are not enough CUDA devices.
16 + if cuda_device_count_stateless() < tp_size:
17 + pytest.skip("Not enough CUDA devices for the test.")
18 +
19 # make sure these models can be captured in full graph mode
20 if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
21 os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
@@ -17,7 +28,7 @@ def test_full_graph(model):
28 "The future of AI is",
29 ]
30 sampling_params = SamplingParams(temperature=0)
20 - llm = LLM(model=model, enforce_eager=True)
31 + llm = LLM(model=model, enforce_eager=True, tensor_parallel_size=tp_size)
32
33 outputs = llm.generate(prompts, sampling_params)
34
vllm/_custom_ops.py
-6
@@ -870,12 +870,6 @@ def init_custom_ar(meta: torch.Tensor, rank_data: torch.Tensor,
870 offsets, rank, full_nvlink)
871
872
873 -def should_custom_ar(inp: torch.Tensor, max_size: int, world_size: int,
874 - full_nvlink: bool) -> bool:
875 - return torch.ops._C_custom_ar.should_custom_ar(inp, max_size, world_size,
876 - full_nvlink)
877 -
878 -
873 def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None:
874 torch.ops._C_custom_ar.all_reduce_reg(fa, inp, out)
875
vllm/distributed/device_communicators/custom_all_reduce.py
+19 -2
@@ -33,6 +33,12 @@ def _can_p2p(rank: int, world_size: int) -> bool:
33 return True
34
35
36 +def is_weak_contiguous(inp: torch.Tensor):
37 + return inp.is_contiguous() or (inp.storage().nbytes() -
38 + inp.storage_offset() * inp.element_size()
39 + == inp.numel() * inp.element_size())
40 +
41 +
42 class CustomAllreduce:
43
44 _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
@@ -224,8 +230,19 @@ class CustomAllreduce:
230 ops.register_graph_buffers(self._ptr, handles, offsets)
231
232 def should_custom_ar(self, inp: torch.Tensor):
227 - return ops.should_custom_ar(inp, self.max_size, self.world_size,
228 - self.full_nvlink)
233 + if self.disabled:
234 + return False
235 + inp_size = inp.numel() * inp.element_size()
236 + # custom allreduce requires input byte size to be multiples of 16
237 + if inp_size % 16 != 0:
238 + return False
239 + if not is_weak_contiguous(inp):
240 + return False
241 + # for 4 or more non NVLink-capable GPUs, custom allreduce provides
242 + # little performance improvement over NCCL.
243 + if self.world_size == 2 or self.full_nvlink:
244 + return inp_size < self.max_size
245 + return False
246
247 # all reduce, assuming inp tensor is IPC registered with register_buffer,
248 # or, in the context of cuda graphs, register_graph_buffers
vllm/distributed/parallel_state.py
+102 -14
@@ -21,11 +21,12 @@ If you only need to use the distributed environment without model/pipeline
21 """
22 import contextlib
23 import pickle
24 +import weakref
25 from collections import namedtuple
26 from contextlib import contextmanager, nullcontext
27 from dataclasses import dataclass
28 from multiprocessing import shared_memory
28 -from typing import Any, Dict, List, Optional, Tuple, Union
29 +from typing import Any, Callable, Dict, List, Optional, Tuple, Union
30 from unittest.mock import patch
31
32 import torch
@@ -69,6 +70,58 @@ def _split_tensor_dict(
70 return metadata_list, tensor_list
71
72
73 +_group_name_counter: Dict[str, int] = {}
74 +
75 +
76 +def _get_unique_name(name: str) -> str:
77 + """Get a unique name for the group.
78 + Example:
79 + _get_unique_name("tp") -> "tp:0"
80 + _get_unique_name("tp") -> "tp:1"
81 + """
82 + if name not in _group_name_counter:
83 + _group_name_counter[name] = 0
84 + newname = f"{name}:{_group_name_counter[name]}"
85 + _group_name_counter[name] += 1
86 + return newname
87 +
88 +
89 +_groups: Dict[str, Callable[[], "GroupCoordinator"]] = {}
90 +
91 +
92 +def _register_group(group: "GroupCoordinator") -> None:
93 + # looks like Python 3.8 does not understand `ReferenceType`
94 + _groups[group.unique_name] = weakref.ref(group) # type: ignore
95 +
96 +
97 +@torch.library.custom_op("vllm::inplace_all_reduce", mutates_args=["tensor"])
98 +def inplace_all_reduce(tensor: torch.Tensor, group_name: str) -> None:
99 + assert group_name in _groups, f"Group {group_name} is not found."
100 + group = _groups[group_name]()
101 + if group is None:
102 + raise ValueError(f"Group {group_name} is destroyed.")
103 + group._all_reduce(tensor)
104 +
105 +
106 +@inplace_all_reduce.register_fake
107 +def _(tensor: torch.Tensor, group_name: str) -> None:
108 + return
109 +
110 +
111 +@torch.library.custom_op("vllm::outplace_all_reduce", mutates_args=[])
112 +def outplace_all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
113 + assert group_name in _groups, f"Group {group_name} is not found."
114 + group = _groups[group_name]()
115 + if group is None:
116 + raise ValueError(f"Group {group_name} is destroyed.")
117 + return group._all_reduce(tensor)
118 +
119 +
120 +@outplace_all_reduce.register_fake
121 +def _(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
122 + return torch.empty_like(tensor)
123 +
124 +
125 class GroupCoordinator:
126 """
127 PyTorch ProcessGroup wrapper for a group of processes.
@@ -111,7 +164,11 @@ class GroupCoordinator:
164 use_custom_allreduce: bool,
165 use_tpu_communicator: bool,
166 use_message_queue_broadcaster: bool = False,
167 + group_name: Optional[str] = None,
168 ):
169 + group_name = group_name or "anonymous"
170 + self.unique_name = _get_unique_name(group_name)
171 + _register_group(self)
172
173 self.rank = torch.distributed.get_rank()
174 self.local_rank = local_rank
@@ -149,28 +206,24 @@ class GroupCoordinator:
206 from vllm.distributed.device_communicators.pynccl import (
207 PyNcclCommunicator)
208
152 - self.pynccl_comm: Optional[PyNcclCommunicator]
209 + self.pynccl_comm: Optional[PyNcclCommunicator] = None
210 if use_pynccl and self.world_size > 1:
211 self.pynccl_comm = PyNcclCommunicator(
212 group=self.cpu_group,
213 device=self.device,
214 )
158 - else:
159 - self.pynccl_comm = None
215
161 - self.ca_comm: Optional[CustomAllreduce]
216 + self.ca_comm: Optional[CustomAllreduce] = None
217 if use_custom_allreduce and self.world_size > 1:
218 # Initialize a custom fast all-reduce implementation.
219 self.ca_comm = CustomAllreduce(
220 group=self.cpu_group,
221 device=self.device,
222 )
168 - else:
169 - self.ca_comm = None
223
224 from vllm.distributed.device_communicators.tpu_communicator import (
225 TpuCommunicator)
173 - self.tpu_communicator: Optional[TpuCommunicator]
226 + self.tpu_communicator: Optional[TpuCommunicator] = None
227 if use_tpu_communicator and self.world_size > 1:
228 self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
229
@@ -264,16 +317,46 @@ class GroupCoordinator:
317
318 def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
319 """
320 + User-facing all-reduce function before we actually call the
321 + all-reduce operation.
322 +
323 + We need this because Dynamo does not support passing an arbitrary
324 + object (`self` in this case) to a custom op. We need to pass the
325 + group name as a string, and then look up the group coordinator from
326 + the group name, dispatch the all-reduce operation to the group
327 + coordinator.
328 +
329 + In addition, PyTorch custom ops do not support mutation or returning
330 + a new tensor in the same op. So we need to figure out if the op is
331 + in-place or out-of-place ahead of time.
332 + """
333 + # Bypass the function if we are using only 1 GPU.
334 + if self.world_size == 1:
335 + return input_
336 +
337 + if self.tpu_communicator is not None and \
338 + not self.tpu_communicator.disabled:
339 + # TPU handles Dynamo with its own logic.
340 + return self._all_reduce(input_)
341 +
342 + if self.ca_comm is not None and self.ca_comm.should_custom_ar(input_):
343 + return torch.ops.vllm.outplace_all_reduce(
344 + input_, group_name=self.unique_name)
345 + else:
346 + torch.ops.vllm.inplace_all_reduce(input_,
347 + group_name=self.unique_name)
348 + return input_
349 +
350 + def _all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
351 + """
352 + The actual all-reduce implementation.
353 +
354 NOTE: This operation will be applied in-place or out-of-place.
355 Always assume this function modifies its input, but use the return
356 value as the output.
357 """
358 ca_comm = self.ca_comm
359
273 - # Bypass the function if we are using only 1 GPU.
274 - if self.world_size == 1:
275 - return input_
276 -
360 # For TPUs, use TPU communicator.
361 tpu_comm = self.tpu_communicator
362 if tpu_comm is not None and not tpu_comm.disabled:
@@ -758,6 +841,7 @@ def init_world_group(ranks: List[int], local_rank: int,
841 use_pynccl=False,
842 use_custom_allreduce=False,
843 use_tpu_communicator=False,
844 + group_name="world",
845 )
846
847
@@ -767,6 +851,7 @@ def init_model_parallel_group(
851 backend: str,
852 use_custom_allreduce: Optional[bool] = None,
853 use_message_queue_broadcaster: bool = False,
854 + group_name: Optional[str] = None,
855 ) -> GroupCoordinator:
856 if use_custom_allreduce is None:
857 use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
@@ -778,6 +863,7 @@ def init_model_parallel_group(
863 use_custom_allreduce=use_custom_allreduce,
864 use_tpu_communicator=True,
865 use_message_queue_broadcaster=use_message_queue_broadcaster,
866 + group_name=group_name,
867 )
868
869
@@ -931,7 +1017,8 @@ def initialize_model_parallel(
1017 _TP = init_model_parallel_group(group_ranks,
1018 get_world_group().local_rank,
1019 backend,
934 - use_message_queue_broadcaster=True)
1020 + use_message_queue_broadcaster=True,
1021 + group_name="tp")
1022
1023 # Build the pipeline model-parallel groups.
1024 num_pipeline_model_parallel_groups: int = (world_size //
@@ -947,7 +1034,8 @@ def initialize_model_parallel(
1034 _PP = init_model_parallel_group(group_ranks,
1035 get_world_group().local_rank,
1036 backend,
950 - use_custom_allreduce=False)
1037 + use_custom_allreduce=False,
1038 + group_name="pp")
1039
1040
1041 def ensure_model_parallel_initialized(