A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[perf]fix current stream (#11870)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 9, 2025 at 15:18 UTC
310aca88c984983189a57f1b72e3b1dde89fb92f
4 files changed
+46
-15
vllm/distributed/device_communicators/pynccl.py
+8
-7
@@ -10,6 +10,7 @@ from vllm.distributed.device_communicators.pynccl_wrapper import (
10
ncclRedOpTypeEnum, ncclUniqueId)
11
from vllm.distributed.utils import StatelessProcessGroup
12
from vllm.logger import init_logger
13
+from vllm.utils import current_stream
14
15
logger = init_logger(__name__)
16
@@ -96,7 +97,7 @@ class PyNcclCommunicator:
97
self.comm: ncclComm_t = self.nccl.ncclCommInitRank(
98
self.world_size, self.unique_id, self.rank)
99
99
- stream = torch.cuda.current_stream()
100
+ stream = current_stream()
101
# A small all_reduce for warmup.
102
data = torch.zeros(1, device=device)
103
self.all_reduce(data)
@@ -119,7 +120,7 @@ class PyNcclCommunicator:
120
out_tensor = torch.empty_like(in_tensor)
121
122
if stream is None:
122
- stream = torch.cuda.current_stream()
123
+ stream = current_stream()
124
self.nccl.ncclAllReduce(buffer_type(in_tensor.data_ptr()),
125
buffer_type(out_tensor.data_ptr()),
126
in_tensor.numel(),
@@ -141,7 +142,7 @@ class PyNcclCommunicator:
142
f"this nccl communicator is created to work on {self.device}, "
143
f"but the input tensor is on {input_tensor.device}")
144
if stream is None:
144
- stream = torch.cuda.current_stream()
145
+ stream = current_stream()
146
self.nccl.ncclAllGather(
147
buffer_type(input_tensor.data_ptr()),
148
buffer_type(output_tensor.data_ptr()), input_tensor.numel(),
@@ -162,7 +163,7 @@ class PyNcclCommunicator:
163
f"this nccl communicator is created to work on {self.device}, "
164
f"but the input tensor is on {input_tensor.device}")
165
if stream is None:
165
- stream = torch.cuda.current_stream()
166
+ stream = current_stream()
167
self.nccl.ncclReduceScatter(
168
buffer_type(input_tensor.data_ptr()),
169
buffer_type(output_tensor.data_ptr()), output_tensor.numel(),
@@ -177,7 +178,7 @@ class PyNcclCommunicator:
178
f"this nccl communicator is created to work on {self.device}, "
179
f"but the input tensor is on {tensor.device}")
180
if stream is None:
180
- stream = torch.cuda.current_stream()
181
+ stream = current_stream()
182
self.nccl.ncclSend(buffer_type(tensor.data_ptr()), tensor.numel(),
183
ncclDataTypeEnum.from_torch(tensor.dtype), dst,
184
self.comm, cudaStream_t(stream.cuda_stream))
@@ -189,7 +190,7 @@ class PyNcclCommunicator:
190
f"this nccl communicator is created to work on {self.device}, "
191
f"but the input tensor is on {tensor.device}")
192
if stream is None:
192
- stream = torch.cuda.current_stream()
193
+ stream = current_stream()
194
self.nccl.ncclRecv(buffer_type(tensor.data_ptr()), tensor.numel(),
195
ncclDataTypeEnum.from_torch(tensor.dtype), src,
196
self.comm, cudaStream_t(stream.cuda_stream))
@@ -201,7 +202,7 @@ class PyNcclCommunicator:
202
f"this nccl communicator is created to work on {self.device}, "
203
f"but the input tensor is on {tensor.device}")
204
if stream is None:
204
- stream = torch.cuda.current_stream()
205
+ stream = current_stream()
206
if src == self.rank:
207
sendbuff = buffer_type(tensor.data_ptr())
208
# NCCL requires the sender also to have a receive buffer
vllm/distributed/parallel_state.py
+1
-4
@@ -357,10 +357,7 @@ class GroupCoordinator:
357
return out
358
pynccl_comm = self.pynccl_comm
359
assert pynccl_comm is not None
360
- # TODO: pynccl should not use `stream=`
361
- # it can just always use the current stream.
362
- out = pynccl_comm.all_reduce(input_,
363
- stream=torch.cuda.current_stream())
360
+ out = pynccl_comm.all_reduce(input_)
361
if out is None:
362
# fall back to the default all-reduce using PyTorch.
363
# this usually happens during testing.
vllm/utils.py
+33
@@ -944,6 +944,39 @@ def find_nccl_library() -> str:
944
return so_file
945
946
947
+prev_set_stream = torch.cuda.set_stream
948
+
949
+_current_stream = None
950
+
951
+
952
+def _patched_set_stream(stream: torch.cuda.Stream) -> None:
953
+ global _current_stream
954
+ _current_stream = stream
955
+ prev_set_stream(stream)
956
+
957
+
958
+torch.cuda.set_stream = _patched_set_stream
959
+
960
+
961
+def current_stream() -> torch.cuda.Stream:
962
+ """
963
+ replace `torch.cuda.current_stream()` with `vllm.utils.current_stream()`.
964
+ it turns out that `torch.cuda.current_stream()` is quite expensive,
965
+ as it will construct a new stream object at each call.
966
+ here we patch `torch.cuda.set_stream` to keep track of the current stream
967
+ directly, so that we can avoid calling `torch.cuda.current_stream()`.
968
+
969
+ the underlying hypothesis is that we do not call `torch._C._cuda_setStream`
970
+ from C/C++ code.
971
+ """
972
+ global _current_stream
973
+ if _current_stream is None:
974
+ # when this function is called before any stream is set,
975
+ # we return the default stream.
976
+ _current_stream = torch.cuda.current_stream()
977
+ return _current_stream
978
+
979
+
980
def enable_trace_function_call_for_thread(vllm_config: "VllmConfig") -> None:
981
"""Set up function tracing for the current thread,
982
if enabled via the VLLM_TRACE_FUNCTION environment variable
vllm/worker/multi_step_model_runner.py
+4
-4
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
14
get_pythonized_sample_results)
15
from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
16
Logprob, SequenceGroupMetadata, SequenceOutput)
17
-from vllm.utils import PyObjectCache, async_tensor_h2d
17
+from vllm.utils import PyObjectCache, async_tensor_h2d, current_stream
18
from vllm.worker.model_runner import (GPUModelRunnerBase,
19
ModelInputForGPUWithSamplingMetadata)
20
from vllm.worker.model_runner_base import (
@@ -498,7 +498,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
498
# appended sampler output from last iteration
499
# - also maybe pythonize if CPU is ahead of GPU
500
501
- current_stream = torch.cuda.current_stream()
501
+ stream = current_stream()
502
if not model_input.is_first_multi_step:
503
# Explicitly block on the previous step's forward to make sure we
504
# don't clobber any GPU tensors still in use.
@@ -541,7 +541,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
541
num_steps=1)
542
543
# record the event for the current step so that the next step can sync
544
- model_input.record_step_event(current_stream)
544
+ model_input.record_step_event(stream)
545
546
if get_pp_group().is_last_rank and self.is_driver_worker:
547
assert isinstance(output, list)
@@ -552,7 +552,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
552
# event for the pythonization so that we only pythonize if the
553
# tensors are ready. May be able to be combined with the step event
554
output_ready_event = torch.cuda.Event()
555
- output_ready_event.record(current_stream)
555
+ output_ready_event.record(stream)
556
if self.parallel_config.pipeline_parallel_size > 1:
557
output[0].sampled_token_ids_cpu = output[
558
0].sampled_token_ids.cpu()