@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 310aca88c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[perf]fix current stream (#11870)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 9, 2025 at 15:18 UTC 310aca88c984983189a57f1b72e3b1dde89fb92f
4 files changed +46 -15
vllm/distributed/device_communicators/pynccl.py
+8 -7
@@ -10,6 +10,7 @@ from vllm.distributed.device_communicators.pynccl_wrapper import (
10 ncclRedOpTypeEnum, ncclUniqueId)
11 from vllm.distributed.utils import StatelessProcessGroup
12 from vllm.logger import init_logger
13 +from vllm.utils import current_stream
14
15 logger = init_logger(__name__)
16
@@ -96,7 +97,7 @@ class PyNcclCommunicator:
97 self.comm: ncclComm_t = self.nccl.ncclCommInitRank(
98 self.world_size, self.unique_id, self.rank)
99
99 - stream = torch.cuda.current_stream()
100 + stream = current_stream()
101 # A small all_reduce for warmup.
102 data = torch.zeros(1, device=device)
103 self.all_reduce(data)
@@ -119,7 +120,7 @@ class PyNcclCommunicator:
120 out_tensor = torch.empty_like(in_tensor)
121
122 if stream is None:
122 - stream = torch.cuda.current_stream()
123 + stream = current_stream()
124 self.nccl.ncclAllReduce(buffer_type(in_tensor.data_ptr()),
125 buffer_type(out_tensor.data_ptr()),
126 in_tensor.numel(),
@@ -141,7 +142,7 @@ class PyNcclCommunicator:
142 f"this nccl communicator is created to work on {self.device}, "
143 f"but the input tensor is on {input_tensor.device}")
144 if stream is None:
144 - stream = torch.cuda.current_stream()
145 + stream = current_stream()
146 self.nccl.ncclAllGather(
147 buffer_type(input_tensor.data_ptr()),
148 buffer_type(output_tensor.data_ptr()), input_tensor.numel(),
@@ -162,7 +163,7 @@ class PyNcclCommunicator:
163 f"this nccl communicator is created to work on {self.device}, "
164 f"but the input tensor is on {input_tensor.device}")
165 if stream is None:
165 - stream = torch.cuda.current_stream()
166 + stream = current_stream()
167 self.nccl.ncclReduceScatter(
168 buffer_type(input_tensor.data_ptr()),
169 buffer_type(output_tensor.data_ptr()), output_tensor.numel(),
@@ -177,7 +178,7 @@ class PyNcclCommunicator:
178 f"this nccl communicator is created to work on {self.device}, "
179 f"but the input tensor is on {tensor.device}")
180 if stream is None:
180 - stream = torch.cuda.current_stream()
181 + stream = current_stream()
182 self.nccl.ncclSend(buffer_type(tensor.data_ptr()), tensor.numel(),
183 ncclDataTypeEnum.from_torch(tensor.dtype), dst,
184 self.comm, cudaStream_t(stream.cuda_stream))
@@ -189,7 +190,7 @@ class PyNcclCommunicator:
190 f"this nccl communicator is created to work on {self.device}, "
191 f"but the input tensor is on {tensor.device}")
192 if stream is None:
192 - stream = torch.cuda.current_stream()
193 + stream = current_stream()
194 self.nccl.ncclRecv(buffer_type(tensor.data_ptr()), tensor.numel(),
195 ncclDataTypeEnum.from_torch(tensor.dtype), src,
196 self.comm, cudaStream_t(stream.cuda_stream))
@@ -201,7 +202,7 @@ class PyNcclCommunicator:
202 f"this nccl communicator is created to work on {self.device}, "
203 f"but the input tensor is on {tensor.device}")
204 if stream is None:
204 - stream = torch.cuda.current_stream()
205 + stream = current_stream()
206 if src == self.rank:
207 sendbuff = buffer_type(tensor.data_ptr())
208 # NCCL requires the sender also to have a receive buffer
vllm/distributed/parallel_state.py
+1 -4
@@ -357,10 +357,7 @@ class GroupCoordinator:
357 return out
358 pynccl_comm = self.pynccl_comm
359 assert pynccl_comm is not None
360 - # TODO: pynccl should not use `stream=`
361 - # it can just always use the current stream.
362 - out = pynccl_comm.all_reduce(input_,
363 - stream=torch.cuda.current_stream())
360 + out = pynccl_comm.all_reduce(input_)
361 if out is None:
362 # fall back to the default all-reduce using PyTorch.
363 # this usually happens during testing.
vllm/utils.py
+33
@@ -944,6 +944,39 @@ def find_nccl_library() -> str:
944 return so_file
945
946
947 +prev_set_stream = torch.cuda.set_stream
948 +
949 +_current_stream = None
950 +
951 +
952 +def _patched_set_stream(stream: torch.cuda.Stream) -> None:
953 + global _current_stream
954 + _current_stream = stream
955 + prev_set_stream(stream)
956 +
957 +
958 +torch.cuda.set_stream = _patched_set_stream
959 +
960 +
961 +def current_stream() -> torch.cuda.Stream:
962 + """
963 + replace `torch.cuda.current_stream()` with `vllm.utils.current_stream()`.
964 + it turns out that `torch.cuda.current_stream()` is quite expensive,
965 + as it will construct a new stream object at each call.
966 + here we patch `torch.cuda.set_stream` to keep track of the current stream
967 + directly, so that we can avoid calling `torch.cuda.current_stream()`.
968 +
969 + the underlying hypothesis is that we do not call `torch._C._cuda_setStream`
970 + from C/C++ code.
971 + """
972 + global _current_stream
973 + if _current_stream is None:
974 + # when this function is called before any stream is set,
975 + # we return the default stream.
976 + _current_stream = torch.cuda.current_stream()
977 + return _current_stream
978 +
979 +
980 def enable_trace_function_call_for_thread(vllm_config: "VllmConfig") -> None:
981 """Set up function tracing for the current thread,
982 if enabled via the VLLM_TRACE_FUNCTION environment variable
vllm/worker/multi_step_model_runner.py
+4 -4
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
14 get_pythonized_sample_results)
15 from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
16 Logprob, SequenceGroupMetadata, SequenceOutput)
17 -from vllm.utils import PyObjectCache, async_tensor_h2d
17 +from vllm.utils import PyObjectCache, async_tensor_h2d, current_stream
18 from vllm.worker.model_runner import (GPUModelRunnerBase,
19 ModelInputForGPUWithSamplingMetadata)
20 from vllm.worker.model_runner_base import (
@@ -498,7 +498,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
498 # appended sampler output from last iteration
499 # - also maybe pythonize if CPU is ahead of GPU
500
501 - current_stream = torch.cuda.current_stream()
501 + stream = current_stream()
502 if not model_input.is_first_multi_step:
503 # Explicitly block on the previous step's forward to make sure we
504 # don't clobber any GPU tensors still in use.
@@ -541,7 +541,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
541 num_steps=1)
542
543 # record the event for the current step so that the next step can sync
544 - model_input.record_step_event(current_stream)
544 + model_input.record_step_event(stream)
545
546 if get_pp_group().is_last_rank and self.is_driver_worker:
547 assert isinstance(output, list)
@@ -552,7 +552,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
552 # event for the pythonization so that we only pythonize if the
553 # tensors are ready. May be able to be combined with the step event
554 output_ready_event = torch.cuda.Event()
555 - output_ready_event.record(current_stream)
555 + output_ready_event.record(stream)
556 if self.parallel_config.pipeline_parallel_size > 1:
557 output[0].sampled_token_ids_cpu = output[
558 0].sampled_token_ids.cpu()