A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 12:15 UTC
[BugFix][MRV2] Fix cuda event reuse race (#39115)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Apr 6, 2026 at 17:21 UTC
62095e82c1ef7ed91be5223d36c6109958e9aec6
2 files changed
+2
-7
vllm/v1/worker/gpu/async_utils.py
+2
-4
@@ -17,7 +17,6 @@ class AsyncOutput(AsyncModelRunnerOutput):
17
num_sampled_tokens: torch.Tensor,
18
main_stream: torch.cuda.Stream,
19
copy_stream: torch.cuda.Stream,
20
- copy_event: torch.cuda.Event,
20
):
21
# NOTE(woosuk): We must retain references to the GPU tensors,
22
# as the copy operations are performed on a different CUDA stream than
@@ -25,7 +24,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
24
self.model_runner_output = model_runner_output
25
self.sampler_output = sampler_output
26
self.num_sampled_tokens = num_sampled_tokens
28
- self.copy_event = copy_event
27
+ self.copy_event = torch.cuda.Event()
28
29
with stream(copy_stream, main_stream):
30
copy_stream.wait_stream(main_stream)
@@ -78,12 +77,11 @@ class AsyncPoolingOutput(AsyncModelRunnerOutput):
77
is_valid: torch.Tensor | None,
78
main_stream: torch.cuda.Stream,
79
copy_stream: torch.cuda.Stream,
81
- copy_event: torch.cuda.Event,
80
):
81
self.model_runner_output = model_runner_output
82
self.pooler_output = pooler_output
83
self.is_valid = is_valid
86
- self.copy_event = copy_event
84
+ self.copy_event = torch.cuda.Event()
85
86
with stream(copy_stream, main_stream):
87
copy_stream.wait_stream(main_stream)
vllm/v1/worker/gpu/model_runner.py
-3
@@ -130,7 +130,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
130
131
self.use_async_scheduling = self.scheduler_config.async_scheduling
132
self.output_copy_stream = torch.cuda.Stream(self.device)
133
- self.output_copy_event = torch.cuda.Event()
133
134
# Pipeline parallelism.
135
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
@@ -1180,7 +1179,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1179
num_sampled_tokens=num_sampled,
1180
main_stream=self.main_stream,
1181
copy_stream=self.output_copy_stream,
1183
- copy_event=self.output_copy_event,
1182
)
1183
1184
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
@@ -1270,7 +1268,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1268
is_valid=is_valid,
1269
main_stream=self.main_stream,
1270
copy_stream=self.output_copy_stream,
1273
- copy_event=self.output_copy_event,
1271
)
1272
1273
self.postprocess_pool(input_batch)