@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a23e93302
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[BugFix] lazy init _copy_stream to avoid torch init wrong gpu instance (#8403)

WANGWEI committed Sep 13, 2024 at 01:47 UTC 8a23e933026bdb66b0b141c69454457428aa056d
1 file changed +5 -2
vllm/worker/multi_step_model_runner.py
+5 -2
@@ -230,12 +230,15 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
230 self._base_model_runner: GPUModelRunnerBase = base_model_runner
231
232 self.is_multi_step = self.scheduler_config.is_multi_step
233 - # used to copy tensors from GPU to CPU asynchronously
234 - self._copy_stream = torch.cuda.Stream()
233 self.pinned_sampled_token_ids: Optional[torch.Tensor] = None
234
235 self.pythonization_cache = PythonizationCache()
236
237 + @functools.cached_property
238 + def _copy_stream(self):
239 + # used to copy tensors from GPU to CPU asynchronously
240 + return torch.cuda.Stream()
241 +
242 def make_model_input_from_broadcasted_tensor_dict(
243 self, tensor_dict: Dict[str, Any]) -> StatefulModelInput:
244 model_input = (StatefulModelInput.from_broadcasted_tensor_dict(