A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[BugFix] lazy init _copy_stream to avoid torch init wrong gpu instance (#8403)
WANGWEI committed
Sep 13, 2024 at 01:47 UTC
8a23e933026bdb66b0b141c69454457428aa056d
1 file changed
+5
-2
vllm/worker/multi_step_model_runner.py
+5
-2
@@ -230,12 +230,15 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
230
self._base_model_runner: GPUModelRunnerBase = base_model_runner
231
232
self.is_multi_step = self.scheduler_config.is_multi_step
233
- # used to copy tensors from GPU to CPU asynchronously
234
- self._copy_stream = torch.cuda.Stream()
233
self.pinned_sampled_token_ids: Optional[torch.Tensor] = None
234
235
self.pythonization_cache = PythonizationCache()
236
237
+ @functools.cached_property
238
+ def _copy_stream(self):
239
+ # used to copy tensors from GPU to CPU asynchronously
240
+ return torch.cuda.Stream()
241
+
242
def make_model_input_from_broadcasted_tensor_dict(
243
self, tensor_dict: Dict[str, Any]) -> StatefulModelInput:
244
model_input = (StatefulModelInput.from_broadcasted_tensor_dict(