@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7da2487591
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[torch.compile] fix tensor alias (#8982)

youkaichao committed Sep 30, 2024 at 20:40 UTC 7da2487591888da043254f8c7045a48d5dbcc753
3 files changed +9 -3
vllm/worker/embedding_model_runner.py
+2 -1
@@ -103,7 +103,8 @@ class EmbeddingModelRunner(
103 # a placeholder (it has wide hardware support).
104 kv_caches = [
105 torch.tensor([], dtype=torch.float32, device=self.device)
106 - ] * num_layers
106 + for _ in range(num_layers)
107 + ]
108
109 execute_model_kwargs = {
110 "input_ids":
vllm/worker/enc_dec_model_runner.py
+2 -1
@@ -348,7 +348,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
348 # a placeholder (it has wide hardware support).
349 kv_caches = [
350 torch.tensor([], dtype=torch.float32, device=self.device)
351 - ] * num_layers
351 + for _ in range(num_layers)
352 + ]
353 finished_requests_ids = [seq.request_id for seq in seqs]
354 model_input = self.prepare_model_input(
355 seqs, finished_requests_ids=finished_requests_ids)
vllm/worker/model_runner.py
+5 -1
@@ -1244,9 +1244,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1244 # it by reference, rather by specializing on the value ``None``.
1245 # the `dtype` argument does not matter, and we use `float32` as
1246 # a placeholder (it has wide hardware support).
1247 + # it is important to create tensors inside the loop, rather than
1248 + # multiplying the list, to avoid Dynamo from treating them as
1249 + # tensor aliasing.
1250 kv_caches = [
1251 torch.tensor([], dtype=torch.float32, device=self.device)
1249 - ] * num_layers
1252 + for _ in range(num_layers)
1253 + ]
1254 finished_requests_ids = [seq.request_id for seq in seqs]
1255 model_input = self.prepare_model_input(
1256 seqs, finished_requests_ids=finished_requests_ids)