A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[torch.compile] fix tensor alias (#8982)
youkaichao committed
Sep 30, 2024 at 20:40 UTC
7da2487591888da043254f8c7045a48d5dbcc753
3 files changed
+9
-3
vllm/worker/embedding_model_runner.py
+2
-1
@@ -103,7 +103,8 @@ class EmbeddingModelRunner(
103
# a placeholder (it has wide hardware support).
104
kv_caches = [
105
torch.tensor([], dtype=torch.float32, device=self.device)
106
- ] * num_layers
106
+ for _ in range(num_layers)
107
+ ]
108
109
execute_model_kwargs = {
110
"input_ids":
vllm/worker/enc_dec_model_runner.py
+2
-1
@@ -348,7 +348,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
348
# a placeholder (it has wide hardware support).
349
kv_caches = [
350
torch.tensor([], dtype=torch.float32, device=self.device)
351
- ] * num_layers
351
+ for _ in range(num_layers)
352
+ ]
353
finished_requests_ids = [seq.request_id for seq in seqs]
354
model_input = self.prepare_model_input(
355
seqs, finished_requests_ids=finished_requests_ids)
vllm/worker/model_runner.py
+5
-1
@@ -1244,9 +1244,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1244
# it by reference, rather by specializing on the value ``None``.
1245
# the `dtype` argument does not matter, and we use `float32` as
1246
# a placeholder (it has wide hardware support).
1247
+ # it is important to create tensors inside the loop, rather than
1248
+ # multiplying the list, to avoid Dynamo from treating them as
1249
+ # tensor aliasing.
1250
kv_caches = [
1251
torch.tensor([], dtype=torch.float32, device=self.device)
1249
- ] * num_layers
1252
+ for _ in range(num_layers)
1253
+ ]
1254
finished_requests_ids = [seq.request_id for seq in seqs]
1255
model_input = self.prepare_model_input(
1256
seqs, finished_requests_ids=finished_requests_ids)