A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Replace DtypeTensor (#1123)
Woosuk Kwon committed
Sep 21, 2023 at 00:51 UTC
2ac4d5e2bf033306fdb9b5002b2adbaafb864a3a
1 file changed
+15
-5
vllm/worker/worker.py
+15
-5
@@ -228,15 +228,25 @@ class Worker:
228
input_positions = _pad_to_alignment(input_positions, multiple_of=8)
229
230
# Convert to tensors.
231
- tokens_tensor = torch.cuda.LongTensor(input_tokens)
232
- positions_tensor = torch.cuda.LongTensor(input_positions)
233
- slot_mapping_tensor = torch.cuda.IntTensor(slot_mapping)
234
- context_lens_tensor = torch.cuda.IntTensor(context_lens)
231
+ tokens_tensor = torch.tensor(input_tokens,
232
+ dtype=torch.long,
233
+ device="cuda")
234
+ positions_tensor = torch.tensor(input_positions,
235
+ dtype=torch.long,
236
+ device="cuda")
237
+ slot_mapping_tensor = torch.tensor(slot_mapping,
238
+ dtype=torch.int,
239
+ device="cuda")
240
+ context_lens_tensor = torch.tensor(context_lens,
241
+ dtype=torch.int,
242
+ device="cuda")
243
padded_block_tables = [
244
_pad_to_max(block_table, max_num_blocks_per_seq)
245
for block_table in generation_block_tables
246
]
239
- block_tables_tensor = torch.cuda.IntTensor(padded_block_tables)
247
+ block_tables_tensor = torch.tensor(padded_block_tables,
248
+ dtype=torch.int,
249
+ device="cuda")
250
251
seq_data: Dict[int, SequenceData] = {}
252
for seq_group_metadata in seq_group_metadata_list: