@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ac4d5e2bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Replace DtypeTensor (#1123)

Woosuk Kwon committed Sep 21, 2023 at 00:51 UTC 2ac4d5e2bf033306fdb9b5002b2adbaafb864a3a
1 file changed +15 -5
vllm/worker/worker.py
+15 -5
@@ -228,15 +228,25 @@ class Worker:
228 input_positions = _pad_to_alignment(input_positions, multiple_of=8)
229
230 # Convert to tensors.
231 - tokens_tensor = torch.cuda.LongTensor(input_tokens)
232 - positions_tensor = torch.cuda.LongTensor(input_positions)
233 - slot_mapping_tensor = torch.cuda.IntTensor(slot_mapping)
234 - context_lens_tensor = torch.cuda.IntTensor(context_lens)
231 + tokens_tensor = torch.tensor(input_tokens,
232 + dtype=torch.long,
233 + device="cuda")
234 + positions_tensor = torch.tensor(input_positions,
235 + dtype=torch.long,
236 + device="cuda")
237 + slot_mapping_tensor = torch.tensor(slot_mapping,
238 + dtype=torch.int,
239 + device="cuda")
240 + context_lens_tensor = torch.tensor(context_lens,
241 + dtype=torch.int,
242 + device="cuda")
243 padded_block_tables = [
244 _pad_to_max(block_table, max_num_blocks_per_seq)
245 for block_table in generation_block_tables
246 ]
239 - block_tables_tensor = torch.cuda.IntTensor(padded_block_tables)
247 + block_tables_tensor = torch.tensor(padded_block_tables,
248 + dtype=torch.int,
249 + device="cuda")
250
251 seq_data: Dict[int, SequenceData] = {}
252 for seq_group_metadata in seq_group_metadata_list: