@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef527be06c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[MISC] Use non-blocking transfer in prepare_input (#7172)

Cody Yu committed Aug 5, 2024 at 16:41 UTC ef527be06c4064f3a2753a3b2c7ede862fe459e8
4 files changed +43 -49
vllm/attention/backends/flash_attn.py
+12 -15
@@ -13,7 +13,7 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
14 compute_slot_mapping_start_idx,
15 is_block_tables_empty)
16 -from vllm.utils import make_tensor_with_pad
16 +from vllm.utils import async_tensor_h2d, make_tensor_with_pad
17
18 if TYPE_CHECKING:
19 from vllm.worker.model_runner import ModelInputForGPUBuilder
@@ -310,7 +310,8 @@ class FlashAttentionMetadataBuilder(
310 for i, block_table in enumerate(self.block_tables):
311 if block_table:
312 input_block_tables[i, :len(block_table)] = block_table
313 - block_tables = torch.tensor(input_block_tables, device=device)
313 + block_tables = torch.from_numpy(input_block_tables).to(
314 + device=device, non_blocking=True)
315 else:
316 block_tables = make_tensor_with_pad(
317 self.block_tables,
@@ -320,15 +321,15 @@ class FlashAttentionMetadataBuilder(
321 )
322 assert max_query_len > 0, ("query_lens: {}".format(query_lens))
323
323 - context_lens_tensor = torch.tensor(self.context_lens,
324 - dtype=torch.int,
325 - device=device)
326 - seq_lens_tensor = torch.tensor(seq_lens,
327 - dtype=torch.int,
328 - device=device)
329 - query_lens_tensor = torch.tensor(query_lens,
330 - dtype=torch.long,
331 - device=device)
324 + assert device is not None
325 + context_lens_tensor = async_tensor_h2d(self.context_lens, torch.int,
326 + device, self.runner.pin_memory)
327 + seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
328 + self.runner.pin_memory)
329 + query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
330 + self.runner.pin_memory)
331 + slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
332 + device, self.runner.pin_memory)
333 query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
334 dtype=torch.int32,
335 device=device)
@@ -344,10 +345,6 @@ class FlashAttentionMetadataBuilder(
345 dtype=query_start_loc.dtype,
346 out=query_start_loc[1:])
347
347 - slot_mapping_tensor = torch.tensor(self.slot_mapping,
348 - dtype=torch.long,
349 - device=device)
350 -
348 return FlashAttentionMetadata(
349 num_prefills=self.num_prefills,
350 slot_mapping=slot_mapping_tensor,
vllm/attention/backends/flashinfer.py
+11 -12
@@ -21,7 +21,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
21 compute_slot_mapping_start_idx,
22 is_block_tables_empty)
23 from vllm.attention.ops.paged_attn import PagedAttention
24 -from vllm.utils import get_kv_cache_torch_dtype, make_tensor_with_pad
24 +from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
25 + make_tensor_with_pad)
26
27 if TYPE_CHECKING:
28 from vllm.worker.model_runner import ModelInputForGPUBuilder
@@ -356,7 +357,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
357 for i, block_table in enumerate(self.block_tables):
358 if block_table:
359 input_block_tables[i, :len(block_table)] = block_table
359 - block_tables = torch.tensor(input_block_tables, device=device)
360 + block_tables = torch.from_numpy(input_block_tables).to(
361 + device, non_blocking=True)
362
363 last_paged_kv_indptr = self.paged_kv_indptr[-1]
364 self.paged_kv_indptr.extend([last_paged_kv_indptr] *
@@ -371,12 +373,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
373 )
374 assert max_query_len > 0, ("query_lens: {}".format(query_lens))
375
374 - seq_lens_tensor = torch.tensor(seq_lens,
375 - dtype=torch.int,
376 - device=device)
377 - query_lens_tensor = torch.tensor(query_lens,
378 - dtype=torch.long,
379 - device=device)
376 + assert device is not None
377 + seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
378 + self.runner.pin_memory)
379 + query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
380 + self.runner.pin_memory)
381 + slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
382 + device, self.runner.pin_memory)
383 query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
384 dtype=torch.int32,
385 device=device)
@@ -392,10 +395,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
395 dtype=query_start_loc.dtype,
396 out=query_start_loc[1:])
397
395 - slot_mapping_tensor = torch.tensor(self.slot_mapping,
396 - dtype=torch.long,
397 - device=device)
398 -
398 if len(self.paged_kv_indptr) > 0:
399 paged_kv_indices_tensor = torch.tensor(self.paged_kv_indices,
400 device="cpu",
vllm/attention/backends/utils.py
+12 -15
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Dict, List, Type, TypeVar, Union
4 import torch
5
6 from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
7 -from vllm.utils import make_tensor_with_pad
7 +from vllm.utils import async_tensor_h2d, make_tensor_with_pad
8
9 # Error string(s) for encoder/decoder
10 # unsupported attention scenarios
@@ -181,7 +181,8 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
181 for i, block_table in enumerate(self.block_tables):
182 if block_table:
183 input_block_tables[i, :len(block_table)] = block_table
184 - block_tables = torch.tensor(input_block_tables, device=device)
184 + block_tables = torch.from_numpy(input_block_tables).to(
185 + device, non_blocking=True)
186 else:
187 block_tables = make_tensor_with_pad(
188 self.block_tables,
@@ -191,15 +192,15 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
192 )
193 assert max_query_len > 0, "query_lens: {}".format(query_lens)
194
194 - context_lens_tensor = torch.tensor(self.context_lens,
195 - dtype=torch.int,
196 - device=device)
197 - seq_lens_tensor = torch.tensor(seq_lens,
198 - dtype=torch.int,
199 - device=device)
200 - query_lens_tensor = torch.tensor(query_lens,
201 - dtype=torch.long,
202 - device=device)
195 + assert device is not None
196 + context_lens_tensor = async_tensor_h2d(self.context_lens, torch.int,
197 + device, self.runner.pin_memory)
198 + seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
199 + self.runner.pin_memory)
200 + query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
201 + self.runner.pin_memory)
202 + slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
203 + device, self.runner.pin_memory)
204 query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
205 dtype=torch.int32,
206 device=device)
@@ -215,10 +216,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
216 dtype=query_start_loc.dtype,
217 out=query_start_loc[1:])
218
218 - slot_mapping_tensor = torch.tensor(self.slot_mapping,
219 - dtype=torch.long,
220 - device=device)
221 -
219 return self._metadata_cls( # type: ignore
220 num_prefills=self.num_prefills,
221 slot_mapping=slot_mapping_tensor,
vllm/worker/model_runner.py
+8 -7
@@ -50,7 +50,7 @@ from vllm.prompt_adapter.worker_manager import (
50 from vllm.sampling_params import SamplingParams
51 from vllm.sequence import (IntermediateTensors, SamplerOutput,
52 SequenceGroupMetadata)
53 -from vllm.utils import (CudaMemoryProfiler, flatten_2d_lists,
53 +from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d, flatten_2d_lists,
54 get_kv_cache_torch_dtype, is_hip,
55 is_pin_memory_available)
56 from vllm.worker.model_runner_base import (
@@ -549,12 +549,13 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
549 # Tokens and positions.
550 input_tokens.extend([0] * cuda_graph_pad_size)
551 input_positions.extend([0] * cuda_graph_pad_size)
552 - input_tokens_tensor = torch.tensor(input_tokens,
553 - dtype=torch.long,
554 - device=self.runner.device)
555 - input_positions_tensor = torch.tensor(input_positions,
556 - dtype=torch.long,
557 - device=self.runner.device)
552 + assert self.runner.device is not None
553 + input_tokens_tensor = async_tensor_h2d(input_tokens, torch.long,
554 + self.runner.device,
555 + self.runner.pin_memory)
556 + input_positions_tensor = async_tensor_h2d(input_positions, torch.long,
557 + self.runner.device,
558 + self.runner.pin_memory)
559
560 # Sequence and query lengths.
561 seq_lens.extend([1] * cuda_graph_pad_size)