A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[MISC] Use non-blocking transfer in prepare_input (#7172)
Cody Yu committed
Aug 5, 2024 at 16:41 UTC
ef527be06c4064f3a2753a3b2c7ede862fe459e8
4 files changed
+43
-49
vllm/attention/backends/flash_attn.py
+12
-15
@@ -13,7 +13,7 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13
from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
14
compute_slot_mapping_start_idx,
15
is_block_tables_empty)
16
-from vllm.utils import make_tensor_with_pad
16
+from vllm.utils import async_tensor_h2d, make_tensor_with_pad
17
18
if TYPE_CHECKING:
19
from vllm.worker.model_runner import ModelInputForGPUBuilder
@@ -310,7 +310,8 @@ class FlashAttentionMetadataBuilder(
310
for i, block_table in enumerate(self.block_tables):
311
if block_table:
312
input_block_tables[i, :len(block_table)] = block_table
313
- block_tables = torch.tensor(input_block_tables, device=device)
313
+ block_tables = torch.from_numpy(input_block_tables).to(
314
+ device=device, non_blocking=True)
315
else:
316
block_tables = make_tensor_with_pad(
317
self.block_tables,
@@ -320,15 +321,15 @@ class FlashAttentionMetadataBuilder(
321
)
322
assert max_query_len > 0, ("query_lens: {}".format(query_lens))
323
323
- context_lens_tensor = torch.tensor(self.context_lens,
324
- dtype=torch.int,
325
- device=device)
326
- seq_lens_tensor = torch.tensor(seq_lens,
327
- dtype=torch.int,
328
- device=device)
329
- query_lens_tensor = torch.tensor(query_lens,
330
- dtype=torch.long,
331
- device=device)
324
+ assert device is not None
325
+ context_lens_tensor = async_tensor_h2d(self.context_lens, torch.int,
326
+ device, self.runner.pin_memory)
327
+ seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
328
+ self.runner.pin_memory)
329
+ query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
330
+ self.runner.pin_memory)
331
+ slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
332
+ device, self.runner.pin_memory)
333
query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
334
dtype=torch.int32,
335
device=device)
@@ -344,10 +345,6 @@ class FlashAttentionMetadataBuilder(
345
dtype=query_start_loc.dtype,
346
out=query_start_loc[1:])
347
347
- slot_mapping_tensor = torch.tensor(self.slot_mapping,
348
- dtype=torch.long,
349
- device=device)
350
-
348
return FlashAttentionMetadata(
349
num_prefills=self.num_prefills,
350
slot_mapping=slot_mapping_tensor,
vllm/attention/backends/flashinfer.py
+11
-12
@@ -21,7 +21,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
21
compute_slot_mapping_start_idx,
22
is_block_tables_empty)
23
from vllm.attention.ops.paged_attn import PagedAttention
24
-from vllm.utils import get_kv_cache_torch_dtype, make_tensor_with_pad
24
+from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
25
+ make_tensor_with_pad)
26
27
if TYPE_CHECKING:
28
from vllm.worker.model_runner import ModelInputForGPUBuilder
@@ -356,7 +357,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
357
for i, block_table in enumerate(self.block_tables):
358
if block_table:
359
input_block_tables[i, :len(block_table)] = block_table
359
- block_tables = torch.tensor(input_block_tables, device=device)
360
+ block_tables = torch.from_numpy(input_block_tables).to(
361
+ device, non_blocking=True)
362
363
last_paged_kv_indptr = self.paged_kv_indptr[-1]
364
self.paged_kv_indptr.extend([last_paged_kv_indptr] *
@@ -371,12 +373,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
373
)
374
assert max_query_len > 0, ("query_lens: {}".format(query_lens))
375
374
- seq_lens_tensor = torch.tensor(seq_lens,
375
- dtype=torch.int,
376
- device=device)
377
- query_lens_tensor = torch.tensor(query_lens,
378
- dtype=torch.long,
379
- device=device)
376
+ assert device is not None
377
+ seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
378
+ self.runner.pin_memory)
379
+ query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
380
+ self.runner.pin_memory)
381
+ slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
382
+ device, self.runner.pin_memory)
383
query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
384
dtype=torch.int32,
385
device=device)
@@ -392,10 +395,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
395
dtype=query_start_loc.dtype,
396
out=query_start_loc[1:])
397
395
- slot_mapping_tensor = torch.tensor(self.slot_mapping,
396
- dtype=torch.long,
397
- device=device)
398
-
398
if len(self.paged_kv_indptr) > 0:
399
paged_kv_indices_tensor = torch.tensor(self.paged_kv_indices,
400
device="cpu",
vllm/attention/backends/utils.py
+12
-15
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Dict, List, Type, TypeVar, Union
4
import torch
5
6
from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
7
-from vllm.utils import make_tensor_with_pad
7
+from vllm.utils import async_tensor_h2d, make_tensor_with_pad
8
9
# Error string(s) for encoder/decoder
10
# unsupported attention scenarios
@@ -181,7 +181,8 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
181
for i, block_table in enumerate(self.block_tables):
182
if block_table:
183
input_block_tables[i, :len(block_table)] = block_table
184
- block_tables = torch.tensor(input_block_tables, device=device)
184
+ block_tables = torch.from_numpy(input_block_tables).to(
185
+ device, non_blocking=True)
186
else:
187
block_tables = make_tensor_with_pad(
188
self.block_tables,
@@ -191,15 +192,15 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
192
)
193
assert max_query_len > 0, "query_lens: {}".format(query_lens)
194
194
- context_lens_tensor = torch.tensor(self.context_lens,
195
- dtype=torch.int,
196
- device=device)
197
- seq_lens_tensor = torch.tensor(seq_lens,
198
- dtype=torch.int,
199
- device=device)
200
- query_lens_tensor = torch.tensor(query_lens,
201
- dtype=torch.long,
202
- device=device)
195
+ assert device is not None
196
+ context_lens_tensor = async_tensor_h2d(self.context_lens, torch.int,
197
+ device, self.runner.pin_memory)
198
+ seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
199
+ self.runner.pin_memory)
200
+ query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
201
+ self.runner.pin_memory)
202
+ slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
203
+ device, self.runner.pin_memory)
204
query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
205
dtype=torch.int32,
206
device=device)
@@ -215,10 +216,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
216
dtype=query_start_loc.dtype,
217
out=query_start_loc[1:])
218
218
- slot_mapping_tensor = torch.tensor(self.slot_mapping,
219
- dtype=torch.long,
220
- device=device)
221
-
219
return self._metadata_cls( # type: ignore
220
num_prefills=self.num_prefills,
221
slot_mapping=slot_mapping_tensor,
vllm/worker/model_runner.py
+8
-7
@@ -50,7 +50,7 @@ from vllm.prompt_adapter.worker_manager import (
50
from vllm.sampling_params import SamplingParams
51
from vllm.sequence import (IntermediateTensors, SamplerOutput,
52
SequenceGroupMetadata)
53
-from vllm.utils import (CudaMemoryProfiler, flatten_2d_lists,
53
+from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d, flatten_2d_lists,
54
get_kv_cache_torch_dtype, is_hip,
55
is_pin_memory_available)
56
from vllm.worker.model_runner_base import (
@@ -549,12 +549,13 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
549
# Tokens and positions.
550
input_tokens.extend([0] * cuda_graph_pad_size)
551
input_positions.extend([0] * cuda_graph_pad_size)
552
- input_tokens_tensor = torch.tensor(input_tokens,
553
- dtype=torch.long,
554
- device=self.runner.device)
555
- input_positions_tensor = torch.tensor(input_positions,
556
- dtype=torch.long,
557
- device=self.runner.device)
552
+ assert self.runner.device is not None
553
+ input_tokens_tensor = async_tensor_h2d(input_tokens, torch.long,
554
+ self.runner.device,
555
+ self.runner.pin_memory)
556
+ input_positions_tensor = async_tensor_h2d(input_positions, torch.long,
557
+ self.runner.device,
558
+ self.runner.pin_memory)
559
560
# Sequence and query lengths.
561
seq_lens.extend([1] * cuda_graph_pad_size)