A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Misc] Consolidate and optimize logic for building padded tensors (#6541)
Cyrus Leung committed
Jul 20, 2024 at 12:17 UTC
9042d683620a7e3fa75c953fe9cca29086ce2b9a
9 files changed
+77
-75
tests/conftest.py
+3
-9
@@ -21,7 +21,8 @@ from vllm.distributed import (destroy_distributed_environment,
21
from vllm.inputs import TextPrompt
22
from vllm.logger import init_logger
23
from vllm.sequence import SampleLogprobs
24
-from vllm.utils import cuda_device_count_stateless, is_cpu
24
+from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
25
+ is_cpu)
26
27
logger = init_logger(__name__)
28
@@ -124,12 +125,6 @@ def image_assets() -> _ImageAssets:
125
return IMAGE_ASSETS
126
127
127
-_STR_DTYPE_TO_TORCH_DTYPE = {
128
- "half": torch.half,
129
- "bfloat16": torch.bfloat16,
130
- "float": torch.float,
131
-}
132
-
128
_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding)
129
130
@@ -151,8 +146,7 @@ class HfRunner:
146
is_vision_model: bool = False,
147
is_sparseml_model: bool = False,
148
) -> None:
154
- assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
155
- torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
149
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
150
151
self.model_name = model_name
152
vllm/attention/backends/flash_attn.py
-3
@@ -306,11 +306,8 @@ class FlashAttentionMetadataBuilder(
306
input_block_tables[i, :len(block_table)] = block_table
307
block_tables = torch.tensor(input_block_tables, device=device)
308
else:
309
- max_block_table_len = max(
310
- len(block_table) for block_table in self.block_tables)
309
block_tables = make_tensor_with_pad(
310
self.block_tables,
313
- max_len=max_block_table_len,
311
pad=0,
312
dtype=torch.int,
313
device=device,
vllm/attention/backends/flashinfer.py
-3
@@ -344,11 +344,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
344
cuda_graph_pad_size)
345
self.paged_kv_last_page_len.extend([0] * cuda_graph_pad_size)
346
else:
347
- max_block_table_len = max(
348
- len(block_table) for block_table in self.block_tables)
347
block_tables = make_tensor_with_pad(
348
self.block_tables,
351
- max_len=max_block_table_len,
349
pad=0,
350
dtype=torch.int,
351
device=device,
vllm/attention/backends/utils.py
-3
@@ -182,11 +182,8 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
182
input_block_tables[i, :len(block_table)] = block_table
183
block_tables = torch.tensor(input_block_tables, device=device)
184
else:
185
- max_block_table_len = max(
186
- len(block_table) for block_table in self.block_tables)
185
block_tables = make_tensor_with_pad(
186
self.block_tables,
189
- max_len=max_block_table_len,
187
pad=0,
188
dtype=torch.int,
189
device=device,
vllm/model_executor/sampling_metadata.py
+19
-37
@@ -2,14 +2,13 @@ import random
2
from dataclasses import dataclass
3
from typing import Dict, List, Optional, Tuple
4
5
-import numpy as np
5
import torch
6
7
from vllm.model_executor.layers.ops.sample import get_num_triton_sampler_splits
8
from vllm.sampling_params import SamplingParams, SamplingType
9
from vllm.sequence import SequenceData, SequenceGroupMetadata
10
from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
12
- maybe_expand_dim)
11
+ make_tensor_with_pad, maybe_expand_dim)
12
13
_SAMPLING_EPS = 1e-5
14
_SEED_0_REPLACEMENT = 3403598558
@@ -466,22 +465,24 @@ class SamplingTensors:
465
do_penalties = prompt_tokens or output_tokens
466
467
if do_penalties:
469
- prompt_max_len = max([len(tokens) for tokens in prompt_tokens],
470
- default=0)
471
- prompt_padded_tokens = np.full(
472
- (len(prompt_tokens), prompt_max_len),
468
+ prompt_t = make_tensor_with_pad(
469
+ prompt_tokens,
470
vocab_size,
474
- dtype=np.int64)
475
- for i, tokens in enumerate(prompt_tokens):
476
- prompt_padded_tokens[i, :len(tokens)] = tokens
477
- output_max_len = max([len(tokens) for tokens in output_tokens],
478
- default=0)
479
- output_padded_tokens = np.full(
480
- (len(output_tokens), output_max_len),
471
+ device="cpu",
472
+ dtype=torch.int64,
473
+ pin_memory=pin_memory,
474
+ )
475
+ output_t = make_tensor_with_pad(
476
+ output_tokens,
477
vocab_size,
482
- dtype=np.int64)
483
- for i, tokens in enumerate(output_tokens):
484
- output_padded_tokens[i, :len(tokens)] = tokens
478
+ device="cpu",
479
+ dtype=torch.int64,
480
+ pin_memory=pin_memory,
481
+ )
482
+ else:
483
+ empty_tensor = torch.empty(0, device=device, dtype=torch.long)
484
+ prompt_t = empty_tensor
485
+ output_t = empty_tensor
486
487
temperatures_t = torch.tensor(
488
temperatures,
@@ -531,15 +532,6 @@ class SamplingTensors:
532
dtype=torch.long,
533
pin_memory=pin_memory,
534
)
534
- if do_penalties:
535
- prompt_tensor = torch.from_numpy(prompt_padded_tokens)
536
- output_tensor = torch.from_numpy(output_padded_tokens)
537
- if pin_memory:
538
- prompt_tensor = prompt_tensor.pin_memory()
539
- output_tensor = output_tensor.pin_memory()
540
- else:
541
- prompt_tensor = None
542
- output_tensor = None
535
# need to transpose and make contiguous to
536
# copy the tensor correctly.
537
# [batch_size, n_seeds] -> [n_seeds, batch_size]
@@ -562,16 +554,6 @@ class SamplingTensors:
554
extra_seeds_gpu = None
555
sampling_seeds_gpu = sampling_seeds_gpu[:num_base_seeds]
556
565
- if do_penalties:
566
- prompt_tokens_gpu = prompt_tensor.to(device=device,
567
- non_blocking=True)
568
- output_tokens_gpu = output_tensor.to(device=device,
569
- non_blocking=True)
570
- else:
571
- empty_tensor = torch.empty(0, device=device, dtype=torch.long)
572
- prompt_tokens_gpu = empty_tensor
573
- output_tokens_gpu = empty_tensor
574
-
557
return cls(
558
temperatures=temperatures_t.to(device=device, non_blocking=True),
559
top_ps=top_ps_t.to(device=device, non_blocking=True),
@@ -583,8 +565,8 @@ class SamplingTensors:
565
non_blocking=True),
566
repetition_penalties=repetition_penalties_t.to(device=device,
567
non_blocking=True),
586
- prompt_tokens=prompt_tokens_gpu,
587
- output_tokens=output_tokens_gpu,
568
+ prompt_tokens=prompt_t.to(device=device, non_blocking=True),
569
+ output_tokens=output_t.to(device=device, non_blocking=True),
570
sampling_seeds=sampling_seeds_gpu,
571
sample_indices=sample_indices_t.to(device=device,
572
non_blocking=True),
vllm/utils.py
+51
-10
@@ -20,6 +20,7 @@ from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
20
Union)
21
22
import numpy as np
23
+import numpy.typing as npt
24
import psutil
25
import torch
26
import torch.types
@@ -40,6 +41,15 @@ STR_DTYPE_TO_TORCH_DTYPE = {
41
"fp8_e5m2": torch.uint8,
42
}
43
44
+TORCH_DTYPE_TO_NUMPY_DTYPE = {
45
+ torch.float16: np.float16,
46
+ torch.float32: np.float32,
47
+ torch.float64: np.float64,
48
+ torch.uint8: np.uint8,
49
+ torch.int32: np.int32,
50
+ torch.int64: np.int64,
51
+}
52
+
53
P = ParamSpec('P')
54
K = TypeVar("K")
55
T = TypeVar("T")
@@ -617,23 +627,54 @@ def str_to_int_tuple(s: str) -> Tuple[int, ...]:
627
f"(e.g., 1, 2, 3). Given input: {s}") from e
628
629
620
-def make_tensor_with_pad(
621
- x: List[List[int]],
622
- max_len: int,
623
- pad: int,
624
- dtype: torch.dtype,
625
- device: Optional[Union[str, torch.device]],
626
-) -> torch.Tensor:
627
- """Make a padded tensor of a 2D inputs.
630
+def make_ndarray_with_pad(
631
+ x: List[List[T]],
632
+ pad: T,
633
+ dtype: npt.DTypeLike,
634
+ *,
635
+ max_len: Optional[int] = None,
636
+) -> npt.NDArray:
637
+ """
638
+ Make a padded array from 2D inputs.
639
640
The padding is applied to the end of each inner list until it reaches
641
`max_len`.
642
"""
632
- padded_x = np.zeros([len(x), max_len], dtype=np.int32) + pad
643
+ if max_len is None:
644
+ # Unlike for most functions, map is faster than a genexpr over `len`
645
+ max_len = max(map(len, x), default=0)
646
+
647
+ padded_x = np.full((len(x), max_len), pad, dtype=dtype)
648
for ind, blocktb in enumerate(x):
649
assert len(blocktb) <= max_len
650
padded_x[ind, :len(blocktb)] = blocktb
636
- return torch.tensor(padded_x, dtype=dtype, device=device)
651
+
652
+ return padded_x
653
+
654
+
655
+def make_tensor_with_pad(
656
+ x: List[List[T]],
657
+ pad: T,
658
+ dtype: torch.dtype,
659
+ *,
660
+ max_len: Optional[int] = None,
661
+ device: Optional[Union[str, torch.device]] = None,
662
+ pin_memory: bool = False,
663
+) -> torch.Tensor:
664
+ """
665
+ Make a padded tensor from 2D inputs.
666
+
667
+ The padding is applied to the end of each inner list until it reaches
668
+ `max_len`.
669
+ """
670
+ np_dtype = TORCH_DTYPE_TO_NUMPY_DTYPE[dtype]
671
+ padded_x = make_ndarray_with_pad(x, pad, np_dtype, max_len=max_len)
672
+
673
+ tensor = torch.from_numpy(padded_x).to(device)
674
+ if pin_memory:
675
+ tensor = tensor.pin_memory()
676
+
677
+ return tensor
678
679
680
def async_tensor_h2d(
vllm/worker/cpu_model_runner.py
-3
@@ -276,11 +276,8 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
276
dtype=torch.int,
277
device=self.device)
278
279
- max_block_table_len = max(
280
- len(block_table) for block_table in block_tables)
279
block_tables = make_tensor_with_pad(
280
block_tables,
283
- max_len=max_block_table_len,
281
pad=0,
282
dtype=torch.int,
283
device=self.device,
vllm/worker/neuron_model_runner.py
+4
-4
@@ -121,13 +121,13 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
121
max_seq_len = max(seq_lens)
122
assert max_seq_len > 0
123
input_tokens = make_tensor_with_pad(input_tokens,
124
- max_seq_len,
124
pad=0,
125
+ max_len=max_seq_len,
126
dtype=torch.long,
127
device=self.device)
128
input_positions = make_tensor_with_pad(input_positions,
129
- max_seq_len,
129
pad=0,
130
+ max_len=max_seq_len,
131
dtype=torch.long,
132
device=self.device)
133
input_block_ids = torch.tensor(input_block_ids,
@@ -171,13 +171,13 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
171
input_block_ids.append(block_table[0])
172
173
input_tokens = make_tensor_with_pad(input_tokens,
174
- max_len=1,
174
pad=0,
175
+ max_len=1,
176
dtype=torch.long,
177
device=self.device)
178
input_positions = make_tensor_with_pad(input_positions,
179
- max_len=1,
179
pad=0,
180
+ max_len=1,
181
dtype=torch.long,
182
device=self.device)
183
context_lens = torch.tensor(context_lens,
vllm/worker/xpu_model_runner.py
-3
@@ -335,11 +335,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
335
dtype=torch.int,
336
device=self.device)
337
338
- max_block_table_len = max(
339
- len(block_table) for block_table in block_tables)
338
block_tables = make_tensor_with_pad(
339
block_tables,
342
- max_len=max_block_table_len,
340
pad=0,
341
dtype=torch.int,
342
device=self.device,