@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9042d68362
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc] Consolidate and optimize logic for building padded tensors (#6541)

Cyrus Leung committed Jul 20, 2024 at 12:17 UTC 9042d683620a7e3fa75c953fe9cca29086ce2b9a
9 files changed +77 -75
tests/conftest.py
+3 -9
@@ -21,7 +21,8 @@ from vllm.distributed import (destroy_distributed_environment,
21 from vllm.inputs import TextPrompt
22 from vllm.logger import init_logger
23 from vllm.sequence import SampleLogprobs
24 -from vllm.utils import cuda_device_count_stateless, is_cpu
24 +from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
25 + is_cpu)
26
27 logger = init_logger(__name__)
28
@@ -124,12 +125,6 @@ def image_assets() -> _ImageAssets:
125 return IMAGE_ASSETS
126
127
127 -_STR_DTYPE_TO_TORCH_DTYPE = {
128 - "half": torch.half,
129 - "bfloat16": torch.bfloat16,
130 - "float": torch.float,
131 -}
132 -
128 _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding)
129
130
@@ -151,8 +146,7 @@ class HfRunner:
146 is_vision_model: bool = False,
147 is_sparseml_model: bool = False,
148 ) -> None:
154 - assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
155 - torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
149 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
150
151 self.model_name = model_name
152
vllm/attention/backends/flash_attn.py
-3
@@ -306,11 +306,8 @@ class FlashAttentionMetadataBuilder(
306 input_block_tables[i, :len(block_table)] = block_table
307 block_tables = torch.tensor(input_block_tables, device=device)
308 else:
309 - max_block_table_len = max(
310 - len(block_table) for block_table in self.block_tables)
309 block_tables = make_tensor_with_pad(
310 self.block_tables,
313 - max_len=max_block_table_len,
311 pad=0,
312 dtype=torch.int,
313 device=device,
vllm/attention/backends/flashinfer.py
-3
@@ -344,11 +344,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
344 cuda_graph_pad_size)
345 self.paged_kv_last_page_len.extend([0] * cuda_graph_pad_size)
346 else:
347 - max_block_table_len = max(
348 - len(block_table) for block_table in self.block_tables)
347 block_tables = make_tensor_with_pad(
348 self.block_tables,
351 - max_len=max_block_table_len,
349 pad=0,
350 dtype=torch.int,
351 device=device,
vllm/attention/backends/utils.py
-3
@@ -182,11 +182,8 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
182 input_block_tables[i, :len(block_table)] = block_table
183 block_tables = torch.tensor(input_block_tables, device=device)
184 else:
185 - max_block_table_len = max(
186 - len(block_table) for block_table in self.block_tables)
185 block_tables = make_tensor_with_pad(
186 self.block_tables,
189 - max_len=max_block_table_len,
187 pad=0,
188 dtype=torch.int,
189 device=device,
vllm/model_executor/sampling_metadata.py
+19 -37
@@ -2,14 +2,13 @@ import random
2 from dataclasses import dataclass
3 from typing import Dict, List, Optional, Tuple
4
5 -import numpy as np
5 import torch
6
7 from vllm.model_executor.layers.ops.sample import get_num_triton_sampler_splits
8 from vllm.sampling_params import SamplingParams, SamplingType
9 from vllm.sequence import SequenceData, SequenceGroupMetadata
10 from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
12 - maybe_expand_dim)
11 + make_tensor_with_pad, maybe_expand_dim)
12
13 _SAMPLING_EPS = 1e-5
14 _SEED_0_REPLACEMENT = 3403598558
@@ -466,22 +465,24 @@ class SamplingTensors:
465 do_penalties = prompt_tokens or output_tokens
466
467 if do_penalties:
469 - prompt_max_len = max([len(tokens) for tokens in prompt_tokens],
470 - default=0)
471 - prompt_padded_tokens = np.full(
472 - (len(prompt_tokens), prompt_max_len),
468 + prompt_t = make_tensor_with_pad(
469 + prompt_tokens,
470 vocab_size,
474 - dtype=np.int64)
475 - for i, tokens in enumerate(prompt_tokens):
476 - prompt_padded_tokens[i, :len(tokens)] = tokens
477 - output_max_len = max([len(tokens) for tokens in output_tokens],
478 - default=0)
479 - output_padded_tokens = np.full(
480 - (len(output_tokens), output_max_len),
471 + device="cpu",
472 + dtype=torch.int64,
473 + pin_memory=pin_memory,
474 + )
475 + output_t = make_tensor_with_pad(
476 + output_tokens,
477 vocab_size,
482 - dtype=np.int64)
483 - for i, tokens in enumerate(output_tokens):
484 - output_padded_tokens[i, :len(tokens)] = tokens
478 + device="cpu",
479 + dtype=torch.int64,
480 + pin_memory=pin_memory,
481 + )
482 + else:
483 + empty_tensor = torch.empty(0, device=device, dtype=torch.long)
484 + prompt_t = empty_tensor
485 + output_t = empty_tensor
486
487 temperatures_t = torch.tensor(
488 temperatures,
@@ -531,15 +532,6 @@ class SamplingTensors:
532 dtype=torch.long,
533 pin_memory=pin_memory,
534 )
534 - if do_penalties:
535 - prompt_tensor = torch.from_numpy(prompt_padded_tokens)
536 - output_tensor = torch.from_numpy(output_padded_tokens)
537 - if pin_memory:
538 - prompt_tensor = prompt_tensor.pin_memory()
539 - output_tensor = output_tensor.pin_memory()
540 - else:
541 - prompt_tensor = None
542 - output_tensor = None
535 # need to transpose and make contiguous to
536 # copy the tensor correctly.
537 # [batch_size, n_seeds] -> [n_seeds, batch_size]
@@ -562,16 +554,6 @@ class SamplingTensors:
554 extra_seeds_gpu = None
555 sampling_seeds_gpu = sampling_seeds_gpu[:num_base_seeds]
556
565 - if do_penalties:
566 - prompt_tokens_gpu = prompt_tensor.to(device=device,
567 - non_blocking=True)
568 - output_tokens_gpu = output_tensor.to(device=device,
569 - non_blocking=True)
570 - else:
571 - empty_tensor = torch.empty(0, device=device, dtype=torch.long)
572 - prompt_tokens_gpu = empty_tensor
573 - output_tokens_gpu = empty_tensor
574 -
557 return cls(
558 temperatures=temperatures_t.to(device=device, non_blocking=True),
559 top_ps=top_ps_t.to(device=device, non_blocking=True),
@@ -583,8 +565,8 @@ class SamplingTensors:
565 non_blocking=True),
566 repetition_penalties=repetition_penalties_t.to(device=device,
567 non_blocking=True),
586 - prompt_tokens=prompt_tokens_gpu,
587 - output_tokens=output_tokens_gpu,
568 + prompt_tokens=prompt_t.to(device=device, non_blocking=True),
569 + output_tokens=output_t.to(device=device, non_blocking=True),
570 sampling_seeds=sampling_seeds_gpu,
571 sample_indices=sample_indices_t.to(device=device,
572 non_blocking=True),
vllm/utils.py
+51 -10
@@ -20,6 +20,7 @@ from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
20 Union)
21
22 import numpy as np
23 +import numpy.typing as npt
24 import psutil
25 import torch
26 import torch.types
@@ -40,6 +41,15 @@ STR_DTYPE_TO_TORCH_DTYPE = {
41 "fp8_e5m2": torch.uint8,
42 }
43
44 +TORCH_DTYPE_TO_NUMPY_DTYPE = {
45 + torch.float16: np.float16,
46 + torch.float32: np.float32,
47 + torch.float64: np.float64,
48 + torch.uint8: np.uint8,
49 + torch.int32: np.int32,
50 + torch.int64: np.int64,
51 +}
52 +
53 P = ParamSpec('P')
54 K = TypeVar("K")
55 T = TypeVar("T")
@@ -617,23 +627,54 @@ def str_to_int_tuple(s: str) -> Tuple[int, ...]:
627 f"(e.g., 1, 2, 3). Given input: {s}") from e
628
629
620 -def make_tensor_with_pad(
621 - x: List[List[int]],
622 - max_len: int,
623 - pad: int,
624 - dtype: torch.dtype,
625 - device: Optional[Union[str, torch.device]],
626 -) -> torch.Tensor:
627 - """Make a padded tensor of a 2D inputs.
630 +def make_ndarray_with_pad(
631 + x: List[List[T]],
632 + pad: T,
633 + dtype: npt.DTypeLike,
634 + *,
635 + max_len: Optional[int] = None,
636 +) -> npt.NDArray:
637 + """
638 + Make a padded array from 2D inputs.
639
640 The padding is applied to the end of each inner list until it reaches
641 `max_len`.
642 """
632 - padded_x = np.zeros([len(x), max_len], dtype=np.int32) + pad
643 + if max_len is None:
644 + # Unlike for most functions, map is faster than a genexpr over `len`
645 + max_len = max(map(len, x), default=0)
646 +
647 + padded_x = np.full((len(x), max_len), pad, dtype=dtype)
648 for ind, blocktb in enumerate(x):
649 assert len(blocktb) <= max_len
650 padded_x[ind, :len(blocktb)] = blocktb
636 - return torch.tensor(padded_x, dtype=dtype, device=device)
651 +
652 + return padded_x
653 +
654 +
655 +def make_tensor_with_pad(
656 + x: List[List[T]],
657 + pad: T,
658 + dtype: torch.dtype,
659 + *,
660 + max_len: Optional[int] = None,
661 + device: Optional[Union[str, torch.device]] = None,
662 + pin_memory: bool = False,
663 +) -> torch.Tensor:
664 + """
665 + Make a padded tensor from 2D inputs.
666 +
667 + The padding is applied to the end of each inner list until it reaches
668 + `max_len`.
669 + """
670 + np_dtype = TORCH_DTYPE_TO_NUMPY_DTYPE[dtype]
671 + padded_x = make_ndarray_with_pad(x, pad, np_dtype, max_len=max_len)
672 +
673 + tensor = torch.from_numpy(padded_x).to(device)
674 + if pin_memory:
675 + tensor = tensor.pin_memory()
676 +
677 + return tensor
678
679
680 def async_tensor_h2d(
vllm/worker/cpu_model_runner.py
-3
@@ -276,11 +276,8 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
276 dtype=torch.int,
277 device=self.device)
278
279 - max_block_table_len = max(
280 - len(block_table) for block_table in block_tables)
279 block_tables = make_tensor_with_pad(
280 block_tables,
283 - max_len=max_block_table_len,
281 pad=0,
282 dtype=torch.int,
283 device=self.device,
vllm/worker/neuron_model_runner.py
+4 -4
@@ -121,13 +121,13 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
121 max_seq_len = max(seq_lens)
122 assert max_seq_len > 0
123 input_tokens = make_tensor_with_pad(input_tokens,
124 - max_seq_len,
124 pad=0,
125 + max_len=max_seq_len,
126 dtype=torch.long,
127 device=self.device)
128 input_positions = make_tensor_with_pad(input_positions,
129 - max_seq_len,
129 pad=0,
130 + max_len=max_seq_len,
131 dtype=torch.long,
132 device=self.device)
133 input_block_ids = torch.tensor(input_block_ids,
@@ -171,13 +171,13 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
171 input_block_ids.append(block_table[0])
172
173 input_tokens = make_tensor_with_pad(input_tokens,
174 - max_len=1,
174 pad=0,
175 + max_len=1,
176 dtype=torch.long,
177 device=self.device)
178 input_positions = make_tensor_with_pad(input_positions,
179 - max_len=1,
179 pad=0,
180 + max_len=1,
181 dtype=torch.long,
182 device=self.device)
183 context_lens = torch.tensor(context_lens,
vllm/worker/xpu_model_runner.py
-3
@@ -335,11 +335,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
335 dtype=torch.int,
336 device=self.device)
337
338 - max_block_table_len = max(
339 - len(block_table) for block_table in block_tables)
338 block_tables = make_tensor_with_pad(
339 block_tables,
342 - max_len=max_block_table_len,
340 pad=0,
341 dtype=torch.int,
342 device=self.device,