A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc]: optimize eager mode host time (#4196)
Co-authored-by: xuhao <xuhao@cambricon.com>
functionxu123 committed
May 31, 2024 at 13:14 UTC
a377f0bd5e1fa0ca069e3dbf28f4de5af64d0bb1
1 file changed
+5
-6
vllm/utils.py
+5
-6
@@ -17,6 +17,7 @@ from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
17
Hashable, List, Optional, OrderedDict, Tuple, TypeVar,
18
Union)
19
20
+import numpy as np
21
import psutil
22
import torch
23
@@ -501,11 +502,6 @@ def str_to_int_tuple(s: str) -> Tuple[int, ...]:
502
f"(e.g., 1, 2, 3). Given input: {s}") from e
503
504
504
-def pad_to_max_length(x: List[int], max_len: int, pad: int) -> List[int]:
505
- assert len(x) <= max_len
506
- return x + [pad] * (max_len - len(x))
507
-
508
-
505
def make_tensor_with_pad(
506
x: List[List[int]],
507
max_len: int,
@@ -518,7 +514,10 @@ def make_tensor_with_pad(
514
The padding is applied to the end of each inner list until it reaches
515
`max_len`.
516
"""
521
- padded_x = [pad_to_max_length(x_i, max_len, pad) for x_i in x]
517
+ padded_x = np.zeros([len(x), max_len], dtype=np.int32) + pad
518
+ for ind, blocktb in enumerate(x):
519
+ assert len(blocktb) <= max_len
520
+ padded_x[ind, :len(blocktb)] = blocktb
521
return torch.tensor(padded_x, dtype=dtype, device=device)
522
523