@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a377f0bd5e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc]: optimize eager mode host time (#4196)

Co-authored-by: xuhao <xuhao@cambricon.com>

functionxu123 committed May 31, 2024 at 13:14 UTC a377f0bd5e1fa0ca069e3dbf28f4de5af64d0bb1
1 file changed +5 -6
vllm/utils.py
+5 -6
@@ -17,6 +17,7 @@ from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
17 Hashable, List, Optional, OrderedDict, Tuple, TypeVar,
18 Union)
19
20 +import numpy as np
21 import psutil
22 import torch
23
@@ -501,11 +502,6 @@ def str_to_int_tuple(s: str) -> Tuple[int, ...]:
502 f"(e.g., 1, 2, 3). Given input: {s}") from e
503
504
504 -def pad_to_max_length(x: List[int], max_len: int, pad: int) -> List[int]:
505 - assert len(x) <= max_len
506 - return x + [pad] * (max_len - len(x))
507 -
508 -
505 def make_tensor_with_pad(
506 x: List[List[int]],
507 max_len: int,
@@ -518,7 +514,10 @@ def make_tensor_with_pad(
514 The padding is applied to the end of each inner list until it reaches
515 `max_len`.
516 """
521 - padded_x = [pad_to_max_length(x_i, max_len, pad) for x_i in x]
517 + padded_x = np.zeros([len(x), max_len], dtype=np.int32) + pad
518 + for ind, blocktb in enumerate(x):
519 + assert len(blocktb) <= max_len
520 + padded_x[ind, :len(blocktb)] = blocktb
521 return torch.tensor(padded_x, dtype=dtype, device=device)
522
523