@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2bb0489cb3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] Use numpy to speed up padded token processing (#6442)

Peng Guanwen committed Jul 16, 2024 at 23:13 UTC 2bb0489cb3367e46e201e84ab629df535544495b
1 file changed +18 -20
vllm/model_executor/sampling_metadata.py
+18 -20
@@ -2,6 +2,7 @@ import random
2 from dataclasses import dataclass
3 from typing import Dict, List, Optional, Tuple
4
5 +import numpy as np
6 import torch
7
8 from vllm.model_executor.layers.ops.sample import get_num_triton_sampler_splits
@@ -457,16 +458,20 @@ class SamplingTensors:
458 if do_penalties:
459 prompt_max_len = max([len(tokens) for tokens in prompt_tokens],
460 default=0)
460 - prompt_padded_tokens = [
461 - tokens + [vocab_size] * (prompt_max_len - len(tokens))
462 - for tokens in prompt_tokens
463 - ]
461 + prompt_padded_tokens = np.full(
462 + (len(prompt_tokens), prompt_max_len),
463 + vocab_size,
464 + dtype=np.int64)
465 + for i, tokens in enumerate(prompt_tokens):
466 + prompt_padded_tokens[i, :len(tokens)] = tokens
467 output_max_len = max([len(tokens) for tokens in output_tokens],
468 default=0)
466 - output_padded_tokens = [
467 - tokens + [vocab_size] * (output_max_len - len(tokens))
468 - for tokens in output_tokens
469 - ]
469 + output_padded_tokens = np.full(
470 + (len(output_tokens), output_max_len),
471 + vocab_size,
472 + dtype=np.int64)
473 + for i, tokens in enumerate(output_tokens):
474 + output_padded_tokens[i, :len(tokens)] = tokens
475
476 temperatures_t = torch.tensor(
477 temperatures,
@@ -517,18 +522,11 @@ class SamplingTensors:
522 pin_memory=pin_memory,
523 )
524 if do_penalties:
520 - prompt_tensor = torch.tensor(
521 - prompt_padded_tokens,
522 - device="cpu",
523 - dtype=torch.long,
524 - pin_memory=pin_memory,
525 - )
526 - output_tensor = torch.tensor(
527 - output_padded_tokens,
528 - device="cpu",
529 - dtype=torch.long,
530 - pin_memory=pin_memory,
531 - )
525 + prompt_tensor = torch.from_numpy(prompt_padded_tokens)
526 + output_tensor = torch.from_numpy(output_padded_tokens)
527 + if pin_memory:
528 + prompt_tensor = prompt_tensor.pin_memory()
529 + output_tensor = output_tensor.pin_memory()
530 else:
531 prompt_tensor = None
532 output_tensor = None