A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Core] Use array to speedup padding (#6779)
Peng Guanwen committed
Jul 26, 2024 at 12:31 UTC
89a84b0bb7b30706a02836234a94493ea8f780bf
3 files changed
+29
-17
vllm/model_executor/layers/sampler.py
+1
-1
@@ -220,7 +220,7 @@ def _apply_min_tokens_penalty(
220
seqs_to_penalize: List[int] = []
221
for j, seq_id in enumerate(seq_ids):
222
seq_data = seq_group.seq_data[seq_id]
223
- if len(seq_data.output_token_ids) < min_tokens:
223
+ if len(seq_data.output_token_ids_array) < min_tokens:
224
seqs_to_penalize.append(j)
225
226
if seqs_to_penalize:
vllm/model_executor/sampling_metadata.py
+12
-9
@@ -1,4 +1,5 @@
1
import random
2
+from array import array
3
from dataclasses import dataclass
4
from typing import Dict, List, Optional, Tuple
5
@@ -329,8 +330,8 @@ class SamplingTensors:
330
user-defined seed for each sequence.
331
extra_entropy: extra entropy to use when generating seeds.
332
"""
332
- prompt_tokens: List[List[int]] = []
333
- output_tokens: List[List[int]] = []
333
+ prompt_tokens: List[array] = []
334
+ output_tokens: List[array] = []
335
top_ks: List[int] = []
336
temperatures: List[float] = []
337
top_ps: List[float] = []
@@ -432,13 +433,15 @@ class SamplingTensors:
433
if (seq_group.is_prompt
434
and sampling_params.prompt_logprobs is not None):
435
prefill_len = len(seq_group.prompt_logprob_indices)
435
- prompt_tokens.extend([] for _ in range(prefill_len))
436
- output_tokens.extend([] for _ in range(prefill_len))
436
+ prompt_tokens.extend(
437
+ array('l') for _ in range(prefill_len))
438
+ output_tokens.extend(
439
+ array('l') for _ in range(prefill_len))
440
if seq_group.do_sample:
441
for seq_id in seq_ids:
442
seq_data = seq_group.seq_data[seq_id]
440
- prompt_tokens.append(list(seq_data.prompt_token_ids))
441
- output_tokens.append(list(seq_data.output_token_ids))
443
+ prompt_tokens.append(seq_data.prompt_token_ids_array)
444
+ output_tokens.append(seq_data.output_token_ids_array)
445
446
sampling_tensors = SamplingTensors.from_lists(
447
temperatures, top_ps, top_ks, min_ps, presence_penalties,
@@ -454,9 +457,9 @@ class SamplingTensors:
457
frequency_penalties: List[float],
458
repetition_penalties: List[float],
459
sampling_seeds: List[int], sample_indices: List[int],
457
- prompt_tokens: List[List[int]],
458
- output_tokens: List[List[int]], vocab_size: int,
459
- extra_seeds_to_generate: int, device: torch.device,
460
+ prompt_tokens: List[array], output_tokens: List[array],
461
+ vocab_size: int, extra_seeds_to_generate: int,
462
+ device: torch.device,
463
dtype: torch.dtype) -> "SamplingTensors":
464
# Note that the performance will be very bad without
465
# pinned memory.
vllm/sequence.py
+16
-7
@@ -3,6 +3,7 @@ import copy
3
import enum
4
import math
5
from abc import ABC, abstractmethod
6
+from array import array
7
from collections import defaultdict
8
from dataclasses import dataclass, field
9
from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
@@ -119,10 +120,10 @@ class SequenceData:
120
prompt_token_ids: List[int],
121
output_token_ids: Optional[List[int]] = None,
122
) -> None:
122
- self._prompt_token_ids: List[int] = list(prompt_token_ids)
123
+ self._prompt_token_ids = array('l', prompt_token_ids)
124
self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(prompt_token_ids)
124
- self._output_token_ids: List[int] = (
125
- list(output_token_ids) if output_token_ids is not None else [])
125
+ self._output_token_ids = array(
126
+ 'l', output_token_ids if output_token_ids is not None else [])
127
128
self.cumulative_logprob = 0.0
129
# The number of tokens that are computed (that run against the model).
@@ -132,8 +133,8 @@ class SequenceData:
133
self._update_cached_all_tokens()
134
135
def _update_cached_all_tokens(self):
135
- self._cached_all_token_ids: List[int] = (self._prompt_token_ids +
136
- self._output_token_ids)
136
+ self._cached_all_token_ids: List[int] = list(self._prompt_token_ids +
137
+ self._output_token_ids)
138
139
@property
140
def prompt_token_ids(self) -> Tuple[int, ...]:
@@ -141,19 +142,27 @@ class SequenceData:
142
143
@prompt_token_ids.setter
144
def prompt_token_ids(self, new_prompt_token_ids) -> None:
144
- self._prompt_token_ids = list(new_prompt_token_ids)
145
+ self._prompt_token_ids = array('l', new_prompt_token_ids)
146
self._prompt_token_ids_tuple = tuple(new_prompt_token_ids)
147
self._update_cached_all_tokens()
148
149
+ @property
150
+ def prompt_token_ids_array(self) -> array:
151
+ return self._prompt_token_ids
152
+
153
@property
154
def output_token_ids(self) -> Tuple[int, ...]:
155
return tuple(self._output_token_ids)
156
157
@output_token_ids.setter
158
def output_token_ids(self, new_output_token_ids) -> None:
154
- self._output_token_ids = list(new_output_token_ids)
159
+ self._output_token_ids = array('l', new_output_token_ids)
160
self._update_cached_all_tokens()
161
162
+ @property
163
+ def output_token_ids_array(self) -> array:
164
+ return self._output_token_ids
165
+
166
def append_token_id(self, token_id: int, logprob: float) -> None:
167
self._output_token_ids.append(token_id)
168
self._cached_all_token_ids.append(token_id)