@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 89a84b0bb7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] Use array to speedup padding (#6779)

Peng Guanwen committed Jul 26, 2024 at 12:31 UTC 89a84b0bb7b30706a02836234a94493ea8f780bf
3 files changed +29 -17
vllm/model_executor/layers/sampler.py
+1 -1
@@ -220,7 +220,7 @@ def _apply_min_tokens_penalty(
220 seqs_to_penalize: List[int] = []
221 for j, seq_id in enumerate(seq_ids):
222 seq_data = seq_group.seq_data[seq_id]
223 - if len(seq_data.output_token_ids) < min_tokens:
223 + if len(seq_data.output_token_ids_array) < min_tokens:
224 seqs_to_penalize.append(j)
225
226 if seqs_to_penalize:
vllm/model_executor/sampling_metadata.py
+12 -9
@@ -1,4 +1,5 @@
1 import random
2 +from array import array
3 from dataclasses import dataclass
4 from typing import Dict, List, Optional, Tuple
5
@@ -329,8 +330,8 @@ class SamplingTensors:
330 user-defined seed for each sequence.
331 extra_entropy: extra entropy to use when generating seeds.
332 """
332 - prompt_tokens: List[List[int]] = []
333 - output_tokens: List[List[int]] = []
333 + prompt_tokens: List[array] = []
334 + output_tokens: List[array] = []
335 top_ks: List[int] = []
336 temperatures: List[float] = []
337 top_ps: List[float] = []
@@ -432,13 +433,15 @@ class SamplingTensors:
433 if (seq_group.is_prompt
434 and sampling_params.prompt_logprobs is not None):
435 prefill_len = len(seq_group.prompt_logprob_indices)
435 - prompt_tokens.extend([] for _ in range(prefill_len))
436 - output_tokens.extend([] for _ in range(prefill_len))
436 + prompt_tokens.extend(
437 + array('l') for _ in range(prefill_len))
438 + output_tokens.extend(
439 + array('l') for _ in range(prefill_len))
440 if seq_group.do_sample:
441 for seq_id in seq_ids:
442 seq_data = seq_group.seq_data[seq_id]
440 - prompt_tokens.append(list(seq_data.prompt_token_ids))
441 - output_tokens.append(list(seq_data.output_token_ids))
443 + prompt_tokens.append(seq_data.prompt_token_ids_array)
444 + output_tokens.append(seq_data.output_token_ids_array)
445
446 sampling_tensors = SamplingTensors.from_lists(
447 temperatures, top_ps, top_ks, min_ps, presence_penalties,
@@ -454,9 +457,9 @@ class SamplingTensors:
457 frequency_penalties: List[float],
458 repetition_penalties: List[float],
459 sampling_seeds: List[int], sample_indices: List[int],
457 - prompt_tokens: List[List[int]],
458 - output_tokens: List[List[int]], vocab_size: int,
459 - extra_seeds_to_generate: int, device: torch.device,
460 + prompt_tokens: List[array], output_tokens: List[array],
461 + vocab_size: int, extra_seeds_to_generate: int,
462 + device: torch.device,
463 dtype: torch.dtype) -> "SamplingTensors":
464 # Note that the performance will be very bad without
465 # pinned memory.
vllm/sequence.py
+16 -7
@@ -3,6 +3,7 @@ import copy
3 import enum
4 import math
5 from abc import ABC, abstractmethod
6 +from array import array
7 from collections import defaultdict
8 from dataclasses import dataclass, field
9 from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
@@ -119,10 +120,10 @@ class SequenceData:
120 prompt_token_ids: List[int],
121 output_token_ids: Optional[List[int]] = None,
122 ) -> None:
122 - self._prompt_token_ids: List[int] = list(prompt_token_ids)
123 + self._prompt_token_ids = array('l', prompt_token_ids)
124 self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(prompt_token_ids)
124 - self._output_token_ids: List[int] = (
125 - list(output_token_ids) if output_token_ids is not None else [])
125 + self._output_token_ids = array(
126 + 'l', output_token_ids if output_token_ids is not None else [])
127
128 self.cumulative_logprob = 0.0
129 # The number of tokens that are computed (that run against the model).
@@ -132,8 +133,8 @@ class SequenceData:
133 self._update_cached_all_tokens()
134
135 def _update_cached_all_tokens(self):
135 - self._cached_all_token_ids: List[int] = (self._prompt_token_ids +
136 - self._output_token_ids)
136 + self._cached_all_token_ids: List[int] = list(self._prompt_token_ids +
137 + self._output_token_ids)
138
139 @property
140 def prompt_token_ids(self) -> Tuple[int, ...]:
@@ -141,19 +142,27 @@ class SequenceData:
142
143 @prompt_token_ids.setter
144 def prompt_token_ids(self, new_prompt_token_ids) -> None:
144 - self._prompt_token_ids = list(new_prompt_token_ids)
145 + self._prompt_token_ids = array('l', new_prompt_token_ids)
146 self._prompt_token_ids_tuple = tuple(new_prompt_token_ids)
147 self._update_cached_all_tokens()
148
149 + @property
150 + def prompt_token_ids_array(self) -> array:
151 + return self._prompt_token_ids
152 +
153 @property
154 def output_token_ids(self) -> Tuple[int, ...]:
155 return tuple(self._output_token_ids)
156
157 @output_token_ids.setter
158 def output_token_ids(self, new_output_token_ids) -> None:
154 - self._output_token_ids = list(new_output_token_ids)
159 + self._output_token_ids = array('l', new_output_token_ids)
160 self._update_cached_all_tokens()
161
162 + @property
163 + def output_token_ids_array(self) -> array:
164 + return self._output_token_ids
165 +
166 def append_token_id(self, token_id: int, logprob: float) -> None:
167 self._output_token_ids.append(token_id)
168 self._cached_all_token_ids.append(token_id)