@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a7347d9a6d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Make sampler less blocking (#1889)

Antoni Baum committed Dec 17, 2023 at 07:03 UTC a7347d9a6d2391734d838ab6a4f3a702e348d9fa
2 files changed +310 -198
vllm/model_executor/layers/sampler.py
+123 -198
@@ -6,13 +6,11 @@ import torch.nn as nn
6
7 from vllm.model_executor.parallel_utils.communication_op import (
8 tensor_model_parallel_all_gather)
9 -from vllm.model_executor.sampling_metadata import SamplingMetadata
9 +from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTensors
10 from vllm.sampling_params import SamplingParams, SamplingType
11 from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12 SequenceData, SequenceGroupOutput, SequenceOutput)
13
14 -_SAMPLING_EPS = 1e-5
15 -
14
15 class Sampler(nn.Module):
16 """Samples the next tokens from the model's outputs.
@@ -32,6 +30,7 @@ class Sampler(nn.Module):
30 def __init__(self, vocab_size: int) -> None:
31 super().__init__()
32 self.vocab_size = vocab_size
33 + self._copy_stream: torch.cuda.Stream = torch.cuda.Stream()
34
35 def forward(
36 self,
@@ -47,40 +46,38 @@ class Sampler(nn.Module):
46 logits = _get_logits(hidden_states, embedding, embedding_bias,
47 self.vocab_size)
48
49 + _, vocab_size = logits.shape
50 +
51 # Apply logits processors (if any).
52 logits = _apply_logits_processors(logits, sampling_metadata)
53 +
54 + # Prepare sampling tensors in another stream to overlap
55 + # CPU<->GPU data transfer with GPU computation in forward pass.
56 + with torch.cuda.stream(self._copy_stream):
57 + (sampling_tensors, do_penalties, do_top_p_top_k,
58 + do_min_p) = SamplingTensors.from_sampling_metadata(
59 + sampling_metadata, vocab_size, logits.device, logits.dtype)
60 +
61 + torch.cuda.current_stream().wait_stream(self._copy_stream)
62 +
63 # Apply presence and frequency penalties.
53 - presence_penalties, frequency_penalties, repetition_penalties = (
54 - _get_penalties(sampling_metadata))
55 - assert len(presence_penalties) == logits.shape[0]
56 - assert len(frequency_penalties) == logits.shape[0]
57 - assert len(repetition_penalties) == logits.shape[0]
58 - logits = _apply_penalties(logits, sampling_metadata,
59 - presence_penalties, frequency_penalties,
60 - repetition_penalties)
64 + if do_penalties:
65 + logits = _apply_penalties(logits, sampling_tensors.prompt_tokens,
66 + sampling_tensors.output_tokens,
67 + sampling_tensors.presence_penalties,
68 + sampling_tensors.frequency_penalties,
69 + sampling_tensors.repetition_penalties)
70
71 # Apply temperature scaling.
63 - temperatures = _get_temperatures(sampling_metadata)
64 - assert len(temperatures) == logits.shape[0]
65 - if any(t != 1.0 for t in temperatures):
66 - t = torch.tensor(temperatures,
67 - dtype=logits.dtype,
68 - device=logits.device)
69 - # Use in-place division to avoid creating a new tensor.
70 - logits.div_(t.unsqueeze(dim=1))
71 -
72 - # Apply top-p and top-k truncation.
73 - top_ps, top_ks, min_ps = _get_top_p_top_k_min_p(
74 - sampling_metadata, self.vocab_size)
75 - assert len(top_ps) == len(top_ks) == logits.shape[0]
76 - do_top_p = any(p < 1.0 - _SAMPLING_EPS for p in top_ps)
77 - do_top_k = any(k != self.vocab_size for k in top_ks)
78 - if do_top_p or do_top_k:
79 - logits = _apply_top_p_top_k(logits, top_ps, top_ks)
80 -
81 - do_min_p = any(mp > _SAMPLING_EPS for mp in min_ps)
72 + # Use in-place division to avoid creating a new tensor.
73 + logits.div_(sampling_tensors.temperatures.unsqueeze_(dim=1))
74 +
75 + if do_top_p_top_k:
76 + logits = _apply_top_p_top_k(logits, sampling_tensors.top_ps,
77 + sampling_tensors.top_ks)
78 +
79 if do_min_p:
83 - logits = _apply_min_p(logits, min_ps)
80 + logits = _apply_min_p(logits, sampling_tensors.min_ps)
81
82 # We use float32 for probabilities and log probabilities.
83 # Compute the probabilities.
@@ -120,32 +117,6 @@ def _prune_hidden_states(
117 sampling_metadata.selected_token_indices)
118
119
123 -def _get_penalties(
124 - sampling_metadata: SamplingMetadata
125 -) -> Tuple[List[float], List[float], List[float]]:
126 - # Collect the presence and frequency penalties.
127 - presence_penalties: List[float] = []
128 - frequency_penalties: List[float] = []
129 - repetition_penalties: List[float] = []
130 - for i, seq_group in enumerate(sampling_metadata.seq_groups):
131 - seq_ids, sampling_params = seq_group
132 - p = sampling_params.presence_penalty
133 - f = sampling_params.frequency_penalty
134 - r = sampling_params.repetition_penalty
135 - if (i < sampling_metadata.num_prompts
136 - and sampling_params.prompt_logprobs is not None):
137 - # NOTE: We do not apply presence and frequency penalties for the
138 - # prompt token positions where we don't sample new tokens.
139 - prompt_len = sampling_metadata.prompt_lens[i]
140 - presence_penalties += [0] * (prompt_len - 1)
141 - frequency_penalties += [0] * (prompt_len - 1)
142 - repetition_penalties += [1] * (prompt_len - 1)
143 - presence_penalties += [p] * len(seq_ids)
144 - frequency_penalties += [f] * len(seq_ids)
145 - repetition_penalties += [r] * len(seq_ids)
146 - return presence_penalties, frequency_penalties, repetition_penalties
147 -
148 -
120 def _get_prompt_and_output_tokens(
121 sampling_metadata: SamplingMetadata,
122 ) -> Tuple[List[List[int]], List[List[int]]]:
@@ -168,25 +139,16 @@ def _get_prompt_and_output_tokens(
139
140
141 def _get_bin_counts_and_mask(
171 - logits: torch.Tensor,
172 - tokens: List[List[int]],
142 + tokens: torch.Tensor,
143 vocab_size: int,
144 num_seqs: int,
145 ) -> Tuple[torch.Tensor, torch.Tensor]:
176 - max_len = max(len(tokens) for tokens in tokens)
177 - padded_tokens = [
178 - tokens + [vocab_size] * (max_len - len(tokens)) for tokens in tokens
179 - ]
180 - tokens_tensor = torch.tensor(padded_tokens,
181 - dtype=torch.long,
182 - device=logits.device)
183 -
146 # Compute the bin counts for the tokens.
147 # vocab_size + 1 for padding.
148 bin_counts = torch.zeros((num_seqs, vocab_size + 1),
149 dtype=torch.long,
188 - device=logits.device)
189 - bin_counts.scatter_add_(1, tokens_tensor, torch.ones_like(tokens_tensor))
150 + device=tokens.device)
151 + bin_counts.scatter_add_(1, tokens, torch.ones_like(tokens))
152 bin_counts = bin_counts[:, :vocab_size]
153 mask = bin_counts > 0
154
@@ -217,45 +179,16 @@ def _apply_logits_processors(
179 return logits
180
181
220 -def _apply_penalties(
221 - logits: torch.Tensor,
222 - sampling_metadata: SamplingMetadata,
223 - presence_penalties: List[float],
224 - frequency_penalties: List[float],
225 - repetition_penalties: List[float],
226 -) -> torch.Tensor:
182 +def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
183 + output_tokens_tensor: torch.Tensor,
184 + presence_penalties: torch.Tensor,
185 + frequency_penalties: torch.Tensor,
186 + repetition_penalties: torch.Tensor) -> torch.Tensor:
187 num_seqs, vocab_size = logits.shape
228 - for i in range(num_seqs):
229 - p = presence_penalties[i]
230 - f = frequency_penalties[i]
231 - r = repetition_penalties[i]
232 - if abs(p) < _SAMPLING_EPS and abs(f) < _SAMPLING_EPS and abs(
233 - r - 1.0) < _SAMPLING_EPS:
234 - continue
235 - break
236 - else:
237 - # Return early if all sequences have zero penalties.
238 - return logits
239 -
240 - prompt_tokens, output_tokens = (
241 - _get_prompt_and_output_tokens(sampling_metadata))
242 - assert len(prompt_tokens) == logits.shape[0]
243 - assert len(output_tokens) == logits.shape[0]
244 -
245 - prompt_bin_counts, prompt_mask = _get_bin_counts_and_mask(
246 - logits, prompt_tokens, vocab_size, num_seqs)
188 + _, prompt_mask = _get_bin_counts_and_mask(prompt_tokens_tensor, vocab_size,
189 + num_seqs)
190 output_bin_counts, output_mask = _get_bin_counts_and_mask(
248 - logits, output_tokens, vocab_size, num_seqs)
249 -
250 - repetition_penalties = torch.tensor(repetition_penalties,
251 - dtype=logits.dtype,
252 - device=logits.device)
253 - frequency_penalties = torch.tensor(frequency_penalties,
254 - dtype=logits.dtype,
255 - device=logits.device)
256 - presence_penalties = torch.tensor(presence_penalties,
257 - dtype=logits.dtype,
258 - device=logits.device)
191 + output_tokens_tensor, vocab_size, num_seqs)
192
193 repetition_penalties = repetition_penalties[:, None].repeat(1, vocab_size)
194 repetition_penalties[~(prompt_mask | output_mask)] = 1.0
@@ -264,109 +197,65 @@ def _apply_penalties(
197
198 # We follow the definition in OpenAI API.
199 # Refer to https://platform.openai.com/docs/api-reference/parameter-details
267 - logits -= frequency_penalties.unsqueeze(dim=1) * output_bin_counts
268 - logits -= presence_penalties.unsqueeze(dim=1) * output_mask
200 + logits -= frequency_penalties.unsqueeze_(dim=1) * output_bin_counts
201 + logits -= presence_penalties.unsqueeze_(dim=1) * output_mask
202 return logits
203
204
272 -def _get_temperatures(sampling_metadata: SamplingMetadata) -> List[float]:
273 - # Collect the temperatures for the logits.
274 - temperatures: List[float] = []
275 - for i, seq_group in enumerate(sampling_metadata.seq_groups):
276 - seq_ids, sampling_params = seq_group
277 - temperature = sampling_params.temperature
278 - if temperature < _SAMPLING_EPS:
279 - # NOTE: Zero temperature means deterministic sampling
280 - # (i.e., greedy sampling or beam search).
281 - # Set the temperature to 1 to avoid division by zero.
282 - temperature = 1.0
283 - if (i < sampling_metadata.num_prompts
284 - and sampling_params.prompt_logprobs is not None):
285 - prompt_len = sampling_metadata.prompt_lens[i]
286 - temperatures += [temperature] * (prompt_len - 1)
287 - temperatures += [temperature] * len(seq_ids)
288 - return temperatures
289 -
290 -
291 -def _get_top_p_top_k_min_p(
292 - sampling_metadata: SamplingMetadata,
293 - vocab_size: int,
294 -) -> Tuple[List[float], List[int], List[float]]:
295 - top_ps: List[float] = []
296 - top_ks: List[int] = []
297 - min_ps: List[float] = []
298 - for i, seq_group in enumerate(sampling_metadata.seq_groups):
299 - seq_ids, sampling_params = seq_group
300 - top_p = sampling_params.top_p
301 - min_p = sampling_params.min_p
302 - # k should not be greater than the vocab size.
303 - top_k = min(sampling_params.top_k, vocab_size)
304 - # k=-1 means no truncation.
305 - top_k = vocab_size if top_k == -1 else top_k
306 - if (i < sampling_metadata.num_prompts
307 - and sampling_params.prompt_logprobs is not None):
308 - prompt_len = sampling_metadata.prompt_lens[i]
309 - top_ps += [top_p] * (prompt_len - 1)
310 - top_ks += [top_k] * (prompt_len - 1)
311 - min_ps += [min_p] * (prompt_len - 1)
312 - top_ps += [top_p] * len(seq_ids)
313 - top_ks += [top_k] * len(seq_ids)
314 - min_ps += [min_p] * len(seq_ids)
315 - return top_ps, top_ks, min_ps
316 -
317 -
205 def _apply_top_p_top_k(
206 logits: torch.Tensor,
320 - top_ps: List[float],
321 - top_ks: List[int],
207 + p: torch.Tensor,
208 + k: torch.Tensor,
209 ) -> torch.Tensor:
323 - p = torch.tensor(top_ps, dtype=logits.dtype, device=logits.device)
324 - k = torch.tensor(top_ks, dtype=torch.int, device=logits.device)
210 logits_sort, logits_idx = logits.sort(dim=-1, descending=True)
211
212 # Apply top-p.
213 probs_sort = logits_sort.softmax(dim=-1)
329 - probs_sum = probs_sort.cumsum(dim=-1)
330 - top_p_mask = (probs_sum - probs_sort) > p.unsqueeze(dim=1)
331 - logits_sort[top_p_mask] = -float("inf")
214 + probs_sum = probs_sort.cumsum(dim=-1).sub_(probs_sort)
215 + top_p_mask = probs_sum > p.unsqueeze_(dim=1)
216
217 # Apply top-k.
218 # Create a mask for the top-k elements.
219 top_k_mask = torch.arange(logits_idx.shape[-1], device=logits_idx.device)
220 top_k_mask = top_k_mask.expand(logits_idx.shape[0], -1)
337 - top_k_mask = top_k_mask >= k.unsqueeze(dim=1)
338 - logits_sort[top_k_mask] = -float("inf")
221 + top_k_mask = top_k_mask >= k.unsqueeze_(dim=1)
222 +
223 + # Final mask.
224 + mask = (top_p_mask | top_k_mask)
225 + logits_sort.masked_fill_(mask, -float("inf"))
226
227 # Re-sort the probabilities.
341 - logits = torch.gather(logits_sort,
342 - dim=-1,
343 - index=torch.argsort(logits_idx, dim=-1))
228 + src = torch.arange(logits_idx.shape[-1],
229 + device=logits_idx.device).expand_as(logits_idx)
230 + logits_idx_inv = torch.empty_like(logits_idx).scatter_(dim=-1,
231 + index=logits_idx,
232 + src=src)
233 + logits = torch.gather(logits_sort, dim=-1, index=logits_idx_inv)
234 return logits
235
236
237 def _apply_min_p(
238 logits: torch.Tensor,
349 - min_ps: List[float],
239 + min_p: torch.Tensor,
240 ) -> torch.Tensor:
241 """
242 Adapted from
243 https://github.com/oobabooga/text-generation-webui/blob/3146124ec01f02c8fb1650a6517cf1b60b537aaf/modules/sampler_hijack.py#L16C17-L16C17
244 """
355 - min_p = torch.tensor(min_ps, dtype=logits.dtype, device=logits.device)
245 probs = torch.softmax(logits, dim=-1)
246 top_probs, _ = probs.max(dim=-1, keepdim=True)
358 - scaled_min_p = min_p.unsqueeze(dim=1) * top_probs
247 + scaled_min_p = min_p.unsqueeze_(dim=1) * top_probs
248 tokens_to_remove = probs < scaled_min_p
360 - logits = logits.masked_fill(tokens_to_remove, -float("inf"))
249 + logits = logits.masked_fill_(tokens_to_remove, -float("inf"))
250
251 return logits
252
253
254 def _greedy_sample(
255 selected_seq_groups: List[Tuple[List[int], SamplingParams]],
367 - logprobs: torch.Tensor,
256 + samples: torch.Tensor,
257 ) -> List[Tuple[List[int], List[int]]]:
369 - samples = torch.argmax(logprobs, dim=-1).cpu()
258 + samples = samples.tolist()
259 sample_idx = 0
260 results = []
261 for seq_group in selected_seq_groups:
@@ -375,27 +264,19 @@ def _greedy_sample(
264 assert num_parent_seqs == 1, (
265 "Greedy sampling should have only one seq.")
266 parent_ids = list(range(num_parent_seqs))
378 - next_token_ids = [samples[sample_idx].item()]
267 + next_token_ids = [samples[sample_idx]]
268 results.append((next_token_ids, parent_ids))
269 sample_idx += num_parent_seqs
381 - assert sample_idx == logprobs.size(0)
270 return results
271
272
273 def _random_sample(
274 selected_seq_groups: List[Tuple[List[int], SamplingParams]],
275 is_prompts: List[bool],
388 - probs: torch.Tensor,
276 + random_samples: torch.Tensor,
277 ) -> List[Tuple[List[int], List[int]]]:
278 # Find the maximum best_of value of the prompt phase requests.
391 - max_best_of = 1
392 - for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
393 - if is_prompt:
394 - seq_ids, sampling_params = seq_group
395 - max_best_of = max(max_best_of, sampling_params.best_of)
396 - random_samples = torch.multinomial(probs,
397 - num_samples=max_best_of,
398 - replacement=True).cpu()
279 + random_samples = random_samples.cpu()
280 sample_idx = 0
281 results = []
282 for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
@@ -403,8 +284,6 @@ def _random_sample(
284 num_parent_seqs = len(seq_ids)
285 if is_prompt:
286 # Prompt phase.
406 - assert num_parent_seqs == 1, (
407 - "Prompt input should have only one seq.")
287 parent_ids = [0] * sampling_params.best_of
288 next_token_ids = random_samples[
289 sample_idx, :sampling_params.best_of].tolist()
@@ -415,7 +294,6 @@ def _random_sample(
294 num_parent_seqs, 0].tolist()
295 results.append((next_token_ids, parent_ids))
296 sample_idx += num_parent_seqs
418 - assert sample_idx == probs.size(0)
297 return results
298
299
@@ -472,6 +350,28 @@ def _beam_search_sample(
350 return results
351
352
353 +# torch.multinomial forces a GPU<->CPU sync.
354 +# Therefore, we use an optimized implementation instead.
355 +# Note that we always sample with replacement.
356 +# probs will be modified in place, but this is fine, as we pass
357 +# in a copy already.
358 +def _multinomial(
359 + probs: torch.Tensor,
360 + num_samples: int,
361 +):
362 + if num_samples > 1:
363 + # This is equivalent to torch.repeat_interleaved (which also
364 + # forces a GPU<->CPU sync).
365 + # This allows us to do sampling with replacement by creating
366 + # num_samples copies of each row in the tensor, and then
367 + # batch sampling the resulting tensor.
368 + probs = probs[:, None, :].expand(probs.shape[0], num_samples,
369 + probs.shape[1]).contiguous().view(
370 + -1, probs.shape[1])
371 + q = torch.empty_like(probs).exponential_(1)
372 + return probs.div_(q).argmax(dim=1).view(-1, num_samples)
373 +
374 +
375 def _sample(
376 probs: torch.Tensor,
377 logprobs: torch.Tensor,
@@ -485,28 +385,51 @@ def _sample(
385 categorized_seq_group_ids[sampling_type].append(i)
386
387 sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
388 + sample_metadata = {}
389 +
390 + # Counterintiutively, having two loops here is actually faster.
391 + # The first loop can run without waiting on GPU<->CPU sync.
392 for sampling_type in SamplingType:
489 - seq_group_ids = categorized_seq_group_ids[sampling_type]
490 - seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
491 - is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
393 sample_indices = categorized_sample_indices[sampling_type]
394 num_tokens = len(sample_indices)
395 if num_tokens == 0:
396 continue
397 + seq_group_ids = categorized_seq_group_ids[sampling_type]
398 + seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
399 + is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
400 + sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
401 + is_prompts, sample_indices)
402 if sampling_type == SamplingType.GREEDY:
497 - category_logprobs = logprobs[sample_indices]
498 - sample_results = _greedy_sample(seq_groups, category_logprobs)
403 + greedy_samples = torch.argmax(logprobs[sample_indices], dim=-1)
404 + elif sampling_type == SamplingType.RANDOM:
405 + max_best_of = 1
406 + for seq_group, is_prompt in zip(seq_groups, is_prompts):
407 + if is_prompt:
408 + _, sampling_params = seq_group
409 + max_best_of = max(max_best_of, sampling_params.best_of)
410 + multinomial_samples = _multinomial(probs[sample_indices],
411 + max_best_of)
412 + elif sampling_type == SamplingType.BEAM:
413 + beam_search_logprobs = logprobs[sample_indices]
414 + else:
415 + raise ValueError(f"Unsupported sampling type: {sampling_type}")
416 +
417 + # GPU<->CPU sync happens in the loop below.
418 +
419 + for sampling_type in SamplingType:
420 + if sampling_type not in sample_metadata:
421 + continue
422 + seq_group_ids, seq_groups, is_prompts, sample_indices = sample_metadata[
423 + sampling_type]
424 + if sampling_type == SamplingType.GREEDY:
425 + sample_results = _greedy_sample(seq_groups, greedy_samples)
426 elif sampling_type == SamplingType.RANDOM:
500 - category_probs = probs[sample_indices]
427 sample_results = _random_sample(seq_groups, is_prompts,
502 - category_probs)
428 + multinomial_samples)
429 elif sampling_type == SamplingType.BEAM:
504 - category_logprobs = logprobs[sample_indices]
430 sample_results = _beam_search_sample(seq_groups, is_prompts,
431 sampling_metadata.seq_data,
507 - category_logprobs)
508 - else:
509 - raise ValueError(f"Unsupported sampling type: {sampling_type}")
432 + beam_search_logprobs)
433 sample_results_dict.update(zip(seq_group_ids, sample_results))
434
435 sample_results = [
@@ -557,7 +480,7 @@ def _get_logprobs(
480 batched_logprobs_query_result = logprobs[[
481 batched_logprobs_query_seq_indices,
482 batched_logprobs_query_token_indices
560 - ]].cpu()
483 + ]]
484
485 # Batched query for logprobs of topk tokens
486 if largest_num_logprobs > 0:
@@ -569,6 +492,8 @@ def _get_logprobs(
492 else:
493 top_logprobs, top_token_ids = None, None
494
495 + batched_logprobs_query_result = batched_logprobs_query_result.cpu()
496 +
497 # Gather results
498 result_prompt_logprobs: List[Optional[PromptLogprobs]] = []
499 result_sample_logprobs: List[SampleLogprobs] = []
vllm/model_executor/sampling_metadata.py
+187
@@ -1,9 +1,13 @@
1 +from dataclasses import dataclass
2 from typing import Dict, List, Tuple
3
4 import torch
5
6 from vllm.sampling_params import SamplingParams, SamplingType
7 from vllm.sequence import SequenceData
8 +from vllm.utils import in_wsl
9 +
10 +_SAMPLING_EPS = 1e-5
11
12
13 class SamplingMetadata:
@@ -41,3 +45,186 @@ class SamplingMetadata:
45 f"prompt_lens={self.prompt_lens}, "
46 f"selected_token_indices={self.selected_token_indices}, "
47 f"categorized_sample_indices={self.categorized_sample_indices})")
48 +
49 +
50 +@dataclass
51 +class SamplingTensors:
52 + """Tensors for sampling."""
53 +
54 + temperatures: torch.Tensor
55 + top_ps: torch.Tensor
56 + top_ks: torch.Tensor
57 + min_ps: torch.Tensor
58 + presence_penalties: torch.Tensor
59 + frequency_penalties: torch.Tensor
60 + repetition_penalties: torch.Tensor
61 + prompt_tokens: torch.Tensor
62 + output_tokens: torch.Tensor
63 +
64 + @classmethod
65 + def from_sampling_metadata(
66 + cls, sampling_metadata: "SamplingMetadata", vocab_size: int,
67 + device: torch.device,
68 + dtype: torch.dtype) -> Tuple["SamplingTensors", bool, bool, bool]:
69 + prompt_tokens: List[List[int]] = []
70 + output_tokens: List[List[int]] = []
71 + top_ks: List[int] = []
72 + temperatures: List[float] = []
73 + top_ps: List[float] = []
74 + min_ps: List[float] = []
75 + presence_penalties: List[float] = []
76 + frequency_penalties: List[float] = []
77 + repetition_penalties: List[float] = []
78 + do_penalties = False
79 + do_top_p_top_k = False
80 + do_min_p = False
81 + for i, seq_group in enumerate(sampling_metadata.seq_groups):
82 + seq_ids, sampling_params = seq_group
83 + temperature = sampling_params.temperature
84 + p = sampling_params.presence_penalty
85 + f = sampling_params.frequency_penalty
86 + r = sampling_params.repetition_penalty
87 + top_p = sampling_params.top_p
88 + min_p = sampling_params.min_p
89 + # k should not be greater than the vocab size.
90 + top_k = min(sampling_params.top_k, vocab_size)
91 + top_k = vocab_size if top_k == -1 else top_k
92 + if temperature < _SAMPLING_EPS:
93 + # NOTE: Zero temperature means deterministic sampling
94 + # (i.e., greedy sampling or beam search).
95 + # Set the temperature to 1 to avoid division by zero.
96 + temperature = 1.0
97 + if not do_top_p_top_k and (top_p < 1.0 - _SAMPLING_EPS
98 + or top_k != vocab_size):
99 + do_top_p_top_k = True
100 + if not do_min_p and min_p > _SAMPLING_EPS:
101 + do_min_p = True
102 + if not do_penalties and (abs(p) >= _SAMPLING_EPS
103 + or abs(f) >= _SAMPLING_EPS
104 + or abs(r - 1.0) >= _SAMPLING_EPS):
105 + do_penalties = True
106 + if (i < sampling_metadata.num_prompts
107 + and sampling_params.prompt_logprobs is not None):
108 + # For tokens in the prompt that we only need to get their logprobs
109 + prompt_len = sampling_metadata.prompt_lens[i]
110 + temperatures += [temperature] * (prompt_len - 1)
111 + top_ps += [top_p] * (prompt_len - 1)
112 + top_ks += [top_k] * (prompt_len - 1)
113 + min_ps += [min_p] * (prompt_len - 1)
114 + presence_penalties += [0] * (prompt_len - 1)
115 + frequency_penalties += [0] * (prompt_len - 1)
116 + repetition_penalties += [1] * (prompt_len - 1)
117 + prompt_tokens.extend([] for _ in range(prompt_len - 1))
118 + output_tokens.extend([] for _ in range(prompt_len - 1))
119 + for seq_id in seq_ids:
120 + seq_data = sampling_metadata.seq_data[seq_id]
121 + prompt_tokens.append(seq_data.prompt_token_ids)
122 + output_tokens.append(seq_data.output_token_ids)
123 + temperatures += [temperature] * len(seq_ids)
124 + top_ps += [top_p] * len(seq_ids)
125 + top_ks += [top_k] * len(seq_ids)
126 + min_ps += [min_p] * len(seq_ids)
127 + presence_penalties += [p] * len(seq_ids)
128 + frequency_penalties += [f] * len(seq_ids)
129 + repetition_penalties += [r] * len(seq_ids)
130 +
131 + sampling_tensors = SamplingTensors.from_lists(
132 + temperatures, top_ps, top_ks, min_ps, presence_penalties,
133 + frequency_penalties, repetition_penalties, prompt_tokens,
134 + output_tokens, vocab_size, device, dtype)
135 + return (sampling_tensors, do_penalties, do_top_p_top_k, do_min_p)
136 +
137 + @classmethod
138 + def from_lists(cls, temperatures: List[float], top_ps: List[float],
139 + top_ks: List[int], min_ps: List[float],
140 + presence_penalties: List[float],
141 + frequency_penalties: List[float],
142 + repetition_penalties: List[float],
143 + prompt_tokens: List[List[int]],
144 + output_tokens: List[List[int]], vocab_size: int,
145 + device: torch.device,
146 + dtype: torch.dtype) -> "SamplingTensors":
147 + # Note that the performance will be very bad without
148 + # pinned memory.
149 + pin_memory = not in_wsl()
150 + prompt_max_len = max(len(tokens) for tokens in prompt_tokens)
151 + prompt_padded_tokens = [
152 + tokens + [vocab_size] * (prompt_max_len - len(tokens))
153 + for tokens in prompt_tokens
154 + ]
155 + output_max_len = max(len(tokens) for tokens in output_tokens)
156 + output_padded_tokens = [
157 + tokens + [vocab_size] * (output_max_len - len(tokens))
158 + for tokens in output_tokens
159 + ]
160 +
161 + temperatures_t = torch.tensor(
162 + temperatures,
163 + device="cpu",
164 + dtype=dtype,
165 + pin_memory=pin_memory,
166 + )
167 + top_ps_t = torch.tensor(
168 + top_ps,
169 + device="cpu",
170 + dtype=dtype,
171 + pin_memory=pin_memory,
172 + )
173 + min_ps_t = torch.tensor(
174 + min_ps,
175 + device="cpu",
176 + dtype=dtype,
177 + pin_memory=pin_memory,
178 + )
179 + presence_penalties_t = torch.tensor(
180 + presence_penalties,
181 + device="cpu",
182 + dtype=dtype,
183 + pin_memory=pin_memory,
184 + )
185 + frequency_penalties_t = torch.tensor(
186 + frequency_penalties,
187 + device="cpu",
188 + dtype=dtype,
189 + pin_memory=pin_memory,
190 + )
191 + repetition_penalties_t = torch.tensor(
192 + repetition_penalties,
193 + device="cpu",
194 + dtype=dtype,
195 + pin_memory=pin_memory,
196 + )
197 + top_ks_t = torch.tensor(
198 + top_ks,
199 + device="cpu",
200 + dtype=torch.int,
201 + pin_memory=pin_memory,
202 + )
203 + prompt_tensor = torch.tensor(
204 + prompt_padded_tokens,
205 + device="cpu",
206 + dtype=torch.long,
207 + pin_memory=pin_memory,
208 + )
209 + output_tensor = torch.tensor(
210 + output_padded_tokens,
211 + device="cpu",
212 + dtype=torch.long,
213 + pin_memory=pin_memory,
214 + )
215 + # Because the memory is pinned, we can do non-blocking
216 + # transfer to device.
217 + return cls(
218 + temperatures=temperatures_t.to(device=device, non_blocking=True),
219 + top_ps=top_ps_t.to(device=device, non_blocking=True),
220 + top_ks=top_ks_t.to(device=device, non_blocking=True),
221 + min_ps=min_ps_t.to(device=device, non_blocking=True),
222 + presence_penalties=presence_penalties_t.to(device=device,
223 + non_blocking=True),
224 + frequency_penalties=frequency_penalties_t.to(device=device,
225 + non_blocking=True),
226 + repetition_penalties=repetition_penalties_t.to(device=device,
227 + non_blocking=True),
228 + prompt_tokens=prompt_tensor.to(device=device, non_blocking=True),
229 + output_tokens=output_tensor.to(device=device, non_blocking=True),
230 + )