A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Use FlashInfer Sampling Kernel for Top-P & Top-K Sampling (#11394)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 27, 2024 at 09:32 UTC
371d04d39bf056e4cc56100c83d4812b7cb230e4
6 files changed
+355
-190
tests/v1/sample/test_sampler.py
+22
-32
@@ -68,7 +68,7 @@ def _create_default_sampling_metadata(
68
no_top_p=True,
69
no_top_k=True,
70
generators={},
71
- max_num_logprobs=VOCAB_SIZE,
71
+ max_num_logprobs=0,
72
prompt_token_ids=_create_prompt_tokens_tensor(prompt_token_ids,
73
vocab_size, device),
74
output_token_ids=output_token_ids,
@@ -169,20 +169,14 @@ def test_sampler_min_tokens_penalty(device: str, batch_size: int):
169
sampling_metadata.min_tokens = min_tokens
170
sampling_metadata.stop_token_ids = stop_token_ids
171
sampler = Sampler()
172
- sampler_output = sampler(fake_logits, sampling_metadata)
172
+ logits = sampler.apply_penalties(fake_logits, sampling_metadata)
173
+ logits = logits.cpu()
174
for batch_idx in range(batch_size):
174
- for vocab in range(VOCAB_SIZE):
175
- # Verify that the logprobs for stop token ids is set
176
- # to -inf.
177
- logprob_index = torch.where(
178
- sampler_output.logprob_token_ids[batch_idx] ==
179
- vocab)[0].item()
180
- if vocab in stop_token_ids[batch_idx]:
181
- assert sampler_output.logprobs[batch_idx][
182
- logprob_index] == -float("inf")
175
+ for token_id in range(VOCAB_SIZE):
176
+ if token_id in stop_token_ids[batch_idx]:
177
+ assert logits[batch_idx][token_id] == -float("inf")
178
else:
184
- assert sampler_output.logprobs[batch_idx][
185
- logprob_index] != -float("inf")
179
+ assert logits[batch_idx][token_id] != -float("inf")
180
181
182
@pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -205,18 +199,14 @@ def test_sampler_presence_penalty(device: str, batch_size: int,
199
batch_size, presence_penalty, torch.device(device))
200
sampling_metadata.no_penalties = False
201
sampler = Sampler()
208
- sampler_output = sampler(fake_logits, sampling_metadata)
202
+ logits = sampler.apply_penalties(fake_logits, sampling_metadata)
203
+ logits = logits.cpu()
204
for batch_idx in range(batch_size):
210
- # The logprobs in the SamplerOutput are arranged in descending order.
211
- # Since all tokens initially have the same logprobs, the non-penalized
212
- # tokens will appear at the beginning, while the penalized tokens
213
- # will appear at the end of the list.
214
- penalized_token_id = sampler_output.logprob_token_ids[batch_idx][
215
- VOCAB_SIZE - 1]
216
- penalized_log_prod = sampler_output.logprobs[batch_idx][VOCAB_SIZE - 1]
217
- non_penalized_token_id = sampler_output.logprob_token_ids[batch_idx][0]
218
- non_penalized_log_prod = sampler_output.logprobs[batch_idx][0]
219
- assert non_penalized_log_prod > penalized_log_prod
205
+ # Since all tokens initially have the same logits, the non-penalized
206
+ # token ID will be the one with the highest logit value, while the
207
+ # penalized token ID will be the one with the lowest logit value.
208
+ non_penalized_token_id = logits[batch_idx].argmax().item()
209
+ penalized_token_id = logits[batch_idx].argmin().item()
210
if presence_penalty > 0:
211
# If `presence_penalty` is set to a value greater than 0, it
212
# indicates a preference for new tokens over those already
@@ -256,11 +246,11 @@ def test_sampler_frequency_penalty(device: str, batch_size: int,
246
sampling_metadata.output_token_ids = output_token_ids
247
sampling_metadata.no_penalties = False
248
sampler = Sampler()
259
- sampler_output = sampler(fake_logits, sampling_metadata)
249
+ logits = sampler.apply_penalties(fake_logits, sampling_metadata)
250
+ logits = logits.cpu()
251
for batch_idx in range(batch_size):
261
- logprobs_token_ids = sampler_output.logprob_token_ids[batch_idx]
262
- non_penalized_token_id = logprobs_token_ids[0]
263
- penalized_token_id = logprobs_token_ids[VOCAB_SIZE - 1]
252
+ non_penalized_token_id = logits[batch_idx].argmax().item()
253
+ penalized_token_id = logits[batch_idx].argmin().item()
254
distinct_sorted_token_ids_in_output = \
255
sorted_token_ids_in_output[batch_idx]
256
most_frequent_token_id = distinct_sorted_token_ids_in_output[
@@ -305,11 +295,11 @@ def test_sampler_repetition_penalty(device: str, batch_size: int,
295
batch_size, repetition_penalty, torch.device(device))
296
sampling_metadata.no_penalties = False
297
sampler = Sampler()
308
- sampler_output = sampler(fake_logits, sampling_metadata)
298
+ logits = sampler.apply_penalties(fake_logits, sampling_metadata)
299
+ logits = logits.cpu()
300
for batch_idx in range(batch_size):
310
- logprobs_token_ids = sampler_output.logprob_token_ids[batch_idx]
311
- non_penalized_token_id = logprobs_token_ids[0]
312
- penalized_token_id = logprobs_token_ids[VOCAB_SIZE - 1]
301
+ non_penalized_token_id = logits[batch_idx].argmax().item()
302
+ penalized_token_id = logits[batch_idx].argmin().item()
303
prompt_tokens = sampling_metadata.prompt_token_ids[
304
batch_idx][:].tolist()
305
output_tokens = sampling_metadata.output_token_ids[batch_idx]
vllm/envs.py
+3
-2
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
30
VLLM_LOGGING_CONFIG_PATH: Optional[str] = None
31
VLLM_TRACE_FUNCTION: int = 0
32
VLLM_ATTENTION_BACKEND: Optional[str] = None
33
- VLLM_USE_FLASHINFER_SAMPLER: bool = False
33
+ VLLM_USE_FLASHINFER_SAMPLER: Optional[bool] = None
34
VLLM_USE_FLASHINFER_REJECTION_SAMPLER: bool = False
35
VLLM_FLASHINFER_FORCE_TENSOR_CORES: bool = False
36
VLLM_PP_LAYER_PARTITION: Optional[str] = None
@@ -277,7 +277,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
277
278
# If set, vllm will use flashinfer sampler
279
"VLLM_USE_FLASHINFER_SAMPLER":
280
- lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_SAMPLER", "0"))),
280
+ lambda: bool(int(os.environ["VLLM_USE_FLASHINFER_SAMPLER"]))
281
+ if "VLLM_USE_FLASHINFER_SAMPLER" in os.environ else None,
282
283
# If set, vllm will force flashinfer to use tensor cores;
284
# otherwise will use heuristic based on model architecture.
vllm/v1/sample/ops/__init__.py
vllm/v1/sample/ops/penalties.py
new
+57
@@ -0,0 +1,57 @@
1
+from typing import List, Set, Tuple
2
+
3
+import torch
4
+
5
+from vllm.model_executor.layers.utils import (
6
+ apply_penalties as _apply_penalties)
7
+from vllm.utils import is_pin_memory_available, make_tensor_with_pad
8
+
9
+
10
+def apply_min_token_penalties(logits: torch.Tensor,
11
+ output_token_ids: List[List[int]],
12
+ stop_token_ids: List[Set[int]],
13
+ min_tokens: List[int]) -> None:
14
+ """
15
+ Applies minimum token penalty by setting the logits of the stop tokens
16
+ to -inf.
17
+ """
18
+ min_tokens_logits_to_penalize: List[Tuple[int, int]] = []
19
+ for index, min_token in enumerate(min_tokens):
20
+ if (len(output_token_ids[index]) < min_token):
21
+ for stop_token_id in stop_token_ids[index]:
22
+ min_tokens_logits_to_penalize.append((index, stop_token_id))
23
+ if min_tokens_logits_to_penalize:
24
+ logits[tuple(zip(*min_tokens_logits_to_penalize))] = -float("inf")
25
+
26
+
27
+def apply_penalties(logits: torch.Tensor, prompt_token_ids: torch.Tensor,
28
+ presence_penalties: torch.Tensor,
29
+ frequency_penalties: torch.Tensor,
30
+ repetition_penalties: torch.Tensor,
31
+ output_token_ids: List[List[int]]) -> torch.Tensor:
32
+ """
33
+ Applies presence, frequency and repetition penalties to the logits.
34
+ """
35
+ _, vocab_size = logits.shape
36
+ output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size,
37
+ logits.device)
38
+ return _apply_penalties(logits, prompt_token_ids, output_tokens_t,
39
+ presence_penalties, frequency_penalties,
40
+ repetition_penalties)
41
+
42
+
43
+def _convert_to_tensors(output_token_ids: List[List[int]], vocab_size: int,
44
+ device: torch.device) -> torch.Tensor:
45
+ """
46
+ Convert the different list data structures to tensors.
47
+ """
48
+ output_tokens_tensor = make_tensor_with_pad(
49
+ output_token_ids,
50
+ # Use the value of vocab_size as a pad since we don't have a
51
+ # token_id of this value.
52
+ pad=vocab_size,
53
+ device="cpu",
54
+ dtype=torch.int64,
55
+ pin_memory=is_pin_memory_available(),
56
+ )
57
+ return output_tokens_tensor.to(device, non_blocking=True)
vllm/v1/sample/ops/topk_topp_sampler.py
new
+201
@@ -0,0 +1,201 @@
1
+from typing import Dict
2
+
3
+import torch
4
+import torch.nn as nn
5
+
6
+from vllm import envs
7
+from vllm.logger import init_logger
8
+from vllm.platforms import current_platform
9
+
10
+logger = init_logger(__name__)
11
+
12
+try:
13
+ import flashinfer.sampling
14
+ is_flashinfer_available = True
15
+except ImportError:
16
+ is_flashinfer_available = False
17
+
18
+
19
+class TopKTopPSampler(nn.Module):
20
+
21
+ def __init__(self):
22
+ super().__init__()
23
+ if current_platform.is_cuda:
24
+ if is_flashinfer_available:
25
+ if envs.VLLM_USE_FLASHINFER_SAMPLER is not False:
26
+ # NOTE(woosuk): The V0 sampler doesn't use FlashInfer for
27
+ # sampling unless VLLM_USE_FLASHINFER_SAMPLER=1 (i.e., by
28
+ # default it is unused). For backward compatibility, we set
29
+ # `VLLM_USE_FLASHINFER_SAMPLER` as None by default and
30
+ # interpret it differently in V0 and V1 samplers: In V0,
31
+ # None means False, while in V1, None means True. This is
32
+ # why we use the condition
33
+ # `envs.VLLM_USE_FLASHINFER_SAMPLER is not False` here.
34
+ logger.info("Using FlashInfer for top-p & top-k sampling.")
35
+ self.forward = self.forward_cuda
36
+ else:
37
+ logger.warning(
38
+ "FlashInfer is available, but it is not enabled. "
39
+ "Falling back to the PyTorch-native implementation of "
40
+ "top-p & top-k sampling. For the best performance, "
41
+ "please set VLLM_USE_FLASHINFER_SAMPLER=1.")
42
+ self.forward = self.forward_native
43
+ else:
44
+ logger.warning(
45
+ "FlashInfer is not available. Falling back to the PyTorch-"
46
+ "native implementation of top-p & top-k sampling. For the "
47
+ "best performance, please install FalshInfer.")
48
+ self.forward = self.forward_native
49
+ else:
50
+ self.forward = self.forward_native
51
+
52
+ def forward_native(
53
+ self,
54
+ logits: torch.Tensor,
55
+ generators: Dict[int, torch.Generator],
56
+ no_top_k: bool,
57
+ k: torch.Tensor,
58
+ no_top_p: bool,
59
+ p: torch.Tensor,
60
+ ) -> torch.Tensor:
61
+ """PyTorch-native implementation of top-k and top-p sampling."""
62
+ logits = apply_top_k_top_p(logits, no_top_k, k, no_top_p, p)
63
+ probs = logits.softmax(dim=-1, dtype=torch.float32)
64
+ return random_sample(probs, generators)
65
+
66
+ def forward_cuda(
67
+ self,
68
+ logits: torch.Tensor,
69
+ generators: Dict[int, torch.Generator],
70
+ no_top_k: bool,
71
+ k: torch.Tensor,
72
+ no_top_p: bool,
73
+ p: torch.Tensor,
74
+ ) -> torch.Tensor:
75
+ """More optimized implementation for top-k and top-p sampling."""
76
+ probs = logits.softmax(dim=-1, dtype=torch.float32)
77
+ if no_top_k and no_top_p:
78
+ # We prefer `random_sample` over `flashinfer_sample` when sorting is
79
+ # not needed. This is because `random_sample` does not require
80
+ # CPU-GPU synchronization while `flashinfer_sample` does.
81
+ return random_sample(probs, generators)
82
+ return flashinfer_sample(probs, no_top_k, k, no_top_p, p, generators)
83
+
84
+
85
+def apply_top_k_top_p(
86
+ logits: torch.Tensor,
87
+ no_top_k: bool,
88
+ k: torch.Tensor,
89
+ no_top_p: bool,
90
+ p: torch.Tensor,
91
+) -> torch.Tensor:
92
+ """Apply top-k and top-p masks to the logits.
93
+
94
+ This function sorts the logits tensor, which can be slow for large batches.
95
+ """
96
+ if no_top_k and no_top_p:
97
+ return logits
98
+ logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
99
+
100
+ if not no_top_k:
101
+ # Apply top-k.
102
+ top_k_mask = logits_sort.size(1) - k.to(torch.long)
103
+ # Get all the top_k values.
104
+ top_k_mask = logits_sort.gather(1, top_k_mask.unsqueeze(dim=1))
105
+ top_k_mask = logits_sort < top_k_mask
106
+ logits_sort.masked_fill_(top_k_mask, -float("inf"))
107
+
108
+ if not no_top_p:
109
+ # Apply top-p.
110
+ probs_sort = logits_sort.softmax(dim=-1)
111
+ probs_sum = probs_sort.cumsum(dim=-1)
112
+ top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1)
113
+ # at least one
114
+ top_p_mask[:, -1] = False
115
+ logits_sort.masked_fill_(top_p_mask, -float("inf"))
116
+
117
+ # Re-sort the probabilities.
118
+ logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort)
119
+ return logits
120
+
121
+
122
+def random_sample(
123
+ probs: torch.Tensor,
124
+ generators: Dict[int, torch.Generator],
125
+) -> torch.Tensor:
126
+ """Randomly sample from the probabilities.
127
+
128
+ We use this function instead of torch.multinomial because torch.multinomial
129
+ causes CPU-GPU synchronization.
130
+ """
131
+ q = torch.empty_like(probs)
132
+ # NOTE(woosuk): To batch-process the requests without their own seeds,
133
+ # which is the common case, we first assume that every request does
134
+ # not have its own seed. Then, we overwrite the values for the requests
135
+ # that have their own seeds.
136
+ if len(generators) != probs.shape[0]:
137
+ q.exponential_()
138
+ if generators:
139
+ # TODO(woosuk): This can be slow because we handle each request
140
+ # one by one. Optimize this.
141
+ for i, generator in generators.items():
142
+ q[i].exponential_(generator=generator)
143
+ return probs.div_(q).argmax(dim=-1).view(-1)
144
+
145
+
146
+def flashinfer_sample(
147
+ probs: torch.Tensor,
148
+ no_top_k: bool,
149
+ k: torch.Tensor,
150
+ no_top_p: bool,
151
+ p: torch.Tensor,
152
+ generators: Dict[int, torch.Generator],
153
+) -> torch.Tensor:
154
+ """Sample from the probabilities using FlashInfer.
155
+
156
+ Statistically, this function is equivalent to the `random_sample` function.
157
+ However, this function is faster because it avoids sorting the logits tensor
158
+ via rejection sampling.
159
+
160
+ NOTE: The outputs of this function do not necessarily match the outputs of
161
+ the `random_sample` function. It only guarantees that the outputs are
162
+ statistically equivalent.
163
+
164
+ NOTE: This function includes CPU-GPU synchronization, while `random_sample`
165
+ does not. Call this function at the end of the forward pass to minimize
166
+ the synchronization overhead.
167
+ """
168
+ assert not (no_top_k and no_top_p)
169
+ max_top_k_round = 32
170
+ batch_size = probs.shape[0]
171
+ uniform_samples = torch.empty((max_top_k_round, batch_size),
172
+ device=probs.device)
173
+ if len(generators) != batch_size:
174
+ uniform_samples.uniform_()
175
+ if generators:
176
+ for i, generator in generators.items():
177
+ uniform_samples[:, i].uniform_(generator=generator)
178
+
179
+ if no_top_k:
180
+ # Top-p only.
181
+ next_token_ids, success = flashinfer.sampling.top_p_sampling_from_probs(
182
+ probs, uniform_samples, p, deterministic=True)
183
+ elif no_top_p:
184
+ # Top-k only.
185
+ next_token_ids, success = flashinfer.sampling.top_k_sampling_from_probs(
186
+ probs, uniform_samples, k, deterministic=True)
187
+ else:
188
+ # Both top-k and top-p.
189
+ next_token_ids, success = (
190
+ flashinfer.sampling.top_k_top_p_sampling_from_probs(
191
+ probs, uniform_samples, k, p, deterministic=True))
192
+
193
+ # NOTE: CPU-GPU synchronization happens here.
194
+ if not success.all():
195
+ if not no_top_k:
196
+ probs = flashinfer.sampling.top_k_renorm_prob(probs, k)
197
+ if not no_top_p:
198
+ probs = flashinfer.sampling.top_p_renorm_prob(probs, p)
199
+ next_token_ids = flashinfer.sampling.sampling_from_probs(
200
+ probs, uniform_samples[0], deterministic=True)
201
+ return next_token_ids.view(-1)
vllm/v1/sample/sampler.py
+72
-156
@@ -1,53 +1,55 @@
1
"""A layer that samples the next tokens from the model's outputs."""
2
-from typing import Dict, List, Set, Tuple
2
+from typing import Tuple
3
4
import torch
5
import torch.nn as nn
6
7
-from vllm.model_executor.layers.utils import apply_penalties
8
-from vllm.utils import is_pin_memory_available, make_tensor_with_pad
7
from vllm.v1.outputs import SamplerOutput
8
from vllm.v1.sample.metadata import SamplingMetadata
9
+from vllm.v1.sample.ops.penalties import (apply_min_token_penalties,
10
+ apply_penalties)
11
+from vllm.v1.sample.ops.topk_topp_sampler import TopKTopPSampler
12
13
_SAMPLING_EPS = 1e-5
14
15
16
class Sampler(nn.Module):
17
18
+ def __init__(self):
19
+ super().__init__()
20
+ self.topk_topp_sampler = TopKTopPSampler()
21
+
22
def forward(
23
self,
24
logits: torch.Tensor,
25
sampling_metadata: SamplingMetadata,
26
) -> SamplerOutput:
22
- _apply_min_token_penalties(logits, sampling_metadata.output_token_ids,
23
- sampling_metadata.stop_token_ids,
24
- sampling_metadata.min_tokens)
25
- if not sampling_metadata.no_penalties:
26
- assert sampling_metadata.prompt_token_ids is not None
27
- _apply_penalties(logits, sampling_metadata.prompt_token_ids,
28
- sampling_metadata.presence_penalties,
29
- sampling_metadata.frequency_penalties,
30
- sampling_metadata.repetition_penalties,
31
- sampling_metadata.output_token_ids)
32
- logits = self.apply_temperature(logits, sampling_metadata.temperature)
33
- logits = self.apply_top_k_top_p(logits, sampling_metadata)
34
- probs = self.get_probs(logits)
35
- sampled = self.sample(probs, sampling_metadata)
36
- # Use int32 to reduce the tensor size.
37
- sampled = sampled.to(torch.int32)
38
-
39
- if sampling_metadata.max_num_logprobs > 0:
40
- logprobs = self.get_logprobs(logits)
41
- # FIXME: Mask the sampled token_id, get topk logprobs,
42
- # and concatenate the topk with the sampled token_id.
43
- topk_logprobs, topk_indices = torch.topk(
44
- logprobs, sampling_metadata.max_num_logprobs, dim=-1)
45
- # Use int32 to reduce the tensor size.
46
- topk_indices = topk_indices.to(torch.int32)
27
+ needs_logprobs = sampling_metadata.max_num_logprobs > 0
28
+ if needs_logprobs:
29
+ # NOTE(woosuk): Use the original logits (before any penalties or
30
+ # temperature scaling) for the top-k logprobs.
31
+ # This is different from the V0 sampler, which uses the logits that
32
+ # is used for sampling (after penalties and temperature scaling).
33
+ # NOTE: We compute logprobs first because the below ops may
34
+ # modify the logits tensor in-place (and we don't want to clone
35
+ # the logits tensor for memory efficiency).
36
+ topk_logprobs, topk_indices = self.get_topk_logprobs(
37
+ logits, sampling_metadata)
38
else:
39
topk_logprobs = None
40
topk_indices = None
41
42
+ # Use float32 for the logits.
43
+ logits = logits.to(torch.float32)
44
+ # Apply penalties (e.g., min_tokens, freq_penalties).
45
+ logits = self.apply_penalties(logits, sampling_metadata)
46
+ # Apply temperature.
47
+ logits = self.apply_temperature(logits, sampling_metadata.temperature)
48
+ # Sample the next token.
49
+ sampled = self.sample(logits, sampling_metadata)
50
+ # Use int32 to reduce the tensor size.
51
+ sampled = sampled.to(torch.int32)
52
+
53
# NOTE: CPU-GPU synchronization happens here.
54
sampler_output = SamplerOutput(
55
sampled_token_ids=sampled.tolist(),
@@ -63,71 +65,37 @@ class Sampler(nn.Module):
65
logits: torch.Tensor,
66
temp: torch.Tensor,
67
) -> torch.Tensor:
66
- # Use float32 to apply temperature scaling.
67
- logits = logits.to(torch.float32)
68
# Avoid division by zero.
69
temp = torch.where(temp < _SAMPLING_EPS, 1.0, temp)
70
# Use in-place division to avoid creating a new tensor.
71
logits.div_(temp.unsqueeze(dim=1))
72
return logits
73
74
- def apply_top_k_top_p(
74
+ def greedy_sample(self, logits: torch.Tensor) -> torch.Tensor:
75
+ return logits.argmax(dim=-1).view(-1)
76
+
77
+ def sample(
78
self,
79
logits: torch.Tensor,
80
sampling_metadata: SamplingMetadata,
81
) -> torch.Tensor:
79
- return _apply_top_k_top_p(
82
+ assert not (sampling_metadata.all_greedy
83
+ and sampling_metadata.all_random)
84
+ if sampling_metadata.all_greedy:
85
+ return self.greedy_sample(logits)
86
+
87
+ random_sampled = self.topk_topp_sampler(
88
logits,
89
+ sampling_metadata.generators,
90
sampling_metadata.no_top_k,
91
sampling_metadata.top_k,
92
sampling_metadata.no_top_p,
93
sampling_metadata.top_p,
94
)
86
-
87
- def get_probs(self, logits: torch.Tensor) -> torch.Tensor:
88
- return torch.softmax(logits, dim=-1, dtype=torch.float32)
89
-
90
- def get_logprobs(self, logits: torch.Tensor) -> torch.Tensor:
91
- return torch.log_softmax(logits, dim=-1, dtype=torch.float32)
92
-
93
- def greedy_sample(self, probs: torch.Tensor) -> torch.Tensor:
94
- return probs.argmax(dim=-1).view(-1)
95
-
96
- def random_sample(
97
- self,
98
- probs: torch.Tensor,
99
- generators: Dict[int, torch.Generator],
100
- ) -> torch.Tensor:
101
- q = torch.empty_like(probs)
102
- # NOTE(woosuk): To batch-process the requests without their own seeds,
103
- # which is the common case, we first assume that every request does
104
- # not have its own seed. Then, we overwrite the values for the requests
105
- # that have their own seeds.
106
- if len(generators) != probs.shape[0]:
107
- # This might still be done here unnecessarily if there are greedies
108
- q.exponential_()
109
- if generators:
110
- # TODO(woosuk): This can be slow because we handle each request
111
- # one by one. Optimize this.
112
- for i, generator in generators.items():
113
- q[i].exponential_(generator=generator)
114
- return probs.div_(q).argmax(dim=-1).view(-1)
115
-
116
- def sample(
117
- self,
118
- probs: torch.Tensor,
119
- sampling_metadata: SamplingMetadata,
120
- ) -> torch.Tensor:
121
- assert not (sampling_metadata.all_greedy
122
- and sampling_metadata.all_random)
123
- if sampling_metadata.all_greedy:
124
- return self.greedy_sample(probs)
95
if sampling_metadata.all_random:
126
- return self.random_sample(probs, sampling_metadata.generators)
96
+ return random_sampled
97
128
- greedy_sampled = self.greedy_sample(probs)
129
- random_sampled = self.random_sample(probs,
130
- sampling_metadata.generators)
98
+ greedy_sampled = self.greedy_sample(logits)
99
sampled = torch.where(
100
sampling_metadata.temperature < _SAMPLING_EPS,
101
greedy_sampled,
@@ -135,86 +103,34 @@ class Sampler(nn.Module):
103
)
104
return sampled
105
106
+ def get_topk_logprobs(
107
+ self,
108
+ logits: torch.Tensor,
109
+ sampling_metadata: SamplingMetadata,
110
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
111
+ logprobs = logits.log_softmax(dim=-1, dtype=torch.float32)
112
+ # FIXME: Mask the sampled token_id, get topk logprobs,
113
+ # and concatenate the topk with the sampled token_id.
114
+ topk_logprobs, topk_indices = torch.topk(
115
+ logprobs, sampling_metadata.max_num_logprobs, dim=-1)
116
+ # Use int32 to reduce the tensor size.
117
+ topk_indices = topk_indices.to(torch.int32)
118
+ return topk_logprobs, topk_indices
119
139
-# TODO(woosuk): Optimize this with a custom kernel.
140
-def _apply_top_k_top_p(
141
- logits: torch.Tensor,
142
- no_top_k: bool,
143
- k: torch.Tensor,
144
- no_top_p: bool,
145
- p: torch.Tensor,
146
-) -> torch.Tensor:
147
- if no_top_k and no_top_p:
120
+ def apply_penalties(
121
+ self,
122
+ logits: torch.Tensor,
123
+ sampling_metadata: SamplingMetadata,
124
+ ) -> torch.Tensor:
125
+ apply_min_token_penalties(logits, sampling_metadata.output_token_ids,
126
+ sampling_metadata.stop_token_ids,
127
+ sampling_metadata.min_tokens)
128
+ if not sampling_metadata.no_penalties:
129
+ assert sampling_metadata.prompt_token_ids is not None
130
+ logits = apply_penalties(logits,
131
+ sampling_metadata.prompt_token_ids,
132
+ sampling_metadata.presence_penalties,
133
+ sampling_metadata.frequency_penalties,
134
+ sampling_metadata.repetition_penalties,
135
+ sampling_metadata.output_token_ids)
136
return logits
149
- logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
150
-
151
- if not no_top_k:
152
- # Apply top-k.
153
- top_k_mask = logits_sort.size(1) - k.to(torch.long)
154
- # Get all the top_k values.
155
- top_k_mask = logits_sort.gather(1, top_k_mask.unsqueeze(dim=1))
156
- top_k_mask = logits_sort < top_k_mask
157
- logits_sort.masked_fill_(top_k_mask, -float("inf"))
158
-
159
- if not no_top_p:
160
- # Apply top-p.
161
- probs_sort = logits_sort.softmax(dim=-1)
162
- probs_sum = probs_sort.cumsum(dim=-1)
163
- top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1)
164
- # at least one
165
- top_p_mask[:, -1] = False
166
- logits_sort.masked_fill_(top_p_mask, -float("inf"))
167
-
168
- # Re-sort the probabilities.
169
- logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort)
170
- return logits
171
-
172
-
173
-def _apply_min_token_penalties(logits: torch.Tensor,
174
- output_token_ids: List[List[int]],
175
- stop_token_ids: List[Set[int]],
176
- min_tokens: List[int]):
177
- """
178
- Applies minimum token penalty by setting the logits of the stop tokens
179
- to -inf.
180
- """
181
- min_tokens_logits_to_penalize: List[Tuple[int, int]] = []
182
- for index, min_token in enumerate(min_tokens):
183
- if (len(output_token_ids[index]) < min_token):
184
- for stop_token_id in stop_token_ids[index]:
185
- min_tokens_logits_to_penalize.append((index, stop_token_id))
186
- if min_tokens_logits_to_penalize:
187
- logits[tuple(zip(*min_tokens_logits_to_penalize))] = -float("inf")
188
-
189
-
190
-def _apply_penalties(logits: torch.Tensor, prompt_token_ids: torch.Tensor,
191
- presence_penalties: torch.Tensor,
192
- frequency_penalties: torch.Tensor,
193
- repetition_penalties: torch.Tensor,
194
- output_token_ids: List[List[int]]):
195
- """
196
- Applies presence, frequency and repetition penalties to the logits.
197
- """
198
- _, vocab_size = logits.shape
199
- output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size,
200
- logits.device)
201
- return apply_penalties(logits, prompt_token_ids, output_tokens_t,
202
- presence_penalties, frequency_penalties,
203
- repetition_penalties)
204
-
205
-
206
-def _convert_to_tensors(output_token_ids: List[List[int]], vocab_size: int,
207
- device: torch.device) -> torch.Tensor:
208
- """
209
- Convert the different list data structures to tensors.
210
- """
211
- output_tokens_tensor = make_tensor_with_pad(
212
- output_token_ids,
213
- # Use the value of vocab_size as a pad since we don't have a
214
- # token_id of this value.
215
- pad=vocab_size,
216
- device="cpu",
217
- dtype=torch.int64,
218
- pin_memory=is_pin_memory_available(),
219
- )
220
- return output_tokens_tensor.to(device, non_blocking=True)