@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 371d04d39b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Use FlashInfer Sampling Kernel for Top-P & Top-K Sampling (#11394)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 27, 2024 at 09:32 UTC 371d04d39bf056e4cc56100c83d4812b7cb230e4
6 files changed +355 -190
tests/v1/sample/test_sampler.py
+22 -32
@@ -68,7 +68,7 @@ def _create_default_sampling_metadata(
68 no_top_p=True,
69 no_top_k=True,
70 generators={},
71 - max_num_logprobs=VOCAB_SIZE,
71 + max_num_logprobs=0,
72 prompt_token_ids=_create_prompt_tokens_tensor(prompt_token_ids,
73 vocab_size, device),
74 output_token_ids=output_token_ids,
@@ -169,20 +169,14 @@ def test_sampler_min_tokens_penalty(device: str, batch_size: int):
169 sampling_metadata.min_tokens = min_tokens
170 sampling_metadata.stop_token_ids = stop_token_ids
171 sampler = Sampler()
172 - sampler_output = sampler(fake_logits, sampling_metadata)
172 + logits = sampler.apply_penalties(fake_logits, sampling_metadata)
173 + logits = logits.cpu()
174 for batch_idx in range(batch_size):
174 - for vocab in range(VOCAB_SIZE):
175 - # Verify that the logprobs for stop token ids is set
176 - # to -inf.
177 - logprob_index = torch.where(
178 - sampler_output.logprob_token_ids[batch_idx] ==
179 - vocab)[0].item()
180 - if vocab in stop_token_ids[batch_idx]:
181 - assert sampler_output.logprobs[batch_idx][
182 - logprob_index] == -float("inf")
175 + for token_id in range(VOCAB_SIZE):
176 + if token_id in stop_token_ids[batch_idx]:
177 + assert logits[batch_idx][token_id] == -float("inf")
178 else:
184 - assert sampler_output.logprobs[batch_idx][
185 - logprob_index] != -float("inf")
179 + assert logits[batch_idx][token_id] != -float("inf")
180
181
182 @pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -205,18 +199,14 @@ def test_sampler_presence_penalty(device: str, batch_size: int,
199 batch_size, presence_penalty, torch.device(device))
200 sampling_metadata.no_penalties = False
201 sampler = Sampler()
208 - sampler_output = sampler(fake_logits, sampling_metadata)
202 + logits = sampler.apply_penalties(fake_logits, sampling_metadata)
203 + logits = logits.cpu()
204 for batch_idx in range(batch_size):
210 - # The logprobs in the SamplerOutput are arranged in descending order.
211 - # Since all tokens initially have the same logprobs, the non-penalized
212 - # tokens will appear at the beginning, while the penalized tokens
213 - # will appear at the end of the list.
214 - penalized_token_id = sampler_output.logprob_token_ids[batch_idx][
215 - VOCAB_SIZE - 1]
216 - penalized_log_prod = sampler_output.logprobs[batch_idx][VOCAB_SIZE - 1]
217 - non_penalized_token_id = sampler_output.logprob_token_ids[batch_idx][0]
218 - non_penalized_log_prod = sampler_output.logprobs[batch_idx][0]
219 - assert non_penalized_log_prod > penalized_log_prod
205 + # Since all tokens initially have the same logits, the non-penalized
206 + # token ID will be the one with the highest logit value, while the
207 + # penalized token ID will be the one with the lowest logit value.
208 + non_penalized_token_id = logits[batch_idx].argmax().item()
209 + penalized_token_id = logits[batch_idx].argmin().item()
210 if presence_penalty > 0:
211 # If `presence_penalty` is set to a value greater than 0, it
212 # indicates a preference for new tokens over those already
@@ -256,11 +246,11 @@ def test_sampler_frequency_penalty(device: str, batch_size: int,
246 sampling_metadata.output_token_ids = output_token_ids
247 sampling_metadata.no_penalties = False
248 sampler = Sampler()
259 - sampler_output = sampler(fake_logits, sampling_metadata)
249 + logits = sampler.apply_penalties(fake_logits, sampling_metadata)
250 + logits = logits.cpu()
251 for batch_idx in range(batch_size):
261 - logprobs_token_ids = sampler_output.logprob_token_ids[batch_idx]
262 - non_penalized_token_id = logprobs_token_ids[0]
263 - penalized_token_id = logprobs_token_ids[VOCAB_SIZE - 1]
252 + non_penalized_token_id = logits[batch_idx].argmax().item()
253 + penalized_token_id = logits[batch_idx].argmin().item()
254 distinct_sorted_token_ids_in_output = \
255 sorted_token_ids_in_output[batch_idx]
256 most_frequent_token_id = distinct_sorted_token_ids_in_output[
@@ -305,11 +295,11 @@ def test_sampler_repetition_penalty(device: str, batch_size: int,
295 batch_size, repetition_penalty, torch.device(device))
296 sampling_metadata.no_penalties = False
297 sampler = Sampler()
308 - sampler_output = sampler(fake_logits, sampling_metadata)
298 + logits = sampler.apply_penalties(fake_logits, sampling_metadata)
299 + logits = logits.cpu()
300 for batch_idx in range(batch_size):
310 - logprobs_token_ids = sampler_output.logprob_token_ids[batch_idx]
311 - non_penalized_token_id = logprobs_token_ids[0]
312 - penalized_token_id = logprobs_token_ids[VOCAB_SIZE - 1]
301 + non_penalized_token_id = logits[batch_idx].argmax().item()
302 + penalized_token_id = logits[batch_idx].argmin().item()
303 prompt_tokens = sampling_metadata.prompt_token_ids[
304 batch_idx][:].tolist()
305 output_tokens = sampling_metadata.output_token_ids[batch_idx]
vllm/envs.py
+3 -2
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
30 VLLM_LOGGING_CONFIG_PATH: Optional[str] = None
31 VLLM_TRACE_FUNCTION: int = 0
32 VLLM_ATTENTION_BACKEND: Optional[str] = None
33 - VLLM_USE_FLASHINFER_SAMPLER: bool = False
33 + VLLM_USE_FLASHINFER_SAMPLER: Optional[bool] = None
34 VLLM_USE_FLASHINFER_REJECTION_SAMPLER: bool = False
35 VLLM_FLASHINFER_FORCE_TENSOR_CORES: bool = False
36 VLLM_PP_LAYER_PARTITION: Optional[str] = None
@@ -277,7 +277,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
277
278 # If set, vllm will use flashinfer sampler
279 "VLLM_USE_FLASHINFER_SAMPLER":
280 - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_SAMPLER", "0"))),
280 + lambda: bool(int(os.environ["VLLM_USE_FLASHINFER_SAMPLER"]))
281 + if "VLLM_USE_FLASHINFER_SAMPLER" in os.environ else None,
282
283 # If set, vllm will force flashinfer to use tensor cores;
284 # otherwise will use heuristic based on model architecture.
vllm/v1/sample/ops/__init__.py
vllm/v1/sample/ops/penalties.py new
+57
@@ -0,0 +1,57 @@
1 +from typing import List, Set, Tuple
2 +
3 +import torch
4 +
5 +from vllm.model_executor.layers.utils import (
6 + apply_penalties as _apply_penalties)
7 +from vllm.utils import is_pin_memory_available, make_tensor_with_pad
8 +
9 +
10 +def apply_min_token_penalties(logits: torch.Tensor,
11 + output_token_ids: List[List[int]],
12 + stop_token_ids: List[Set[int]],
13 + min_tokens: List[int]) -> None:
14 + """
15 + Applies minimum token penalty by setting the logits of the stop tokens
16 + to -inf.
17 + """
18 + min_tokens_logits_to_penalize: List[Tuple[int, int]] = []
19 + for index, min_token in enumerate(min_tokens):
20 + if (len(output_token_ids[index]) < min_token):
21 + for stop_token_id in stop_token_ids[index]:
22 + min_tokens_logits_to_penalize.append((index, stop_token_id))
23 + if min_tokens_logits_to_penalize:
24 + logits[tuple(zip(*min_tokens_logits_to_penalize))] = -float("inf")
25 +
26 +
27 +def apply_penalties(logits: torch.Tensor, prompt_token_ids: torch.Tensor,
28 + presence_penalties: torch.Tensor,
29 + frequency_penalties: torch.Tensor,
30 + repetition_penalties: torch.Tensor,
31 + output_token_ids: List[List[int]]) -> torch.Tensor:
32 + """
33 + Applies presence, frequency and repetition penalties to the logits.
34 + """
35 + _, vocab_size = logits.shape
36 + output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size,
37 + logits.device)
38 + return _apply_penalties(logits, prompt_token_ids, output_tokens_t,
39 + presence_penalties, frequency_penalties,
40 + repetition_penalties)
41 +
42 +
43 +def _convert_to_tensors(output_token_ids: List[List[int]], vocab_size: int,
44 + device: torch.device) -> torch.Tensor:
45 + """
46 + Convert the different list data structures to tensors.
47 + """
48 + output_tokens_tensor = make_tensor_with_pad(
49 + output_token_ids,
50 + # Use the value of vocab_size as a pad since we don't have a
51 + # token_id of this value.
52 + pad=vocab_size,
53 + device="cpu",
54 + dtype=torch.int64,
55 + pin_memory=is_pin_memory_available(),
56 + )
57 + return output_tokens_tensor.to(device, non_blocking=True)
vllm/v1/sample/ops/topk_topp_sampler.py new
+201
@@ -0,0 +1,201 @@
1 +from typing import Dict
2 +
3 +import torch
4 +import torch.nn as nn
5 +
6 +from vllm import envs
7 +from vllm.logger import init_logger
8 +from vllm.platforms import current_platform
9 +
10 +logger = init_logger(__name__)
11 +
12 +try:
13 + import flashinfer.sampling
14 + is_flashinfer_available = True
15 +except ImportError:
16 + is_flashinfer_available = False
17 +
18 +
19 +class TopKTopPSampler(nn.Module):
20 +
21 + def __init__(self):
22 + super().__init__()
23 + if current_platform.is_cuda:
24 + if is_flashinfer_available:
25 + if envs.VLLM_USE_FLASHINFER_SAMPLER is not False:
26 + # NOTE(woosuk): The V0 sampler doesn't use FlashInfer for
27 + # sampling unless VLLM_USE_FLASHINFER_SAMPLER=1 (i.e., by
28 + # default it is unused). For backward compatibility, we set
29 + # `VLLM_USE_FLASHINFER_SAMPLER` as None by default and
30 + # interpret it differently in V0 and V1 samplers: In V0,
31 + # None means False, while in V1, None means True. This is
32 + # why we use the condition
33 + # `envs.VLLM_USE_FLASHINFER_SAMPLER is not False` here.
34 + logger.info("Using FlashInfer for top-p & top-k sampling.")
35 + self.forward = self.forward_cuda
36 + else:
37 + logger.warning(
38 + "FlashInfer is available, but it is not enabled. "
39 + "Falling back to the PyTorch-native implementation of "
40 + "top-p & top-k sampling. For the best performance, "
41 + "please set VLLM_USE_FLASHINFER_SAMPLER=1.")
42 + self.forward = self.forward_native
43 + else:
44 + logger.warning(
45 + "FlashInfer is not available. Falling back to the PyTorch-"
46 + "native implementation of top-p & top-k sampling. For the "
47 + "best performance, please install FalshInfer.")
48 + self.forward = self.forward_native
49 + else:
50 + self.forward = self.forward_native
51 +
52 + def forward_native(
53 + self,
54 + logits: torch.Tensor,
55 + generators: Dict[int, torch.Generator],
56 + no_top_k: bool,
57 + k: torch.Tensor,
58 + no_top_p: bool,
59 + p: torch.Tensor,
60 + ) -> torch.Tensor:
61 + """PyTorch-native implementation of top-k and top-p sampling."""
62 + logits = apply_top_k_top_p(logits, no_top_k, k, no_top_p, p)
63 + probs = logits.softmax(dim=-1, dtype=torch.float32)
64 + return random_sample(probs, generators)
65 +
66 + def forward_cuda(
67 + self,
68 + logits: torch.Tensor,
69 + generators: Dict[int, torch.Generator],
70 + no_top_k: bool,
71 + k: torch.Tensor,
72 + no_top_p: bool,
73 + p: torch.Tensor,
74 + ) -> torch.Tensor:
75 + """More optimized implementation for top-k and top-p sampling."""
76 + probs = logits.softmax(dim=-1, dtype=torch.float32)
77 + if no_top_k and no_top_p:
78 + # We prefer `random_sample` over `flashinfer_sample` when sorting is
79 + # not needed. This is because `random_sample` does not require
80 + # CPU-GPU synchronization while `flashinfer_sample` does.
81 + return random_sample(probs, generators)
82 + return flashinfer_sample(probs, no_top_k, k, no_top_p, p, generators)
83 +
84 +
85 +def apply_top_k_top_p(
86 + logits: torch.Tensor,
87 + no_top_k: bool,
88 + k: torch.Tensor,
89 + no_top_p: bool,
90 + p: torch.Tensor,
91 +) -> torch.Tensor:
92 + """Apply top-k and top-p masks to the logits.
93 +
94 + This function sorts the logits tensor, which can be slow for large batches.
95 + """
96 + if no_top_k and no_top_p:
97 + return logits
98 + logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
99 +
100 + if not no_top_k:
101 + # Apply top-k.
102 + top_k_mask = logits_sort.size(1) - k.to(torch.long)
103 + # Get all the top_k values.
104 + top_k_mask = logits_sort.gather(1, top_k_mask.unsqueeze(dim=1))
105 + top_k_mask = logits_sort < top_k_mask
106 + logits_sort.masked_fill_(top_k_mask, -float("inf"))
107 +
108 + if not no_top_p:
109 + # Apply top-p.
110 + probs_sort = logits_sort.softmax(dim=-1)
111 + probs_sum = probs_sort.cumsum(dim=-1)
112 + top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1)
113 + # at least one
114 + top_p_mask[:, -1] = False
115 + logits_sort.masked_fill_(top_p_mask, -float("inf"))
116 +
117 + # Re-sort the probabilities.
118 + logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort)
119 + return logits
120 +
121 +
122 +def random_sample(
123 + probs: torch.Tensor,
124 + generators: Dict[int, torch.Generator],
125 +) -> torch.Tensor:
126 + """Randomly sample from the probabilities.
127 +
128 + We use this function instead of torch.multinomial because torch.multinomial
129 + causes CPU-GPU synchronization.
130 + """
131 + q = torch.empty_like(probs)
132 + # NOTE(woosuk): To batch-process the requests without their own seeds,
133 + # which is the common case, we first assume that every request does
134 + # not have its own seed. Then, we overwrite the values for the requests
135 + # that have their own seeds.
136 + if len(generators) != probs.shape[0]:
137 + q.exponential_()
138 + if generators:
139 + # TODO(woosuk): This can be slow because we handle each request
140 + # one by one. Optimize this.
141 + for i, generator in generators.items():
142 + q[i].exponential_(generator=generator)
143 + return probs.div_(q).argmax(dim=-1).view(-1)
144 +
145 +
146 +def flashinfer_sample(
147 + probs: torch.Tensor,
148 + no_top_k: bool,
149 + k: torch.Tensor,
150 + no_top_p: bool,
151 + p: torch.Tensor,
152 + generators: Dict[int, torch.Generator],
153 +) -> torch.Tensor:
154 + """Sample from the probabilities using FlashInfer.
155 +
156 + Statistically, this function is equivalent to the `random_sample` function.
157 + However, this function is faster because it avoids sorting the logits tensor
158 + via rejection sampling.
159 +
160 + NOTE: The outputs of this function do not necessarily match the outputs of
161 + the `random_sample` function. It only guarantees that the outputs are
162 + statistically equivalent.
163 +
164 + NOTE: This function includes CPU-GPU synchronization, while `random_sample`
165 + does not. Call this function at the end of the forward pass to minimize
166 + the synchronization overhead.
167 + """
168 + assert not (no_top_k and no_top_p)
169 + max_top_k_round = 32
170 + batch_size = probs.shape[0]
171 + uniform_samples = torch.empty((max_top_k_round, batch_size),
172 + device=probs.device)
173 + if len(generators) != batch_size:
174 + uniform_samples.uniform_()
175 + if generators:
176 + for i, generator in generators.items():
177 + uniform_samples[:, i].uniform_(generator=generator)
178 +
179 + if no_top_k:
180 + # Top-p only.
181 + next_token_ids, success = flashinfer.sampling.top_p_sampling_from_probs(
182 + probs, uniform_samples, p, deterministic=True)
183 + elif no_top_p:
184 + # Top-k only.
185 + next_token_ids, success = flashinfer.sampling.top_k_sampling_from_probs(
186 + probs, uniform_samples, k, deterministic=True)
187 + else:
188 + # Both top-k and top-p.
189 + next_token_ids, success = (
190 + flashinfer.sampling.top_k_top_p_sampling_from_probs(
191 + probs, uniform_samples, k, p, deterministic=True))
192 +
193 + # NOTE: CPU-GPU synchronization happens here.
194 + if not success.all():
195 + if not no_top_k:
196 + probs = flashinfer.sampling.top_k_renorm_prob(probs, k)
197 + if not no_top_p:
198 + probs = flashinfer.sampling.top_p_renorm_prob(probs, p)
199 + next_token_ids = flashinfer.sampling.sampling_from_probs(
200 + probs, uniform_samples[0], deterministic=True)
201 + return next_token_ids.view(-1)
vllm/v1/sample/sampler.py
+72 -156
@@ -1,53 +1,55 @@
1 """A layer that samples the next tokens from the model's outputs."""
2 -from typing import Dict, List, Set, Tuple
2 +from typing import Tuple
3
4 import torch
5 import torch.nn as nn
6
7 -from vllm.model_executor.layers.utils import apply_penalties
8 -from vllm.utils import is_pin_memory_available, make_tensor_with_pad
7 from vllm.v1.outputs import SamplerOutput
8 from vllm.v1.sample.metadata import SamplingMetadata
9 +from vllm.v1.sample.ops.penalties import (apply_min_token_penalties,
10 + apply_penalties)
11 +from vllm.v1.sample.ops.topk_topp_sampler import TopKTopPSampler
12
13 _SAMPLING_EPS = 1e-5
14
15
16 class Sampler(nn.Module):
17
18 + def __init__(self):
19 + super().__init__()
20 + self.topk_topp_sampler = TopKTopPSampler()
21 +
22 def forward(
23 self,
24 logits: torch.Tensor,
25 sampling_metadata: SamplingMetadata,
26 ) -> SamplerOutput:
22 - _apply_min_token_penalties(logits, sampling_metadata.output_token_ids,
23 - sampling_metadata.stop_token_ids,
24 - sampling_metadata.min_tokens)
25 - if not sampling_metadata.no_penalties:
26 - assert sampling_metadata.prompt_token_ids is not None
27 - _apply_penalties(logits, sampling_metadata.prompt_token_ids,
28 - sampling_metadata.presence_penalties,
29 - sampling_metadata.frequency_penalties,
30 - sampling_metadata.repetition_penalties,
31 - sampling_metadata.output_token_ids)
32 - logits = self.apply_temperature(logits, sampling_metadata.temperature)
33 - logits = self.apply_top_k_top_p(logits, sampling_metadata)
34 - probs = self.get_probs(logits)
35 - sampled = self.sample(probs, sampling_metadata)
36 - # Use int32 to reduce the tensor size.
37 - sampled = sampled.to(torch.int32)
38 -
39 - if sampling_metadata.max_num_logprobs > 0:
40 - logprobs = self.get_logprobs(logits)
41 - # FIXME: Mask the sampled token_id, get topk logprobs,
42 - # and concatenate the topk with the sampled token_id.
43 - topk_logprobs, topk_indices = torch.topk(
44 - logprobs, sampling_metadata.max_num_logprobs, dim=-1)
45 - # Use int32 to reduce the tensor size.
46 - topk_indices = topk_indices.to(torch.int32)
27 + needs_logprobs = sampling_metadata.max_num_logprobs > 0
28 + if needs_logprobs:
29 + # NOTE(woosuk): Use the original logits (before any penalties or
30 + # temperature scaling) for the top-k logprobs.
31 + # This is different from the V0 sampler, which uses the logits that
32 + # is used for sampling (after penalties and temperature scaling).
33 + # NOTE: We compute logprobs first because the below ops may
34 + # modify the logits tensor in-place (and we don't want to clone
35 + # the logits tensor for memory efficiency).
36 + topk_logprobs, topk_indices = self.get_topk_logprobs(
37 + logits, sampling_metadata)
38 else:
39 topk_logprobs = None
40 topk_indices = None
41
42 + # Use float32 for the logits.
43 + logits = logits.to(torch.float32)
44 + # Apply penalties (e.g., min_tokens, freq_penalties).
45 + logits = self.apply_penalties(logits, sampling_metadata)
46 + # Apply temperature.
47 + logits = self.apply_temperature(logits, sampling_metadata.temperature)
48 + # Sample the next token.
49 + sampled = self.sample(logits, sampling_metadata)
50 + # Use int32 to reduce the tensor size.
51 + sampled = sampled.to(torch.int32)
52 +
53 # NOTE: CPU-GPU synchronization happens here.
54 sampler_output = SamplerOutput(
55 sampled_token_ids=sampled.tolist(),
@@ -63,71 +65,37 @@ class Sampler(nn.Module):
65 logits: torch.Tensor,
66 temp: torch.Tensor,
67 ) -> torch.Tensor:
66 - # Use float32 to apply temperature scaling.
67 - logits = logits.to(torch.float32)
68 # Avoid division by zero.
69 temp = torch.where(temp < _SAMPLING_EPS, 1.0, temp)
70 # Use in-place division to avoid creating a new tensor.
71 logits.div_(temp.unsqueeze(dim=1))
72 return logits
73
74 - def apply_top_k_top_p(
74 + def greedy_sample(self, logits: torch.Tensor) -> torch.Tensor:
75 + return logits.argmax(dim=-1).view(-1)
76 +
77 + def sample(
78 self,
79 logits: torch.Tensor,
80 sampling_metadata: SamplingMetadata,
81 ) -> torch.Tensor:
79 - return _apply_top_k_top_p(
82 + assert not (sampling_metadata.all_greedy
83 + and sampling_metadata.all_random)
84 + if sampling_metadata.all_greedy:
85 + return self.greedy_sample(logits)
86 +
87 + random_sampled = self.topk_topp_sampler(
88 logits,
89 + sampling_metadata.generators,
90 sampling_metadata.no_top_k,
91 sampling_metadata.top_k,
92 sampling_metadata.no_top_p,
93 sampling_metadata.top_p,
94 )
86 -
87 - def get_probs(self, logits: torch.Tensor) -> torch.Tensor:
88 - return torch.softmax(logits, dim=-1, dtype=torch.float32)
89 -
90 - def get_logprobs(self, logits: torch.Tensor) -> torch.Tensor:
91 - return torch.log_softmax(logits, dim=-1, dtype=torch.float32)
92 -
93 - def greedy_sample(self, probs: torch.Tensor) -> torch.Tensor:
94 - return probs.argmax(dim=-1).view(-1)
95 -
96 - def random_sample(
97 - self,
98 - probs: torch.Tensor,
99 - generators: Dict[int, torch.Generator],
100 - ) -> torch.Tensor:
101 - q = torch.empty_like(probs)
102 - # NOTE(woosuk): To batch-process the requests without their own seeds,
103 - # which is the common case, we first assume that every request does
104 - # not have its own seed. Then, we overwrite the values for the requests
105 - # that have their own seeds.
106 - if len(generators) != probs.shape[0]:
107 - # This might still be done here unnecessarily if there are greedies
108 - q.exponential_()
109 - if generators:
110 - # TODO(woosuk): This can be slow because we handle each request
111 - # one by one. Optimize this.
112 - for i, generator in generators.items():
113 - q[i].exponential_(generator=generator)
114 - return probs.div_(q).argmax(dim=-1).view(-1)
115 -
116 - def sample(
117 - self,
118 - probs: torch.Tensor,
119 - sampling_metadata: SamplingMetadata,
120 - ) -> torch.Tensor:
121 - assert not (sampling_metadata.all_greedy
122 - and sampling_metadata.all_random)
123 - if sampling_metadata.all_greedy:
124 - return self.greedy_sample(probs)
95 if sampling_metadata.all_random:
126 - return self.random_sample(probs, sampling_metadata.generators)
96 + return random_sampled
97
128 - greedy_sampled = self.greedy_sample(probs)
129 - random_sampled = self.random_sample(probs,
130 - sampling_metadata.generators)
98 + greedy_sampled = self.greedy_sample(logits)
99 sampled = torch.where(
100 sampling_metadata.temperature < _SAMPLING_EPS,
101 greedy_sampled,
@@ -135,86 +103,34 @@ class Sampler(nn.Module):
103 )
104 return sampled
105
106 + def get_topk_logprobs(
107 + self,
108 + logits: torch.Tensor,
109 + sampling_metadata: SamplingMetadata,
110 + ) -> Tuple[torch.Tensor, torch.Tensor]:
111 + logprobs = logits.log_softmax(dim=-1, dtype=torch.float32)
112 + # FIXME: Mask the sampled token_id, get topk logprobs,
113 + # and concatenate the topk with the sampled token_id.
114 + topk_logprobs, topk_indices = torch.topk(
115 + logprobs, sampling_metadata.max_num_logprobs, dim=-1)
116 + # Use int32 to reduce the tensor size.
117 + topk_indices = topk_indices.to(torch.int32)
118 + return topk_logprobs, topk_indices
119
139 -# TODO(woosuk): Optimize this with a custom kernel.
140 -def _apply_top_k_top_p(
141 - logits: torch.Tensor,
142 - no_top_k: bool,
143 - k: torch.Tensor,
144 - no_top_p: bool,
145 - p: torch.Tensor,
146 -) -> torch.Tensor:
147 - if no_top_k and no_top_p:
120 + def apply_penalties(
121 + self,
122 + logits: torch.Tensor,
123 + sampling_metadata: SamplingMetadata,
124 + ) -> torch.Tensor:
125 + apply_min_token_penalties(logits, sampling_metadata.output_token_ids,
126 + sampling_metadata.stop_token_ids,
127 + sampling_metadata.min_tokens)
128 + if not sampling_metadata.no_penalties:
129 + assert sampling_metadata.prompt_token_ids is not None
130 + logits = apply_penalties(logits,
131 + sampling_metadata.prompt_token_ids,
132 + sampling_metadata.presence_penalties,
133 + sampling_metadata.frequency_penalties,
134 + sampling_metadata.repetition_penalties,
135 + sampling_metadata.output_token_ids)
136 return logits
149 - logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
150 -
151 - if not no_top_k:
152 - # Apply top-k.
153 - top_k_mask = logits_sort.size(1) - k.to(torch.long)
154 - # Get all the top_k values.
155 - top_k_mask = logits_sort.gather(1, top_k_mask.unsqueeze(dim=1))
156 - top_k_mask = logits_sort < top_k_mask
157 - logits_sort.masked_fill_(top_k_mask, -float("inf"))
158 -
159 - if not no_top_p:
160 - # Apply top-p.
161 - probs_sort = logits_sort.softmax(dim=-1)
162 - probs_sum = probs_sort.cumsum(dim=-1)
163 - top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1)
164 - # at least one
165 - top_p_mask[:, -1] = False
166 - logits_sort.masked_fill_(top_p_mask, -float("inf"))
167 -
168 - # Re-sort the probabilities.
169 - logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort)
170 - return logits
171 -
172 -
173 -def _apply_min_token_penalties(logits: torch.Tensor,
174 - output_token_ids: List[List[int]],
175 - stop_token_ids: List[Set[int]],
176 - min_tokens: List[int]):
177 - """
178 - Applies minimum token penalty by setting the logits of the stop tokens
179 - to -inf.
180 - """
181 - min_tokens_logits_to_penalize: List[Tuple[int, int]] = []
182 - for index, min_token in enumerate(min_tokens):
183 - if (len(output_token_ids[index]) < min_token):
184 - for stop_token_id in stop_token_ids[index]:
185 - min_tokens_logits_to_penalize.append((index, stop_token_id))
186 - if min_tokens_logits_to_penalize:
187 - logits[tuple(zip(*min_tokens_logits_to_penalize))] = -float("inf")
188 -
189 -
190 -def _apply_penalties(logits: torch.Tensor, prompt_token_ids: torch.Tensor,
191 - presence_penalties: torch.Tensor,
192 - frequency_penalties: torch.Tensor,
193 - repetition_penalties: torch.Tensor,
194 - output_token_ids: List[List[int]]):
195 - """
196 - Applies presence, frequency and repetition penalties to the logits.
197 - """
198 - _, vocab_size = logits.shape
199 - output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size,
200 - logits.device)
201 - return apply_penalties(logits, prompt_token_ids, output_tokens_t,
202 - presence_penalties, frequency_penalties,
203 - repetition_penalties)
204 -
205 -
206 -def _convert_to_tensors(output_token_ids: List[List[int]], vocab_size: int,
207 - device: torch.device) -> torch.Tensor:
208 - """
209 - Convert the different list data structures to tensors.
210 - """
211 - output_tokens_tensor = make_tensor_with_pad(
212 - output_token_ids,
213 - # Use the value of vocab_size as a pad since we don't have a
214 - # token_id of this value.
215 - pad=vocab_size,
216 - device="cpu",
217 - dtype=torch.int64,
218 - pin_memory=is_pin_memory_available(),
219 - )
220 - return output_tokens_tensor.to(device, non_blocking=True)