A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Aligning `top_p` and `top_k` Sampling (#1885)
* Align top_p and top_k with huggingface * remove _get_prompt_and_output_tokens * rename _apply_top_p_top_k * compare top_p top_k with hf * fix test errors
陈序 committed
Jan 13, 2024 at 05:51 UTC
218dc2ccdab133ffb0faa86cca510730fb917449
2 files changed
+78
-15
tests/samplers/test_sampler.py
+63
@@ -4,6 +4,7 @@ from unittest.mock import patch
4
5
import pytest
6
import torch
7
+from transformers import GenerationConfig, GenerationMixin
8
9
from vllm.model_executor.layers.sampler import Sampler
10
from vllm.model_executor.utils import set_random_seed
@@ -233,3 +234,65 @@ def test_sampler_logits_processors(seed: int):
234
for _, sequence_output in enumerate(sampler_output):
235
for idx, nth_output in enumerate(sequence_output.samples):
236
assert nth_output.output_token == idx
237
+
238
+
239
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
240
+def test_sampler_top_k_top_p(seed: int):
241
+ set_random_seed(seed)
242
+ batch_size = random.randint(1, 256)
243
+ top_k = random.randint(100, 500)
244
+ top_p = random.random() * 0.1
245
+ vocab_size = 32000
246
+ input_tensor = torch.rand((batch_size, 1024),
247
+ device="cuda",
248
+ dtype=torch.float16)
249
+ fake_logits = torch.normal(0,
250
+ 5,
251
+ size=(batch_size, vocab_size),
252
+ device=input_tensor.device,
253
+ dtype=input_tensor.dtype)
254
+ sampler = MockLogitsSampler(32000, fake_logits)
255
+ model_runner = ModelRunner(None, None, None)
256
+
257
+ generation_model = GenerationMixin()
258
+ generation_config = GenerationConfig(top_k=top_k,
259
+ top_p=top_p,
260
+ do_sample=True)
261
+ warpers = generation_model._get_logits_warper(generation_config)
262
+ assert len(warpers) == 2 # top_p and top_k
263
+
264
+ seq_group_metadata_list = []
265
+ prompt_lens = []
266
+ for i in range(batch_size):
267
+ seq_group_metadata_list.append(
268
+ SequenceGroupMetadata(
269
+ request_id=f"test_{i}",
270
+ is_prompt=True,
271
+ seq_data={0: SequenceData([1, 2, 3])},
272
+ sampling_params=SamplingParams(
273
+ temperature=1,
274
+ top_k=top_k,
275
+ top_p=top_p,
276
+ ),
277
+ block_tables={0: [1]},
278
+ ))
279
+ prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
280
+
281
+ sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
282
+ prompt_lens)
283
+
284
+ sample_probs = None
285
+
286
+ def mock_sample(probs, logprobs, sampling_metadata):
287
+ nonlocal sample_probs
288
+ sample_probs = probs
289
+ return [[prob.topk(1, dim=-1).indices.tolist(), [0]] for prob in probs]
290
+
291
+ with patch("vllm.model_executor.layers.sampler._sample", mock_sample):
292
+ sampler(embedding=None,
293
+ hidden_states=input_tensor,
294
+ sampling_metadata=sampling_metadata)
295
+ hf_probs = warpers(torch.zeros_like(fake_logits), fake_logits.clone())
296
+ hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
297
+ assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
298
+ assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
vllm/model_executor/layers/sampler.py
+15
-15
@@ -76,7 +76,7 @@ class Sampler(nn.Module):
76
logits.div_(sampling_tensors.temperatures.unsqueeze_(dim=1))
77
78
if do_top_p_top_k:
79
- logits = _apply_top_p_top_k(logits, sampling_tensors.top_ps,
79
+ logits = _apply_top_k_top_p(logits, sampling_tensors.top_ps,
80
sampling_tensors.top_ks)
81
82
if do_min_p:
@@ -185,27 +185,27 @@ def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
185
return logits
186
187
188
-def _apply_top_p_top_k(
188
+def _apply_top_k_top_p(
189
logits: torch.Tensor,
190
p: torch.Tensor,
191
k: torch.Tensor,
192
) -> torch.Tensor:
193
- logits_sort, logits_idx = logits.sort(dim=-1, descending=True)
193
+ logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
194
+
195
+ # Apply top-k.
196
+ top_k_mask = logits_sort.size(1) - k.to(torch.long)
197
+ # Get all the top_k values.
198
+ top_k_mask = logits_sort.gather(1, top_k_mask.unsqueeze(dim=1))
199
+ top_k_mask = logits_sort < top_k_mask
200
+ logits_sort.masked_fill_(top_k_mask, -float("inf"))
201
202
# Apply top-p.
203
probs_sort = logits_sort.softmax(dim=-1)
197
- probs_sum = probs_sort.cumsum(dim=-1).sub_(probs_sort)
198
- top_p_mask = probs_sum > p.unsqueeze_(dim=1)
199
-
200
- # Apply top-k.
201
- # Create a mask for the top-k elements.
202
- top_k_mask = torch.arange(logits_idx.shape[-1], device=logits_idx.device)
203
- top_k_mask = top_k_mask.expand(logits_idx.shape[0], -1)
204
- top_k_mask = top_k_mask >= k.unsqueeze_(dim=1)
205
-
206
- # Final mask.
207
- mask = (top_p_mask | top_k_mask)
208
- logits_sort.masked_fill_(mask, -float("inf"))
204
+ probs_sum = probs_sort.cumsum(dim=-1)
205
+ top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1)
206
+ # at least one
207
+ top_p_mask[:, -1] = False
208
+ logits_sort.masked_fill_(top_p_mask, -float("inf"))
209
210
# Re-sort the probabilities.
211
src = torch.arange(logits_idx.shape[-1],