@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 218dc2ccda
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Aligning `top_p` and `top_k` Sampling (#1885)

* Align top_p and top_k with huggingface * remove _get_prompt_and_output_tokens * rename _apply_top_p_top_k * compare top_p top_k with hf * fix test errors

陈序 committed Jan 13, 2024 at 05:51 UTC 218dc2ccdab133ffb0faa86cca510730fb917449
2 files changed +78 -15
tests/samplers/test_sampler.py
+63
@@ -4,6 +4,7 @@ from unittest.mock import patch
4
5 import pytest
6 import torch
7 +from transformers import GenerationConfig, GenerationMixin
8
9 from vllm.model_executor.layers.sampler import Sampler
10 from vllm.model_executor.utils import set_random_seed
@@ -233,3 +234,65 @@ def test_sampler_logits_processors(seed: int):
234 for _, sequence_output in enumerate(sampler_output):
235 for idx, nth_output in enumerate(sequence_output.samples):
236 assert nth_output.output_token == idx
237 +
238 +
239 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
240 +def test_sampler_top_k_top_p(seed: int):
241 + set_random_seed(seed)
242 + batch_size = random.randint(1, 256)
243 + top_k = random.randint(100, 500)
244 + top_p = random.random() * 0.1
245 + vocab_size = 32000
246 + input_tensor = torch.rand((batch_size, 1024),
247 + device="cuda",
248 + dtype=torch.float16)
249 + fake_logits = torch.normal(0,
250 + 5,
251 + size=(batch_size, vocab_size),
252 + device=input_tensor.device,
253 + dtype=input_tensor.dtype)
254 + sampler = MockLogitsSampler(32000, fake_logits)
255 + model_runner = ModelRunner(None, None, None)
256 +
257 + generation_model = GenerationMixin()
258 + generation_config = GenerationConfig(top_k=top_k,
259 + top_p=top_p,
260 + do_sample=True)
261 + warpers = generation_model._get_logits_warper(generation_config)
262 + assert len(warpers) == 2 # top_p and top_k
263 +
264 + seq_group_metadata_list = []
265 + prompt_lens = []
266 + for i in range(batch_size):
267 + seq_group_metadata_list.append(
268 + SequenceGroupMetadata(
269 + request_id=f"test_{i}",
270 + is_prompt=True,
271 + seq_data={0: SequenceData([1, 2, 3])},
272 + sampling_params=SamplingParams(
273 + temperature=1,
274 + top_k=top_k,
275 + top_p=top_p,
276 + ),
277 + block_tables={0: [1]},
278 + ))
279 + prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
280 +
281 + sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
282 + prompt_lens)
283 +
284 + sample_probs = None
285 +
286 + def mock_sample(probs, logprobs, sampling_metadata):
287 + nonlocal sample_probs
288 + sample_probs = probs
289 + return [[prob.topk(1, dim=-1).indices.tolist(), [0]] for prob in probs]
290 +
291 + with patch("vllm.model_executor.layers.sampler._sample", mock_sample):
292 + sampler(embedding=None,
293 + hidden_states=input_tensor,
294 + sampling_metadata=sampling_metadata)
295 + hf_probs = warpers(torch.zeros_like(fake_logits), fake_logits.clone())
296 + hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
297 + assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
298 + assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
vllm/model_executor/layers/sampler.py
+15 -15
@@ -76,7 +76,7 @@ class Sampler(nn.Module):
76 logits.div_(sampling_tensors.temperatures.unsqueeze_(dim=1))
77
78 if do_top_p_top_k:
79 - logits = _apply_top_p_top_k(logits, sampling_tensors.top_ps,
79 + logits = _apply_top_k_top_p(logits, sampling_tensors.top_ps,
80 sampling_tensors.top_ks)
81
82 if do_min_p:
@@ -185,27 +185,27 @@ def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
185 return logits
186
187
188 -def _apply_top_p_top_k(
188 +def _apply_top_k_top_p(
189 logits: torch.Tensor,
190 p: torch.Tensor,
191 k: torch.Tensor,
192 ) -> torch.Tensor:
193 - logits_sort, logits_idx = logits.sort(dim=-1, descending=True)
193 + logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
194 +
195 + # Apply top-k.
196 + top_k_mask = logits_sort.size(1) - k.to(torch.long)
197 + # Get all the top_k values.
198 + top_k_mask = logits_sort.gather(1, top_k_mask.unsqueeze(dim=1))
199 + top_k_mask = logits_sort < top_k_mask
200 + logits_sort.masked_fill_(top_k_mask, -float("inf"))
201
202 # Apply top-p.
203 probs_sort = logits_sort.softmax(dim=-1)
197 - probs_sum = probs_sort.cumsum(dim=-1).sub_(probs_sort)
198 - top_p_mask = probs_sum > p.unsqueeze_(dim=1)
199 -
200 - # Apply top-k.
201 - # Create a mask for the top-k elements.
202 - top_k_mask = torch.arange(logits_idx.shape[-1], device=logits_idx.device)
203 - top_k_mask = top_k_mask.expand(logits_idx.shape[0], -1)
204 - top_k_mask = top_k_mask >= k.unsqueeze_(dim=1)
205 -
206 - # Final mask.
207 - mask = (top_p_mask | top_k_mask)
208 - logits_sort.masked_fill_(mask, -float("inf"))
204 + probs_sum = probs_sort.cumsum(dim=-1)
205 + top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1)
206 + # at least one
207 + top_p_mask[:, -1] = False
208 + logits_sort.masked_fill_(top_p_mask, -float("inf"))
209
210 # Re-sort the probabilities.
211 src = torch.arange(logits_idx.shape[-1],