A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][Core] min_p sampling support (#13191)
Signed-off-by: Aoyu <aoyuzhan@amazon.com> Co-authored-by: Aoyu <aoyuzhan@amazon.com>
Aoyu committed
Feb 15, 2025 at 07:50 UTC
a12934d3ec6025910a742b1492092819af634192
4 files changed
+96
tests/v1/sample/test_sampler.py
+42
@@ -81,6 +81,8 @@ def _create_default_sampling_metadata(
81
top_k=torch.empty(batch_size, ),
82
no_top_p=True,
83
no_top_k=True,
84
+ min_p=torch.empty(batch_size, ),
85
+ no_min_p=True,
86
generators={},
87
max_num_logprobs=0,
88
prompt_token_ids=_create_prompt_tokens_tensor(prompt_token_ids,
@@ -336,6 +338,46 @@ def test_sampler_repetition_penalty(device: str, batch_size: int,
338
non_penalized_token_id in output_tokens)
339
340
341
+@pytest.mark.parametrize("device", CUDA_DEVICES)
342
+@pytest.mark.parametrize("batch_size", [1, 2, 32])
343
+@pytest.mark.parametrize("min_p", [0.0, 0.1])
344
+def test_sampler_min_p(device: str, batch_size: int, min_p: float):
345
+ """
346
+ Tests that when min_p is applied, tokens with probability below
347
+ min_p * max_prob are masked with -inf.
348
+ """
349
+ torch.set_default_device(device)
350
+ fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
351
+
352
+ # Create one dominant token per batch
353
+ for i in range(batch_size):
354
+ fake_logits[i, 0] = 10.0 # High logit for first token
355
+ fake_logits[i, 1:] = 1e-2 # Others remain low
356
+
357
+ sampling_metadata = _create_default_sampling_metadata(
358
+ NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
359
+
360
+ # Configure min_p parameters
361
+ sampling_metadata.min_p = torch.full((batch_size, ), min_p, device=device)
362
+
363
+ sampler = Sampler()
364
+ logits = sampler.apply_min_p(fake_logits, sampling_metadata.min_p)
365
+ logits = logits.cpu()
366
+
367
+ for batch_idx in range(batch_size):
368
+ for token_id in range(VOCAB_SIZE):
369
+ if token_id == 0:
370
+ # Dominant token should always be unmasked
371
+ assert logits[batch_idx][token_id] != -float("inf")
372
+ else:
373
+ if min_p > 0.0:
374
+ # Non-dominant tokens should be masked when min_p > 0
375
+ assert logits[batch_idx][token_id] == -float("inf")
376
+ else:
377
+ # No masking when min_p is 0
378
+ assert logits[batch_idx][token_id] != -float("inf")
379
+
380
+
381
@pytest.mark.parametrize("device", CUDA_DEVICES)
382
@pytest.mark.parametrize("batch_size", [1, 2, 32])
383
@pytest.mark.parametrize("bias_value", [-0.1, 1.2])
vllm/v1/sample/metadata.py
+2
@@ -17,6 +17,8 @@ class SamplingMetadata:
17
top_k: torch.Tensor
18
no_top_p: bool
19
no_top_k: bool
20
+ min_p: torch.Tensor
21
+ no_min_p: bool
22
23
generators: Dict[int, torch.Generator]
24
vllm/v1/sample/sampler.py
+26
@@ -93,6 +93,10 @@ class Sampler(nn.Module):
93
sampling_metadata.no_top_p,
94
sampling_metadata.top_p,
95
)
96
+
97
+ if not sampling_metadata.no_min_p:
98
+ logits = self.apply_min_p(logits, sampling_metadata.min_p)
99
+
100
if sampling_metadata.all_random:
101
return random_sampled
102
@@ -169,6 +173,28 @@ class Sampler(nn.Module):
173
sampling_metadata.output_token_ids)
174
return logits
175
176
+ def apply_min_p(
177
+ self,
178
+ logits: torch.Tensor,
179
+ min_p: torch.Tensor,
180
+ ) -> torch.Tensor:
181
+ """
182
+ Filters logits using adaptive probability thresholding.
183
+ """
184
+ # Convert logits to probability distribution
185
+ probability_values = torch.nn.functional.softmax(logits, dim=-1)
186
+ # Calculate maximum probabilities per sequence
187
+ max_probabilities = torch.amax(probability_values,
188
+ dim=-1,
189
+ keepdim=True)
190
+ # Reshape min_p for broadcasting
191
+ adjusted_min_p = min_p.unsqueeze(1) * max_probabilities
192
+ # Identify valid tokens using threshold comparison
193
+ valid_token_mask = probability_values >= adjusted_min_p
194
+ # Apply mask using boolean indexing
195
+ logits[~valid_token_mask] = -float('inf')
196
+ return logits
197
+
198
def apply_logits_bias(
199
self,
200
logits: torch.Tensor,
vllm/v1/worker/gpu_input_batch.py
+26
@@ -14,6 +14,8 @@ from vllm.sampling_params import SamplingParams, SamplingType
14
from vllm.v1.sample.metadata import SamplingMetadata
15
from vllm.v1.worker.block_table import BlockTable
16
17
+_SAMPLING_EPS = 1e-5
18
+
19
if TYPE_CHECKING:
20
from vllm.multimodal.inputs import PlaceholderRange
21
@@ -120,6 +122,16 @@ class InputBatch:
122
self.top_k_cpu = self.top_k_cpu_tensor.numpy()
123
self.top_k_reqs: Set[str] = set()
124
125
+ self.min_p = torch.empty((max_num_reqs, ),
126
+ dtype=torch.float32,
127
+ device=device)
128
+ self.min_p_cpu_tensor = torch.empty((max_num_reqs, ),
129
+ dtype=torch.float32,
130
+ device="cpu",
131
+ pin_memory=pin_memory)
132
+ self.min_p_cpu = self.min_p_cpu_tensor.numpy()
133
+ self.min_p_reqs: Set[str] = set()
134
+
135
# Frequency penalty related data structures
136
self.frequency_penalties = torch.empty((max_num_reqs, ),
137
dtype=torch.float,
@@ -223,8 +235,11 @@ class InputBatch:
235
self.top_k_cpu[req_index] = sampling_params.top_k
236
if sampling_params.top_k > 0:
237
self.top_k_reqs.add(req_id)
238
+ self.min_p_cpu[req_index] = sampling_params.min_p
239
self.frequency_penalties_cpu[
240
req_index] = sampling_params.frequency_penalty
241
+ if sampling_params.min_p > _SAMPLING_EPS:
242
+ self.min_p_reqs.add(req_id)
243
if sampling_params.frequency_penalty != 0.0:
244
self.frequency_penalties_reqs.add(req_id)
245
self.presence_penalties_cpu[
@@ -273,6 +288,7 @@ class InputBatch:
288
self.random_reqs.discard(req_id)
289
self.top_p_reqs.discard(req_id)
290
self.top_k_reqs.discard(req_id)
291
+ self.min_p_reqs.discard(req_id)
292
self.frequency_penalties_reqs.discard(req_id)
293
self.presence_penalties_reqs.discard(req_id)
294
self.repetition_penalties_reqs.discard(req_id)
@@ -299,6 +315,7 @@ class InputBatch:
315
self.random_reqs.clear()
316
self.top_p_reqs.clear()
317
self.top_k_reqs.clear()
318
+ self.min_p_reqs.clear()
319
self.frequency_penalties_reqs.clear()
320
self.presence_penalties_reqs.clear()
321
self.repetition_penalties_reqs.clear()
@@ -354,6 +371,7 @@ class InputBatch:
371
empty_index] = self.presence_penalties_cpu[last_req_index]
372
self.repetition_penalties_cpu[
373
empty_index] = self.repetition_penalties_cpu[last_req_index]
374
+ self.min_p_cpu[empty_index] = self.min_p_cpu[last_req_index]
375
self.min_tokens[empty_index] = self.min_tokens[last_req_index]
376
self.stop_token_ids[empty_index] = self.stop_token_ids[
377
last_req_index]
@@ -381,6 +399,8 @@ class InputBatch:
399
self.top_p_cpu_tensor[:self.num_reqs], non_blocking=True)
400
self.top_k[:self.num_reqs].copy_(
401
self.top_k_cpu_tensor[:self.num_reqs], non_blocking=True)
402
+ self.min_p[:self.num_reqs].copy_(
403
+ self.min_p_cpu_tensor[:self.num_reqs], non_blocking=True)
404
if not self.no_penalties:
405
# Since syncing these tensors is expensive only copy them
406
# if necessary i.e. if there are requests which require
@@ -421,6 +441,8 @@ class InputBatch:
441
all_random=self.all_random,
442
top_p=self.top_p[:self.num_reqs],
443
top_k=self.top_k[:self.num_reqs],
444
+ min_p=self.min_p[:self.num_reqs],
445
+ no_min_p=self.no_min_p,
446
no_top_p=self.no_top_p,
447
no_top_k=self.no_top_k,
448
generators=self.generators,
@@ -497,6 +519,10 @@ class InputBatch:
519
def no_top_k(self) -> bool:
520
return len(self.top_k_reqs) == 0
521
522
+ @property
523
+ def no_min_p(self) -> bool:
524
+ return len(self.min_p_reqs) == 0
525
+
526
@property
527
def no_penalties(self) -> bool:
528
return (len(self.presence_penalties_reqs) == 0