@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a12934d3ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1][Core] min_p sampling support (#13191)

Signed-off-by: Aoyu <aoyuzhan@amazon.com> Co-authored-by: Aoyu <aoyuzhan@amazon.com>

Aoyu committed Feb 15, 2025 at 07:50 UTC a12934d3ec6025910a742b1492092819af634192
4 files changed +96
tests/v1/sample/test_sampler.py
+42
@@ -81,6 +81,8 @@ def _create_default_sampling_metadata(
81 top_k=torch.empty(batch_size, ),
82 no_top_p=True,
83 no_top_k=True,
84 + min_p=torch.empty(batch_size, ),
85 + no_min_p=True,
86 generators={},
87 max_num_logprobs=0,
88 prompt_token_ids=_create_prompt_tokens_tensor(prompt_token_ids,
@@ -336,6 +338,46 @@ def test_sampler_repetition_penalty(device: str, batch_size: int,
338 non_penalized_token_id in output_tokens)
339
340
341 +@pytest.mark.parametrize("device", CUDA_DEVICES)
342 +@pytest.mark.parametrize("batch_size", [1, 2, 32])
343 +@pytest.mark.parametrize("min_p", [0.0, 0.1])
344 +def test_sampler_min_p(device: str, batch_size: int, min_p: float):
345 + """
346 + Tests that when min_p is applied, tokens with probability below
347 + min_p * max_prob are masked with -inf.
348 + """
349 + torch.set_default_device(device)
350 + fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
351 +
352 + # Create one dominant token per batch
353 + for i in range(batch_size):
354 + fake_logits[i, 0] = 10.0 # High logit for first token
355 + fake_logits[i, 1:] = 1e-2 # Others remain low
356 +
357 + sampling_metadata = _create_default_sampling_metadata(
358 + NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
359 +
360 + # Configure min_p parameters
361 + sampling_metadata.min_p = torch.full((batch_size, ), min_p, device=device)
362 +
363 + sampler = Sampler()
364 + logits = sampler.apply_min_p(fake_logits, sampling_metadata.min_p)
365 + logits = logits.cpu()
366 +
367 + for batch_idx in range(batch_size):
368 + for token_id in range(VOCAB_SIZE):
369 + if token_id == 0:
370 + # Dominant token should always be unmasked
371 + assert logits[batch_idx][token_id] != -float("inf")
372 + else:
373 + if min_p > 0.0:
374 + # Non-dominant tokens should be masked when min_p > 0
375 + assert logits[batch_idx][token_id] == -float("inf")
376 + else:
377 + # No masking when min_p is 0
378 + assert logits[batch_idx][token_id] != -float("inf")
379 +
380 +
381 @pytest.mark.parametrize("device", CUDA_DEVICES)
382 @pytest.mark.parametrize("batch_size", [1, 2, 32])
383 @pytest.mark.parametrize("bias_value", [-0.1, 1.2])
vllm/v1/sample/metadata.py
+2
@@ -17,6 +17,8 @@ class SamplingMetadata:
17 top_k: torch.Tensor
18 no_top_p: bool
19 no_top_k: bool
20 + min_p: torch.Tensor
21 + no_min_p: bool
22
23 generators: Dict[int, torch.Generator]
24
vllm/v1/sample/sampler.py
+26
@@ -93,6 +93,10 @@ class Sampler(nn.Module):
93 sampling_metadata.no_top_p,
94 sampling_metadata.top_p,
95 )
96 +
97 + if not sampling_metadata.no_min_p:
98 + logits = self.apply_min_p(logits, sampling_metadata.min_p)
99 +
100 if sampling_metadata.all_random:
101 return random_sampled
102
@@ -169,6 +173,28 @@ class Sampler(nn.Module):
173 sampling_metadata.output_token_ids)
174 return logits
175
176 + def apply_min_p(
177 + self,
178 + logits: torch.Tensor,
179 + min_p: torch.Tensor,
180 + ) -> torch.Tensor:
181 + """
182 + Filters logits using adaptive probability thresholding.
183 + """
184 + # Convert logits to probability distribution
185 + probability_values = torch.nn.functional.softmax(logits, dim=-1)
186 + # Calculate maximum probabilities per sequence
187 + max_probabilities = torch.amax(probability_values,
188 + dim=-1,
189 + keepdim=True)
190 + # Reshape min_p for broadcasting
191 + adjusted_min_p = min_p.unsqueeze(1) * max_probabilities
192 + # Identify valid tokens using threshold comparison
193 + valid_token_mask = probability_values >= adjusted_min_p
194 + # Apply mask using boolean indexing
195 + logits[~valid_token_mask] = -float('inf')
196 + return logits
197 +
198 def apply_logits_bias(
199 self,
200 logits: torch.Tensor,
vllm/v1/worker/gpu_input_batch.py
+26
@@ -14,6 +14,8 @@ from vllm.sampling_params import SamplingParams, SamplingType
14 from vllm.v1.sample.metadata import SamplingMetadata
15 from vllm.v1.worker.block_table import BlockTable
16
17 +_SAMPLING_EPS = 1e-5
18 +
19 if TYPE_CHECKING:
20 from vllm.multimodal.inputs import PlaceholderRange
21
@@ -120,6 +122,16 @@ class InputBatch:
122 self.top_k_cpu = self.top_k_cpu_tensor.numpy()
123 self.top_k_reqs: Set[str] = set()
124
125 + self.min_p = torch.empty((max_num_reqs, ),
126 + dtype=torch.float32,
127 + device=device)
128 + self.min_p_cpu_tensor = torch.empty((max_num_reqs, ),
129 + dtype=torch.float32,
130 + device="cpu",
131 + pin_memory=pin_memory)
132 + self.min_p_cpu = self.min_p_cpu_tensor.numpy()
133 + self.min_p_reqs: Set[str] = set()
134 +
135 # Frequency penalty related data structures
136 self.frequency_penalties = torch.empty((max_num_reqs, ),
137 dtype=torch.float,
@@ -223,8 +235,11 @@ class InputBatch:
235 self.top_k_cpu[req_index] = sampling_params.top_k
236 if sampling_params.top_k > 0:
237 self.top_k_reqs.add(req_id)
238 + self.min_p_cpu[req_index] = sampling_params.min_p
239 self.frequency_penalties_cpu[
240 req_index] = sampling_params.frequency_penalty
241 + if sampling_params.min_p > _SAMPLING_EPS:
242 + self.min_p_reqs.add(req_id)
243 if sampling_params.frequency_penalty != 0.0:
244 self.frequency_penalties_reqs.add(req_id)
245 self.presence_penalties_cpu[
@@ -273,6 +288,7 @@ class InputBatch:
288 self.random_reqs.discard(req_id)
289 self.top_p_reqs.discard(req_id)
290 self.top_k_reqs.discard(req_id)
291 + self.min_p_reqs.discard(req_id)
292 self.frequency_penalties_reqs.discard(req_id)
293 self.presence_penalties_reqs.discard(req_id)
294 self.repetition_penalties_reqs.discard(req_id)
@@ -299,6 +315,7 @@ class InputBatch:
315 self.random_reqs.clear()
316 self.top_p_reqs.clear()
317 self.top_k_reqs.clear()
318 + self.min_p_reqs.clear()
319 self.frequency_penalties_reqs.clear()
320 self.presence_penalties_reqs.clear()
321 self.repetition_penalties_reqs.clear()
@@ -354,6 +371,7 @@ class InputBatch:
371 empty_index] = self.presence_penalties_cpu[last_req_index]
372 self.repetition_penalties_cpu[
373 empty_index] = self.repetition_penalties_cpu[last_req_index]
374 + self.min_p_cpu[empty_index] = self.min_p_cpu[last_req_index]
375 self.min_tokens[empty_index] = self.min_tokens[last_req_index]
376 self.stop_token_ids[empty_index] = self.stop_token_ids[
377 last_req_index]
@@ -381,6 +399,8 @@ class InputBatch:
399 self.top_p_cpu_tensor[:self.num_reqs], non_blocking=True)
400 self.top_k[:self.num_reqs].copy_(
401 self.top_k_cpu_tensor[:self.num_reqs], non_blocking=True)
402 + self.min_p[:self.num_reqs].copy_(
403 + self.min_p_cpu_tensor[:self.num_reqs], non_blocking=True)
404 if not self.no_penalties:
405 # Since syncing these tensors is expensive only copy them
406 # if necessary i.e. if there are requests which require
@@ -421,6 +441,8 @@ class InputBatch:
441 all_random=self.all_random,
442 top_p=self.top_p[:self.num_reqs],
443 top_k=self.top_k[:self.num_reqs],
444 + min_p=self.min_p[:self.num_reqs],
445 + no_min_p=self.no_min_p,
446 no_top_p=self.no_top_p,
447 no_top_k=self.no_top_k,
448 generators=self.generators,
@@ -497,6 +519,10 @@ class InputBatch:
519 def no_top_k(self) -> bool:
520 return len(self.top_k_reqs) == 0
521
522 + @property
523 + def no_min_p(self) -> bool:
524 + return len(self.min_p_reqs) == 0
525 +
526 @property
527 def no_penalties(self) -> bool:
528 return (len(self.presence_penalties_reqs) == 0