A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Fix yapf (#11538)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 27, 2024 at 09:47 UTC
81b979f2a8f7ec91c262dac7dcbf30ed577ebafd
2 files changed
+21
-19
vllm/v1/sample/ops/penalties.py
+13
-11
@@ -2,8 +2,7 @@ from typing import List, Set, Tuple
2
3
import torch
4
5
-from vllm.model_executor.layers.utils import (
6
- apply_penalties as _apply_penalties)
5
+from vllm.model_executor.layers.utils import apply_penalties
6
from vllm.utils import is_pin_memory_available, make_tensor_with_pad
7
8
@@ -17,27 +16,30 @@ def apply_min_token_penalties(logits: torch.Tensor,
16
"""
17
min_tokens_logits_to_penalize: List[Tuple[int, int]] = []
18
for index, min_token in enumerate(min_tokens):
20
- if (len(output_token_ids[index]) < min_token):
19
+ if len(output_token_ids[index]) < min_token:
20
for stop_token_id in stop_token_ids[index]:
21
min_tokens_logits_to_penalize.append((index, stop_token_id))
22
if min_tokens_logits_to_penalize:
23
logits[tuple(zip(*min_tokens_logits_to_penalize))] = -float("inf")
24
25
27
-def apply_penalties(logits: torch.Tensor, prompt_token_ids: torch.Tensor,
28
- presence_penalties: torch.Tensor,
29
- frequency_penalties: torch.Tensor,
30
- repetition_penalties: torch.Tensor,
31
- output_token_ids: List[List[int]]) -> torch.Tensor:
26
+def apply_all_penalties(
27
+ logits: torch.Tensor,
28
+ prompt_token_ids: torch.Tensor,
29
+ presence_penalties: torch.Tensor,
30
+ frequency_penalties: torch.Tensor,
31
+ repetition_penalties: torch.Tensor,
32
+ output_token_ids: List[List[int]],
33
+) -> torch.Tensor:
34
"""
35
Applies presence, frequency and repetition penalties to the logits.
36
"""
37
_, vocab_size = logits.shape
38
output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size,
39
logits.device)
38
- return _apply_penalties(logits, prompt_token_ids, output_tokens_t,
39
- presence_penalties, frequency_penalties,
40
- repetition_penalties)
40
+ return apply_penalties(logits, prompt_token_ids, output_tokens_t,
41
+ presence_penalties, frequency_penalties,
42
+ repetition_penalties)
43
44
45
def _convert_to_tensors(output_token_ids: List[List[int]], vocab_size: int,
vllm/v1/sample/sampler.py
+8
-8
@@ -6,8 +6,8 @@ import torch.nn as nn
6
7
from vllm.v1.outputs import SamplerOutput
8
from vllm.v1.sample.metadata import SamplingMetadata
9
-from vllm.v1.sample.ops.penalties import (apply_min_token_penalties,
10
- apply_penalties)
9
+from vllm.v1.sample.ops.penalties import (apply_all_penalties,
10
+ apply_min_token_penalties)
11
from vllm.v1.sample.ops.topk_topp_sampler import TopKTopPSampler
12
13
_SAMPLING_EPS = 1e-5
@@ -127,10 +127,10 @@ class Sampler(nn.Module):
127
sampling_metadata.min_tokens)
128
if not sampling_metadata.no_penalties:
129
assert sampling_metadata.prompt_token_ids is not None
130
- logits = apply_penalties(logits,
131
- sampling_metadata.prompt_token_ids,
132
- sampling_metadata.presence_penalties,
133
- sampling_metadata.frequency_penalties,
134
- sampling_metadata.repetition_penalties,
135
- sampling_metadata.output_token_ids)
130
+ logits = apply_all_penalties(
131
+ logits, sampling_metadata.prompt_token_ids,
132
+ sampling_metadata.presence_penalties,
133
+ sampling_metadata.frequency_penalties,
134
+ sampling_metadata.repetition_penalties,
135
+ sampling_metadata.output_token_ids)
136
return logits