@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 81b979f2a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Fix yapf (#11538)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 27, 2024 at 09:47 UTC 81b979f2a8f7ec91c262dac7dcbf30ed577ebafd
2 files changed +21 -19
vllm/v1/sample/ops/penalties.py
+13 -11
@@ -2,8 +2,7 @@ from typing import List, Set, Tuple
2
3 import torch
4
5 -from vllm.model_executor.layers.utils import (
6 - apply_penalties as _apply_penalties)
5 +from vllm.model_executor.layers.utils import apply_penalties
6 from vllm.utils import is_pin_memory_available, make_tensor_with_pad
7
8
@@ -17,27 +16,30 @@ def apply_min_token_penalties(logits: torch.Tensor,
16 """
17 min_tokens_logits_to_penalize: List[Tuple[int, int]] = []
18 for index, min_token in enumerate(min_tokens):
20 - if (len(output_token_ids[index]) < min_token):
19 + if len(output_token_ids[index]) < min_token:
20 for stop_token_id in stop_token_ids[index]:
21 min_tokens_logits_to_penalize.append((index, stop_token_id))
22 if min_tokens_logits_to_penalize:
23 logits[tuple(zip(*min_tokens_logits_to_penalize))] = -float("inf")
24
25
27 -def apply_penalties(logits: torch.Tensor, prompt_token_ids: torch.Tensor,
28 - presence_penalties: torch.Tensor,
29 - frequency_penalties: torch.Tensor,
30 - repetition_penalties: torch.Tensor,
31 - output_token_ids: List[List[int]]) -> torch.Tensor:
26 +def apply_all_penalties(
27 + logits: torch.Tensor,
28 + prompt_token_ids: torch.Tensor,
29 + presence_penalties: torch.Tensor,
30 + frequency_penalties: torch.Tensor,
31 + repetition_penalties: torch.Tensor,
32 + output_token_ids: List[List[int]],
33 +) -> torch.Tensor:
34 """
35 Applies presence, frequency and repetition penalties to the logits.
36 """
37 _, vocab_size = logits.shape
38 output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size,
39 logits.device)
38 - return _apply_penalties(logits, prompt_token_ids, output_tokens_t,
39 - presence_penalties, frequency_penalties,
40 - repetition_penalties)
40 + return apply_penalties(logits, prompt_token_ids, output_tokens_t,
41 + presence_penalties, frequency_penalties,
42 + repetition_penalties)
43
44
45 def _convert_to_tensors(output_token_ids: List[List[int]], vocab_size: int,
vllm/v1/sample/sampler.py
+8 -8
@@ -6,8 +6,8 @@ import torch.nn as nn
6
7 from vllm.v1.outputs import SamplerOutput
8 from vllm.v1.sample.metadata import SamplingMetadata
9 -from vllm.v1.sample.ops.penalties import (apply_min_token_penalties,
10 - apply_penalties)
9 +from vllm.v1.sample.ops.penalties import (apply_all_penalties,
10 + apply_min_token_penalties)
11 from vllm.v1.sample.ops.topk_topp_sampler import TopKTopPSampler
12
13 _SAMPLING_EPS = 1e-5
@@ -127,10 +127,10 @@ class Sampler(nn.Module):
127 sampling_metadata.min_tokens)
128 if not sampling_metadata.no_penalties:
129 assert sampling_metadata.prompt_token_ids is not None
130 - logits = apply_penalties(logits,
131 - sampling_metadata.prompt_token_ids,
132 - sampling_metadata.presence_penalties,
133 - sampling_metadata.frequency_penalties,
134 - sampling_metadata.repetition_penalties,
135 - sampling_metadata.output_token_ids)
130 + logits = apply_all_penalties(
131 + logits, sampling_metadata.prompt_token_ids,
132 + sampling_metadata.presence_penalties,
133 + sampling_metadata.frequency_penalties,
134 + sampling_metadata.repetition_penalties,
135 + sampling_metadata.output_token_ids)
136 return logits