A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core]: Option To Use Prompt Token Ids Inside Logits Processor (#4985)
Co-authored-by: Elisei Smirnov <el.smirnov@innopolis.university>
Elisei Smirnov committed
May 24, 2024 at 01:04 UTC
e3470f87538ec86d1094ac4747519c6300213088
2 files changed
+24
-8
vllm/model_executor/layers/logits_processor.py
+14
-3
@@ -1,4 +1,5 @@
1
"""A layer that compute logits from hidden_stats."""
2
+import inspect
3
from typing import Optional
4
5
import torch
@@ -95,15 +96,25 @@ def _apply_logits_processors(
96
seq_ids = seq_group.seq_ids
97
sampling_params = seq_group.sampling_params
98
logits_processors = sampling_params.logits_processors
98
-
99
if logits_processors:
100
found_logits_processors = True
101
+
102
for seq_id, logits_row_idx in zip(seq_ids,
103
seq_group.sample_indices):
104
logits_row = logits[logits_row_idx]
104
- token_ids = seq_group.seq_data[seq_id].output_token_ids
105
+ past_tokens_ids = seq_group.seq_data[seq_id].output_token_ids
106
+ prompt_tokens_ids = seq_group.seq_data[seq_id].prompt_token_ids
107
+
108
for logits_processor in logits_processors:
106
- logits_row = logits_processor(token_ids, logits_row)
109
+ parameters = inspect.signature(logits_processor).parameters
110
+ if len(parameters) == 3:
111
+ logits_row = logits_processor(prompt_tokens_ids,
112
+ past_tokens_ids,
113
+ logits_row)
114
+ else:
115
+ logits_row = logits_processor(past_tokens_ids,
116
+ logits_row)
117
+
118
logits[logits_row_idx] = logits_row
119
120
logits_processed += len(seq_group.sample_indices) + len(
vllm/sampling_params.py
+10
-5
@@ -18,10 +18,14 @@ class SamplingType(IntEnum):
18
BEAM = 3
19
20
21
-LogitsProcessor = Callable[[List[int], torch.Tensor], torch.Tensor]
22
-"""LogitsProcessor is a function that takes a list of previously generated
23
-tokens and a tensor of the logits for the next token, and returns a modified
24
-tensor of logits to sample from."""
21
+LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor],
22
+ Callable[[List[int], List[int], torch.Tensor],
23
+ torch.Tensor]]
24
+"""LogitsProcessor is a function that takes a list
25
+of previously generated tokens, the logits tensor
26
+for the next token and, optionally, prompt tokens as a
27
+first argument, and returns a modified tensor of logits
28
+to sample from."""
29
30
31
class SamplingParams:
@@ -95,7 +99,8 @@ class SamplingParams:
99
spaces_between_special_tokens: Whether to add spaces between special
100
tokens in the output. Defaults to True.
101
logits_processors: List of functions that modify logits based on
98
- previously generated tokens.
102
+ previously generated tokens, and optionally prompt tokens as
103
+ a first argument.
104
truncate_prompt_tokens: If set to an integer k, will use only the last k
105
tokens from the prompt (i.e., left truncation). Defaults to None
106
(i.e., no truncation).