@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e3470f8753
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core]: Option To Use Prompt Token Ids Inside Logits Processor (#4985)

Co-authored-by: Elisei Smirnov <el.smirnov@innopolis.university>

Elisei Smirnov committed May 24, 2024 at 01:04 UTC e3470f87538ec86d1094ac4747519c6300213088
2 files changed +24 -8
vllm/model_executor/layers/logits_processor.py
+14 -3
@@ -1,4 +1,5 @@
1 """A layer that compute logits from hidden_stats."""
2 +import inspect
3 from typing import Optional
4
5 import torch
@@ -95,15 +96,25 @@ def _apply_logits_processors(
96 seq_ids = seq_group.seq_ids
97 sampling_params = seq_group.sampling_params
98 logits_processors = sampling_params.logits_processors
98 -
99 if logits_processors:
100 found_logits_processors = True
101 +
102 for seq_id, logits_row_idx in zip(seq_ids,
103 seq_group.sample_indices):
104 logits_row = logits[logits_row_idx]
104 - token_ids = seq_group.seq_data[seq_id].output_token_ids
105 + past_tokens_ids = seq_group.seq_data[seq_id].output_token_ids
106 + prompt_tokens_ids = seq_group.seq_data[seq_id].prompt_token_ids
107 +
108 for logits_processor in logits_processors:
106 - logits_row = logits_processor(token_ids, logits_row)
109 + parameters = inspect.signature(logits_processor).parameters
110 + if len(parameters) == 3:
111 + logits_row = logits_processor(prompt_tokens_ids,
112 + past_tokens_ids,
113 + logits_row)
114 + else:
115 + logits_row = logits_processor(past_tokens_ids,
116 + logits_row)
117 +
118 logits[logits_row_idx] = logits_row
119
120 logits_processed += len(seq_group.sample_indices) + len(
vllm/sampling_params.py
+10 -5
@@ -18,10 +18,14 @@ class SamplingType(IntEnum):
18 BEAM = 3
19
20
21 -LogitsProcessor = Callable[[List[int], torch.Tensor], torch.Tensor]
22 -"""LogitsProcessor is a function that takes a list of previously generated
23 -tokens and a tensor of the logits for the next token, and returns a modified
24 -tensor of logits to sample from."""
21 +LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor],
22 + Callable[[List[int], List[int], torch.Tensor],
23 + torch.Tensor]]
24 +"""LogitsProcessor is a function that takes a list
25 +of previously generated tokens, the logits tensor
26 +for the next token and, optionally, prompt tokens as a
27 +first argument, and returns a modified tensor of logits
28 +to sample from."""
29
30
31 class SamplingParams:
@@ -95,7 +99,8 @@ class SamplingParams:
99 spaces_between_special_tokens: Whether to add spaces between special
100 tokens in the output. Defaults to True.
101 logits_processors: List of functions that modify logits based on
98 - previously generated tokens.
102 + previously generated tokens, and optionally prompt tokens as
103 + a first argument.
104 truncate_prompt_tokens: If set to an integer k, will use only the last k
105 tokens from the prompt (i.e., left truncation). Defaults to None
106 (i.e., no truncation).