A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Add option to completion API to truncate prompt tokens (#3144)
Thomas Parnell committed
Apr 5, 2024 at 19:15 UTC
1d7c940d74f4923f7507725598bcd08f0f1dbd2d
4 files changed
+41
-8
vllm/entrypoints/openai/protocol.py
+3
-1
@@ -4,7 +4,7 @@ import time
4
from typing import Dict, List, Literal, Optional, Union
5
6
import torch
7
-from pydantic import BaseModel, Field, model_validator
7
+from pydantic import BaseModel, Field, conint, model_validator
8
9
from vllm.sampling_params import SamplingParams
10
from vllm.utils import random_uuid
@@ -229,6 +229,7 @@ class CompletionRequest(BaseModel):
229
min_tokens: Optional[int] = 0
230
skip_special_tokens: Optional[bool] = True
231
spaces_between_special_tokens: Optional[bool] = True
232
+ truncate_prompt_tokens: Optional[conint(ge=1)] = None
233
# doc: end-completion-sampling-params
234
235
# doc: begin-completion-extra-params
@@ -309,6 +310,7 @@ class CompletionRequest(BaseModel):
310
include_stop_str_in_output=self.include_stop_str_in_output,
311
length_penalty=self.length_penalty,
312
logits_processors=logits_processors,
313
+ truncate_prompt_tokens=self.truncate_prompt_tokens,
314
)
315
316
@model_validator(mode="before")
vllm/entrypoints/openai/serving_completion.py
+8
-2
@@ -137,10 +137,16 @@ class OpenAIServingCompletion(OpenAIServing):
137
for i, prompt in enumerate(prompts):
138
if prompt_is_tokens:
139
input_ids = self._validate_prompt_and_tokenize(
140
- request, prompt_ids=prompt)
140
+ request,
141
+ prompt_ids=prompt,
142
+ truncate_prompt_tokens=sampling_params.
143
+ truncate_prompt_tokens)
144
else:
145
input_ids = self._validate_prompt_and_tokenize(
143
- request, prompt=prompt)
146
+ request,
147
+ prompt=prompt,
148
+ truncate_prompt_tokens=sampling_params.
149
+ truncate_prompt_tokens)
150
151
generators.append(
152
self.engine.generate(prompt,
vllm/entrypoints/openai/serving_engine.py
+18
-4
@@ -4,6 +4,8 @@ from dataclasses import dataclass
4
from http import HTTPStatus
5
from typing import Dict, List, Optional, Union
6
7
+from pydantic import conint
8
+
9
from vllm.engine.async_llm_engine import AsyncLLMEngine
10
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
11
CompletionRequest, ErrorResponse,
@@ -66,7 +68,8 @@ class OpenAIServing:
68
self.tokenizer = get_tokenizer(
69
engine_model_config.tokenizer,
70
tokenizer_mode=engine_model_config.tokenizer_mode,
69
- trust_remote_code=engine_model_config.trust_remote_code)
71
+ trust_remote_code=engine_model_config.trust_remote_code,
72
+ truncation_side="left")
73
74
async def show_available_models(self) -> ModelList:
75
"""Show available models. Right now we only have one model."""
@@ -164,15 +167,26 @@ class OpenAIServing:
167
self,
168
request: Union[ChatCompletionRequest, CompletionRequest],
169
prompt: Optional[str] = None,
167
- prompt_ids: Optional[List[int]] = None) -> List[int]:
170
+ prompt_ids: Optional[List[int]] = None,
171
+ truncate_prompt_tokens: Optional[conint(ge=1)] = None
172
+ ) -> List[int]:
173
if not (prompt or prompt_ids):
174
raise ValueError("Either prompt or prompt_ids should be provided.")
175
if (prompt and prompt_ids):
176
raise ValueError(
177
"Only one of prompt or prompt_ids should be provided.")
178
174
- input_ids = prompt_ids if prompt_ids is not None else self.tokenizer(
175
- prompt).input_ids
179
+ if prompt_ids is None:
180
+ tokenizer_kwargs = {} if truncate_prompt_tokens is None else {
181
+ "truncation": True,
182
+ "max_length": truncate_prompt_tokens,
183
+ }
184
+ input_ids = self.tokenizer(prompt, **tokenizer_kwargs).input_ids
185
+ elif truncate_prompt_tokens is not None:
186
+ input_ids = prompt_ids[-truncate_prompt_tokens:]
187
+ else:
188
+ input_ids = prompt_ids
189
+
190
token_num = len(input_ids)
191
192
if request.max_tokens is None:
vllm/sampling_params.py
+12
-1
@@ -5,6 +5,7 @@ from functools import cached_property
5
from typing import Callable, List, Optional, Union
6
7
import torch
8
+from pydantic import conint
9
10
_SAMPLING_EPS = 1e-5
11
@@ -94,6 +95,9 @@ class SamplingParams:
95
tokens in the output. Defaults to True.
96
logits_processors: List of functions that modify logits based on
97
previously generated tokens.
98
+ truncate_prompt_tokens: If set to an integer k, will use only the last k
99
+ tokens from the prompt (i.e., left truncation). Defaults to None
100
+ (i.e., no truncation).
101
"""
102
103
def __init__(
@@ -123,6 +127,7 @@ class SamplingParams:
127
skip_special_tokens: bool = True,
128
spaces_between_special_tokens: bool = True,
129
logits_processors: Optional[List[LogitsProcessor]] = None,
130
+ truncate_prompt_tokens: Optional[conint(ge=1)] = None,
131
) -> None:
132
self.n = n
133
self.best_of = best_of if best_of is not None else n
@@ -160,6 +165,7 @@ class SamplingParams:
165
self.spaces_between_special_tokens = spaces_between_special_tokens
166
self.logits_processors = logits_processors
167
self.include_stop_str_in_output = include_stop_str_in_output
168
+ self.truncate_prompt_tokens = truncate_prompt_tokens
169
self._verify_args()
170
if self.use_beam_search:
171
self._verify_beam_search()
@@ -216,6 +222,10 @@ class SamplingParams:
222
if self.prompt_logprobs is not None and self.prompt_logprobs < 0:
223
raise ValueError(f"prompt_logprobs must be non-negative, got "
224
f"{self.prompt_logprobs}.")
225
+ if (self.truncate_prompt_tokens is not None
226
+ and self.truncate_prompt_tokens < 1):
227
+ raise ValueError(f"truncate_prompt_tokens must be >= 1, "
228
+ f"got {self.truncate_prompt_tokens}")
229
if self.stop and not self.detokenize:
230
raise ValueError(
231
"stop strings are only supported when detokenize is True. "
@@ -300,4 +310,5 @@ class SamplingParams:
310
f"prompt_logprobs={self.prompt_logprobs}, "
311
f"skip_special_tokens={self.skip_special_tokens}, "
312
"spaces_between_special_tokens="
303
- f"{self.spaces_between_special_tokens})")
313
+ f"{self.spaces_between_special_tokens}, "
314
+ f"truncate_prompt_tokens={self.truncate_prompt_tokens})")