@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1d7c940d74
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Add option to completion API to truncate prompt tokens (#3144)

Thomas Parnell committed Apr 5, 2024 at 19:15 UTC 1d7c940d74f4923f7507725598bcd08f0f1dbd2d
4 files changed +41 -8
vllm/entrypoints/openai/protocol.py
+3 -1
@@ -4,7 +4,7 @@ import time
4 from typing import Dict, List, Literal, Optional, Union
5
6 import torch
7 -from pydantic import BaseModel, Field, model_validator
7 +from pydantic import BaseModel, Field, conint, model_validator
8
9 from vllm.sampling_params import SamplingParams
10 from vllm.utils import random_uuid
@@ -229,6 +229,7 @@ class CompletionRequest(BaseModel):
229 min_tokens: Optional[int] = 0
230 skip_special_tokens: Optional[bool] = True
231 spaces_between_special_tokens: Optional[bool] = True
232 + truncate_prompt_tokens: Optional[conint(ge=1)] = None
233 # doc: end-completion-sampling-params
234
235 # doc: begin-completion-extra-params
@@ -309,6 +310,7 @@ class CompletionRequest(BaseModel):
310 include_stop_str_in_output=self.include_stop_str_in_output,
311 length_penalty=self.length_penalty,
312 logits_processors=logits_processors,
313 + truncate_prompt_tokens=self.truncate_prompt_tokens,
314 )
315
316 @model_validator(mode="before")
vllm/entrypoints/openai/serving_completion.py
+8 -2
@@ -137,10 +137,16 @@ class OpenAIServingCompletion(OpenAIServing):
137 for i, prompt in enumerate(prompts):
138 if prompt_is_tokens:
139 input_ids = self._validate_prompt_and_tokenize(
140 - request, prompt_ids=prompt)
140 + request,
141 + prompt_ids=prompt,
142 + truncate_prompt_tokens=sampling_params.
143 + truncate_prompt_tokens)
144 else:
145 input_ids = self._validate_prompt_and_tokenize(
143 - request, prompt=prompt)
146 + request,
147 + prompt=prompt,
148 + truncate_prompt_tokens=sampling_params.
149 + truncate_prompt_tokens)
150
151 generators.append(
152 self.engine.generate(prompt,
vllm/entrypoints/openai/serving_engine.py
+18 -4
@@ -4,6 +4,8 @@ from dataclasses import dataclass
4 from http import HTTPStatus
5 from typing import Dict, List, Optional, Union
6
7 +from pydantic import conint
8 +
9 from vllm.engine.async_llm_engine import AsyncLLMEngine
10 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
11 CompletionRequest, ErrorResponse,
@@ -66,7 +68,8 @@ class OpenAIServing:
68 self.tokenizer = get_tokenizer(
69 engine_model_config.tokenizer,
70 tokenizer_mode=engine_model_config.tokenizer_mode,
69 - trust_remote_code=engine_model_config.trust_remote_code)
71 + trust_remote_code=engine_model_config.trust_remote_code,
72 + truncation_side="left")
73
74 async def show_available_models(self) -> ModelList:
75 """Show available models. Right now we only have one model."""
@@ -164,15 +167,26 @@ class OpenAIServing:
167 self,
168 request: Union[ChatCompletionRequest, CompletionRequest],
169 prompt: Optional[str] = None,
167 - prompt_ids: Optional[List[int]] = None) -> List[int]:
170 + prompt_ids: Optional[List[int]] = None,
171 + truncate_prompt_tokens: Optional[conint(ge=1)] = None
172 + ) -> List[int]:
173 if not (prompt or prompt_ids):
174 raise ValueError("Either prompt or prompt_ids should be provided.")
175 if (prompt and prompt_ids):
176 raise ValueError(
177 "Only one of prompt or prompt_ids should be provided.")
178
174 - input_ids = prompt_ids if prompt_ids is not None else self.tokenizer(
175 - prompt).input_ids
179 + if prompt_ids is None:
180 + tokenizer_kwargs = {} if truncate_prompt_tokens is None else {
181 + "truncation": True,
182 + "max_length": truncate_prompt_tokens,
183 + }
184 + input_ids = self.tokenizer(prompt, **tokenizer_kwargs).input_ids
185 + elif truncate_prompt_tokens is not None:
186 + input_ids = prompt_ids[-truncate_prompt_tokens:]
187 + else:
188 + input_ids = prompt_ids
189 +
190 token_num = len(input_ids)
191
192 if request.max_tokens is None:
vllm/sampling_params.py
+12 -1
@@ -5,6 +5,7 @@ from functools import cached_property
5 from typing import Callable, List, Optional, Union
6
7 import torch
8 +from pydantic import conint
9
10 _SAMPLING_EPS = 1e-5
11
@@ -94,6 +95,9 @@ class SamplingParams:
95 tokens in the output. Defaults to True.
96 logits_processors: List of functions that modify logits based on
97 previously generated tokens.
98 + truncate_prompt_tokens: If set to an integer k, will use only the last k
99 + tokens from the prompt (i.e., left truncation). Defaults to None
100 + (i.e., no truncation).
101 """
102
103 def __init__(
@@ -123,6 +127,7 @@ class SamplingParams:
127 skip_special_tokens: bool = True,
128 spaces_between_special_tokens: bool = True,
129 logits_processors: Optional[List[LogitsProcessor]] = None,
130 + truncate_prompt_tokens: Optional[conint(ge=1)] = None,
131 ) -> None:
132 self.n = n
133 self.best_of = best_of if best_of is not None else n
@@ -160,6 +165,7 @@ class SamplingParams:
165 self.spaces_between_special_tokens = spaces_between_special_tokens
166 self.logits_processors = logits_processors
167 self.include_stop_str_in_output = include_stop_str_in_output
168 + self.truncate_prompt_tokens = truncate_prompt_tokens
169 self._verify_args()
170 if self.use_beam_search:
171 self._verify_beam_search()
@@ -216,6 +222,10 @@ class SamplingParams:
222 if self.prompt_logprobs is not None and self.prompt_logprobs < 0:
223 raise ValueError(f"prompt_logprobs must be non-negative, got "
224 f"{self.prompt_logprobs}.")
225 + if (self.truncate_prompt_tokens is not None
226 + and self.truncate_prompt_tokens < 1):
227 + raise ValueError(f"truncate_prompt_tokens must be >= 1, "
228 + f"got {self.truncate_prompt_tokens}")
229 if self.stop and not self.detokenize:
230 raise ValueError(
231 "stop strings are only supported when detokenize is True. "
@@ -300,4 +310,5 @@ class SamplingParams:
310 f"prompt_logprobs={self.prompt_logprobs}, "
311 f"skip_special_tokens={self.skip_special_tokens}, "
312 "spaces_between_special_tokens="
303 - f"{self.spaces_between_special_tokens})")
313 + f"{self.spaces_between_special_tokens}, "
314 + f"truncate_prompt_tokens={self.truncate_prompt_tokens})")