A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add support for `spaces_between_special_tokens`
Dan Lord committed
Oct 30, 2023 at 16:52 UTC
7013a80170146ca4f7fd9539e4d7fd852d4718b6
5 files changed
+28
-7
vllm/engine/llm_engine.py
+3
-3
@@ -632,8 +632,7 @@ class LLMEngine:
632
f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
633
self.last_logging_time = now
634
635
- def _decode_sequence(self, seq: Sequence,
636
- sampling_params: SamplingParams) -> None:
635
+ def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
636
"""Decodes the new token for a sequence."""
637
(new_tokens, new_output_text, prefix_offset,
638
read_offset) = detokenize_incrementally(
@@ -642,7 +641,8 @@ class LLMEngine:
641
prev_tokens=seq.tokens,
642
prefix_offset=seq.prefix_offset,
643
read_offset=seq.read_offset,
645
- skip_special_tokens=sampling_params.skip_special_tokens,
644
+ skip_special_tokens=prms.skip_special_tokens,
645
+ spaces_between_special_tokens=prms.spaces_between_special_tokens,
646
)
647
if seq.tokens is None:
648
seq.tokens = new_tokens
vllm/entrypoints/openai/api_server.py
+4
@@ -212,6 +212,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
212
request_id = f"cmpl-{random_uuid()}"
213
created_time = int(time.monotonic())
214
try:
215
+ spaces_between_special_tokens = request.spaces_between_special_tokens
216
sampling_params = SamplingParams(
217
n=request.n,
218
presence_penalty=request.presence_penalty,
@@ -226,6 +227,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
227
ignore_eos=request.ignore_eos,
228
use_beam_search=request.use_beam_search,
229
skip_special_tokens=request.skip_special_tokens,
230
+ spaces_between_special_tokens=spaces_between_special_tokens,
231
)
232
except ValueError as e:
233
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
@@ -413,6 +415,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
415
416
created_time = int(time.monotonic())
417
try:
418
+ spaces_between_special_tokens = request.spaces_between_special_tokens
419
sampling_params = SamplingParams(
420
n=request.n,
421
best_of=request.best_of,
@@ -428,6 +431,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
431
logprobs=request.logprobs,
432
use_beam_search=request.use_beam_search,
433
skip_special_tokens=request.skip_special_tokens,
434
+ spaces_between_special_tokens=spaces_between_special_tokens,
435
)
436
except ValueError as e:
437
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
vllm/entrypoints/openai/protocol.py
+2
@@ -72,6 +72,7 @@ class ChatCompletionRequest(BaseModel):
72
use_beam_search: Optional[bool] = False
73
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
74
skip_special_tokens: Optional[bool] = True
75
+ spaces_between_special_tokens: Optional[bool] = True
76
77
78
class CompletionRequest(BaseModel):
@@ -98,6 +99,7 @@ class CompletionRequest(BaseModel):
99
use_beam_search: Optional[bool] = False
100
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
101
skip_special_tokens: Optional[bool] = True
102
+ spaces_between_special_tokens: Optional[bool] = True
103
104
105
class LogProbs(BaseModel):
vllm/sampling_params.py
+7
-1
@@ -71,6 +71,8 @@ class SamplingParams:
71
`logprobs+1` elements in the response.
72
prompt_logprobs: Number of log probabilities to return per prompt token.
73
skip_special_tokens: Whether to skip special tokens in the output.
74
+ spaces_between_special_tokens: Whether to add spaces between special
75
+ tokens in the output. Defaults to True.
76
"""
77
78
def __init__(
@@ -93,6 +95,7 @@ class SamplingParams:
95
logprobs: Optional[int] = None,
96
prompt_logprobs: Optional[int] = None,
97
skip_special_tokens: bool = True,
98
+ spaces_between_special_tokens: bool = True,
99
) -> None:
100
self.n = n
101
self.best_of = best_of if best_of is not None else n
@@ -120,6 +123,7 @@ class SamplingParams:
123
self.logprobs = logprobs
124
self.prompt_logprobs = prompt_logprobs
125
self.skip_special_tokens = skip_special_tokens
126
+ self.spaces_between_special_tokens = spaces_between_special_tokens
127
128
self._verify_args()
129
if self.use_beam_search:
@@ -222,4 +226,6 @@ class SamplingParams:
226
f"max_tokens={self.max_tokens}, "
227
f"logprobs={self.logprobs}, "
228
f"prompt_logprobs={self.prompt_logprobs}, "
225
- f"skip_special_tokens={self.skip_special_tokens})")
229
+ f"skip_special_tokens={self.skip_special_tokens}, "
230
+ "spaces_between_special_tokens="
231
+ f"{self.spaces_between_special_tokens})")
vllm/transformers_utils/tokenizer.py
+12
-3
@@ -73,6 +73,7 @@ def _convert_tokens_to_string_with_added_encoders(
73
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
74
output_tokens: List[str],
75
skip_special_tokens: bool,
76
+ spaces_between_special_tokens: bool,
77
) -> str:
78
# Adapted from
79
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
@@ -96,7 +97,10 @@ def _convert_tokens_to_string_with_added_encoders(
97
if current_sub_text:
98
sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
99
sub_texts.append(sub_text)
99
- return " ".join(sub_texts)
100
+ if spaces_between_special_tokens:
101
+ return " ".join(sub_texts)
102
+ else:
103
+ return "".join(sub_texts)
104
105
106
# Based on
@@ -109,6 +113,7 @@ def detokenize_incrementally(
113
prefix_offset: int = 0,
114
read_offset: int = 0,
115
skip_special_tokens: bool = False,
116
+ spaces_between_special_tokens: bool = True,
117
) -> Tuple[List[str], str, int, int]:
118
new_token_id = all_input_ids[-1]
119
# This is the first iteration for this sequence
@@ -143,11 +148,15 @@ def detokenize_incrementally(
148
prefix_text = _convert_tokens_to_string_with_added_encoders(
149
tokenizer,
150
output_tokens[prefix_offset:read_offset],
146
- skip_special_tokens=skip_special_tokens)
151
+ skip_special_tokens=skip_special_tokens,
152
+ spaces_between_special_tokens=spaces_between_special_tokens,
153
+ )
154
new_text = _convert_tokens_to_string_with_added_encoders(
155
tokenizer,
156
output_tokens[prefix_offset:],
150
- skip_special_tokens=skip_special_tokens)
157
+ skip_special_tokens=skip_special_tokens,
158
+ spaces_between_special_tokens=spaces_between_special_tokens,
159
+ )
160
161
if len(new_text) > len(prefix_text) and not new_text.endswith("�"):
162
# utf-8 char at the end means it's a potential unfinished byte sequence