@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7013a80170
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add support for `spaces_between_special_tokens`

Dan Lord committed Oct 30, 2023 at 16:52 UTC 7013a80170146ca4f7fd9539e4d7fd852d4718b6
5 files changed +28 -7
vllm/engine/llm_engine.py
+3 -3
@@ -632,8 +632,7 @@ class LLMEngine:
632 f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
633 self.last_logging_time = now
634
635 - def _decode_sequence(self, seq: Sequence,
636 - sampling_params: SamplingParams) -> None:
635 + def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
636 """Decodes the new token for a sequence."""
637 (new_tokens, new_output_text, prefix_offset,
638 read_offset) = detokenize_incrementally(
@@ -642,7 +641,8 @@ class LLMEngine:
641 prev_tokens=seq.tokens,
642 prefix_offset=seq.prefix_offset,
643 read_offset=seq.read_offset,
645 - skip_special_tokens=sampling_params.skip_special_tokens,
644 + skip_special_tokens=prms.skip_special_tokens,
645 + spaces_between_special_tokens=prms.spaces_between_special_tokens,
646 )
647 if seq.tokens is None:
648 seq.tokens = new_tokens
vllm/entrypoints/openai/api_server.py
+4
@@ -212,6 +212,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
212 request_id = f"cmpl-{random_uuid()}"
213 created_time = int(time.monotonic())
214 try:
215 + spaces_between_special_tokens = request.spaces_between_special_tokens
216 sampling_params = SamplingParams(
217 n=request.n,
218 presence_penalty=request.presence_penalty,
@@ -226,6 +227,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
227 ignore_eos=request.ignore_eos,
228 use_beam_search=request.use_beam_search,
229 skip_special_tokens=request.skip_special_tokens,
230 + spaces_between_special_tokens=spaces_between_special_tokens,
231 )
232 except ValueError as e:
233 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
@@ -413,6 +415,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
415
416 created_time = int(time.monotonic())
417 try:
418 + spaces_between_special_tokens = request.spaces_between_special_tokens
419 sampling_params = SamplingParams(
420 n=request.n,
421 best_of=request.best_of,
@@ -428,6 +431,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
431 logprobs=request.logprobs,
432 use_beam_search=request.use_beam_search,
433 skip_special_tokens=request.skip_special_tokens,
434 + spaces_between_special_tokens=spaces_between_special_tokens,
435 )
436 except ValueError as e:
437 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
vllm/entrypoints/openai/protocol.py
+2
@@ -72,6 +72,7 @@ class ChatCompletionRequest(BaseModel):
72 use_beam_search: Optional[bool] = False
73 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
74 skip_special_tokens: Optional[bool] = True
75 + spaces_between_special_tokens: Optional[bool] = True
76
77
78 class CompletionRequest(BaseModel):
@@ -98,6 +99,7 @@ class CompletionRequest(BaseModel):
99 use_beam_search: Optional[bool] = False
100 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
101 skip_special_tokens: Optional[bool] = True
102 + spaces_between_special_tokens: Optional[bool] = True
103
104
105 class LogProbs(BaseModel):
vllm/sampling_params.py
+7 -1
@@ -71,6 +71,8 @@ class SamplingParams:
71 `logprobs+1` elements in the response.
72 prompt_logprobs: Number of log probabilities to return per prompt token.
73 skip_special_tokens: Whether to skip special tokens in the output.
74 + spaces_between_special_tokens: Whether to add spaces between special
75 + tokens in the output. Defaults to True.
76 """
77
78 def __init__(
@@ -93,6 +95,7 @@ class SamplingParams:
95 logprobs: Optional[int] = None,
96 prompt_logprobs: Optional[int] = None,
97 skip_special_tokens: bool = True,
98 + spaces_between_special_tokens: bool = True,
99 ) -> None:
100 self.n = n
101 self.best_of = best_of if best_of is not None else n
@@ -120,6 +123,7 @@ class SamplingParams:
123 self.logprobs = logprobs
124 self.prompt_logprobs = prompt_logprobs
125 self.skip_special_tokens = skip_special_tokens
126 + self.spaces_between_special_tokens = spaces_between_special_tokens
127
128 self._verify_args()
129 if self.use_beam_search:
@@ -222,4 +226,6 @@ class SamplingParams:
226 f"max_tokens={self.max_tokens}, "
227 f"logprobs={self.logprobs}, "
228 f"prompt_logprobs={self.prompt_logprobs}, "
225 - f"skip_special_tokens={self.skip_special_tokens})")
229 + f"skip_special_tokens={self.skip_special_tokens}, "
230 + "spaces_between_special_tokens="
231 + f"{self.spaces_between_special_tokens})")
vllm/transformers_utils/tokenizer.py
+12 -3
@@ -73,6 +73,7 @@ def _convert_tokens_to_string_with_added_encoders(
73 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
74 output_tokens: List[str],
75 skip_special_tokens: bool,
76 + spaces_between_special_tokens: bool,
77 ) -> str:
78 # Adapted from
79 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
@@ -96,7 +97,10 @@ def _convert_tokens_to_string_with_added_encoders(
97 if current_sub_text:
98 sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
99 sub_texts.append(sub_text)
99 - return " ".join(sub_texts)
100 + if spaces_between_special_tokens:
101 + return " ".join(sub_texts)
102 + else:
103 + return "".join(sub_texts)
104
105
106 # Based on
@@ -109,6 +113,7 @@ def detokenize_incrementally(
113 prefix_offset: int = 0,
114 read_offset: int = 0,
115 skip_special_tokens: bool = False,
116 + spaces_between_special_tokens: bool = True,
117 ) -> Tuple[List[str], str, int, int]:
118 new_token_id = all_input_ids[-1]
119 # This is the first iteration for this sequence
@@ -143,11 +148,15 @@ def detokenize_incrementally(
148 prefix_text = _convert_tokens_to_string_with_added_encoders(
149 tokenizer,
150 output_tokens[prefix_offset:read_offset],
146 - skip_special_tokens=skip_special_tokens)
151 + skip_special_tokens=skip_special_tokens,
152 + spaces_between_special_tokens=spaces_between_special_tokens,
153 + )
154 new_text = _convert_tokens_to_string_with_added_encoders(
155 tokenizer,
156 output_tokens[prefix_offset:],
150 - skip_special_tokens=skip_special_tokens)
157 + skip_special_tokens=skip_special_tokens,
158 + spaces_between_special_tokens=spaces_between_special_tokens,
159 + )
160
161 if len(new_text) > len(prefix_text) and not new_text.endswith("�"):
162 # utf-8 char at the end means it's a potential unfinished byte sequence