A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add `skip_special_tokens` sampling params (#1186)
Dan Lord committed
Sep 27, 2023 at 19:21 UTC
20f7cc4cdebcbcad788fbe1b06e5e07f8d145b77
4 files changed
+14
-4
vllm/engine/llm_engine.py
+4
-3
@@ -387,7 +387,7 @@ class LLMEngine:
387
child_seqs.append((parent, parent))
388
389
for seq, _ in child_seqs:
390
- self._decode_sequence(seq)
390
+ self._decode_sequence(seq, seq_group.sampling_params)
391
self._check_stop(seq, seq_group.sampling_params)
392
393
# Non-beam search case
@@ -621,7 +621,8 @@ class LLMEngine:
621
f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
622
self.last_logging_time = now
623
624
- def _decode_sequence(self, seq: Sequence) -> None:
624
+ def _decode_sequence(self, seq: Sequence,
625
+ sampling_params: SamplingParams) -> None:
626
"""Decodes the new token for a sequence."""
627
(new_tokens, new_output_text, prefix_offset,
628
read_offset) = detokenize_incrementally(
@@ -630,7 +631,7 @@ class LLMEngine:
631
prev_tokens=seq.tokens,
632
prefix_offset=seq.prefix_offset,
633
read_offset=seq.read_offset,
633
- skip_special_tokens=True,
634
+ skip_special_tokens=sampling_params.skip_special_tokens,
635
)
636
if seq.tokens is None:
637
seq.tokens = new_tokens
vllm/entrypoints/openai/api_server.py
+2
@@ -225,6 +225,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
225
top_k=request.top_k,
226
ignore_eos=request.ignore_eos,
227
use_beam_search=request.use_beam_search,
228
+ skip_special_tokens=request.skip_special_tokens,
229
)
230
except ValueError as e:
231
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
@@ -426,6 +427,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
427
max_tokens=request.max_tokens,
428
logprobs=request.logprobs,
429
use_beam_search=request.use_beam_search,
430
+ skip_special_tokens=request.skip_special_tokens,
431
)
432
except ValueError as e:
433
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
vllm/entrypoints/openai/protocol.py
+2
@@ -71,6 +71,7 @@ class ChatCompletionRequest(BaseModel):
71
ignore_eos: Optional[bool] = False
72
use_beam_search: Optional[bool] = False
73
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
74
+ skip_special_tokens: Optional[bool] = True
75
76
77
class CompletionRequest(BaseModel):
@@ -96,6 +97,7 @@ class CompletionRequest(BaseModel):
97
ignore_eos: Optional[bool] = False
98
use_beam_search: Optional[bool] = False
99
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
100
+ skip_special_tokens: Optional[bool] = True
101
102
103
class LogProbs(BaseModel):
vllm/sampling_params.py
+6
-1
@@ -60,6 +60,8 @@ class SamplingParams:
60
tokens after the EOS token is generated.
61
max_tokens: Maximum number of tokens to generate per output sequence.
62
logprobs: Number of log probabilities to return per output token.
63
+ skip_special_tokens: Whether to skip special tokens in the output.
64
+ Defaults to true.
65
"""
66
67
def __init__(
@@ -79,6 +81,7 @@ class SamplingParams:
81
ignore_eos: bool = False,
82
max_tokens: int = 16,
83
logprobs: Optional[int] = None,
84
+ skip_special_tokens: bool = True,
85
) -> None:
86
self.n = n
87
self.best_of = best_of if best_of is not None else n
@@ -103,6 +106,7 @@ class SamplingParams:
106
self.ignore_eos = ignore_eos
107
self.max_tokens = max_tokens
108
self.logprobs = logprobs
109
+ self.skip_special_tokens = skip_special_tokens
110
111
self._verify_args()
112
if self.use_beam_search:
@@ -196,4 +200,5 @@ class SamplingParams:
200
f"stop={self.stop}, "
201
f"ignore_eos={self.ignore_eos}, "
202
f"max_tokens={self.max_tokens}, "
199
- f"logprobs={self.logprobs})")
203
+ f"logprobs={self.logprobs}, "
204
+ f"skip_special_tokens={self.skip_special_tokens})")