@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 20f7cc4cde
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add `skip_special_tokens` sampling params (#1186)

Dan Lord committed Sep 27, 2023 at 19:21 UTC 20f7cc4cdebcbcad788fbe1b06e5e07f8d145b77
4 files changed +14 -4
vllm/engine/llm_engine.py
+4 -3
@@ -387,7 +387,7 @@ class LLMEngine:
387 child_seqs.append((parent, parent))
388
389 for seq, _ in child_seqs:
390 - self._decode_sequence(seq)
390 + self._decode_sequence(seq, seq_group.sampling_params)
391 self._check_stop(seq, seq_group.sampling_params)
392
393 # Non-beam search case
@@ -621,7 +621,8 @@ class LLMEngine:
621 f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
622 self.last_logging_time = now
623
624 - def _decode_sequence(self, seq: Sequence) -> None:
624 + def _decode_sequence(self, seq: Sequence,
625 + sampling_params: SamplingParams) -> None:
626 """Decodes the new token for a sequence."""
627 (new_tokens, new_output_text, prefix_offset,
628 read_offset) = detokenize_incrementally(
@@ -630,7 +631,7 @@ class LLMEngine:
631 prev_tokens=seq.tokens,
632 prefix_offset=seq.prefix_offset,
633 read_offset=seq.read_offset,
633 - skip_special_tokens=True,
634 + skip_special_tokens=sampling_params.skip_special_tokens,
635 )
636 if seq.tokens is None:
637 seq.tokens = new_tokens
vllm/entrypoints/openai/api_server.py
+2
@@ -225,6 +225,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
225 top_k=request.top_k,
226 ignore_eos=request.ignore_eos,
227 use_beam_search=request.use_beam_search,
228 + skip_special_tokens=request.skip_special_tokens,
229 )
230 except ValueError as e:
231 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
@@ -426,6 +427,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
427 max_tokens=request.max_tokens,
428 logprobs=request.logprobs,
429 use_beam_search=request.use_beam_search,
430 + skip_special_tokens=request.skip_special_tokens,
431 )
432 except ValueError as e:
433 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
vllm/entrypoints/openai/protocol.py
+2
@@ -71,6 +71,7 @@ class ChatCompletionRequest(BaseModel):
71 ignore_eos: Optional[bool] = False
72 use_beam_search: Optional[bool] = False
73 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
74 + skip_special_tokens: Optional[bool] = True
75
76
77 class CompletionRequest(BaseModel):
@@ -96,6 +97,7 @@ class CompletionRequest(BaseModel):
97 ignore_eos: Optional[bool] = False
98 use_beam_search: Optional[bool] = False
99 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
100 + skip_special_tokens: Optional[bool] = True
101
102
103 class LogProbs(BaseModel):
vllm/sampling_params.py
+6 -1
@@ -60,6 +60,8 @@ class SamplingParams:
60 tokens after the EOS token is generated.
61 max_tokens: Maximum number of tokens to generate per output sequence.
62 logprobs: Number of log probabilities to return per output token.
63 + skip_special_tokens: Whether to skip special tokens in the output.
64 + Defaults to true.
65 """
66
67 def __init__(
@@ -79,6 +81,7 @@ class SamplingParams:
81 ignore_eos: bool = False,
82 max_tokens: int = 16,
83 logprobs: Optional[int] = None,
84 + skip_special_tokens: bool = True,
85 ) -> None:
86 self.n = n
87 self.best_of = best_of if best_of is not None else n
@@ -103,6 +106,7 @@ class SamplingParams:
106 self.ignore_eos = ignore_eos
107 self.max_tokens = max_tokens
108 self.logprobs = logprobs
109 + self.skip_special_tokens = skip_special_tokens
110
111 self._verify_args()
112 if self.use_beam_search:
@@ -196,4 +200,5 @@ class SamplingParams:
200 f"stop={self.stop}, "
201 f"ignore_eos={self.ignore_eos}, "
202 f"max_tokens={self.max_tokens}, "
199 - f"logprobs={self.logprobs})")
203 + f"logprobs={self.logprobs}, "
204 + f"skip_special_tokens={self.skip_special_tokens})")