@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e86717833d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Incrementally decode output tokens (#121)

Woosuk Kwon committed May 23, 2023 at 20:46 UTC e86717833da1216222cf0d490c2e3ba198610b13
4 files changed +83 -17
cacheflow/core/scheduler.py
+1 -1
@@ -291,7 +291,7 @@ class Scheduler:
291 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
292 # Append a new token to the sequence.
293 output = seq_outputs[seq.seq_id]
294 - seq.append_token(output.output_token, output.logprobs)
294 + seq.append_token_id(output.output_token, output.logprobs)
295 return self.running.copy()
296
297 def free_seq(self, seq: Sequence) -> None:
cacheflow/sequence.py
+8 -3
@@ -24,7 +24,7 @@ class SequenceData:
24 self.output_token_ids: List[int] = []
25 self.cumulative_logprob = 0.0
26
27 - def append_token(self, token_id: int, logprob: float) -> None:
27 + def append_token_id(self, token_id: int, logprob: float) -> None:
28 self.output_token_ids.append(token_id)
29 self.cumulative_logprob += logprob
30
@@ -64,6 +64,7 @@ class Sequence:
64
65 self.data = SequenceData(prompt_token_ids)
66 self.output_logprobs: List[Dict[int, float]] = []
67 + self.output_tokens: List[str] = []
68 self.output_text = ""
69
70 self.logical_token_blocks: List[LogicalTokenBlock] = []
@@ -92,11 +93,15 @@ class Sequence:
93 last_block.append_tokens(token_ids[:num_empty_slots])
94 token_ids = token_ids[num_empty_slots:]
95
95 - def append_token(self, token_id: int, logprobs: Dict[int, float]) -> None:
96 + def append_token_id(
97 + self,
98 + token_id: int,
99 + logprobs: Dict[int, float],
100 + ) -> None:
101 assert token_id in logprobs
102 self._append_tokens_to_blocks([token_id])
103 self.output_logprobs.append(logprobs)
99 - self.data.append_token(token_id, logprobs[token_id])
104 + self.data.append_token_id(token_id, logprobs[token_id])
105
106 def get_len(self) -> int:
107 return self.data.get_len()
cacheflow/server/llm_server.py
+12 -12
@@ -14,7 +14,8 @@ from cacheflow.outputs import RequestOutput
14 from cacheflow.sampling_params import SamplingParams
15 from cacheflow.server.arg_utils import ServerArgs
16 from cacheflow.server.ray_utils import initialize_cluster
17 -from cacheflow.server.tokenizer_utils import get_tokenizer
17 +from cacheflow.server.tokenizer_utils import (get_tokenizer,
18 + detokenize_incrementally)
19 from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
20 from cacheflow.utils import Counter
21 from cacheflow.worker.worker import Worker
@@ -185,18 +186,17 @@ class LLMServer:
186 return request_outputs
187
188 def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
188 - # Batch-decode the sequence outputs.
189 - seqs: List[Sequence] = []
189 + # Decode the sequence outputs.
190 for seq_group in seq_groups:
191 - seqs.extend(seq_group.get_seqs(status=SequenceStatus.RUNNING))
192 - output_tokens_per_seq = []
193 - for seq in seqs:
194 - output_tokens_per_seq.append(seq.get_output_token_ids())
195 - output_texts = self.tokenizer.batch_decode(output_tokens_per_seq,
196 - skip_special_tokens=True)
197 - # Update the sequences with the output texts.
198 - for seq, output_text in zip(seqs, output_texts):
199 - seq.output_text = output_text
191 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
192 + new_token, new_output_text = detokenize_incrementally(
193 + self.tokenizer,
194 + seq.output_tokens,
195 + seq.get_last_token_id(),
196 + skip_special_tokens=True,
197 + )
198 + seq.output_tokens.append(new_token)
199 + seq.output_text = new_output_text
200
201 def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
202 # Stop the sequences.
cacheflow/server/tokenizer_utils.py
+62 -1
@@ -1,8 +1,12 @@
1 -from typing import Union
1 +from typing import List, Tuple, Union
2
3 from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
4 PreTrainedTokenizerFast)
5
6 +from cacheflow.logger import init_logger
7 +
8 +logger = init_logger(__name__)
9 +
10 _MODEL_TYPES_WITH_SLOW_TOKENIZER = [
11 # LLaMA fast tokenizer has a bug related to protobuf.
12 # See https://github.com/WoosukKwon/cacheflow/issues/80#issue-1698550554
@@ -17,5 +21,62 @@ def get_tokenizer(
21 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
22 config = AutoConfig.from_pretrained(model_name)
23 if config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
24 + if getattr(kwargs, "use_fast", False) == True:
25 + raise ValueError(
26 + f"Cannot use the fast tokenizer for {config.model_type} due to "
27 + "bugs in the fast tokenizer.")
28 + logger.info(
29 + f"Using the slow tokenizer for {config.model_type} due to bugs in "
30 + "the fast tokenizer. This could potentially lead to performance "
31 + "degradation.")
32 kwargs["use_fast"] = False
33 return AutoTokenizer.from_pretrained(model_name, *args, **kwargs)
34 +
35 +
36 +def detokenize_incrementally(
37 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
38 + prev_output_tokens: List[str],
39 + new_token_id: int,
40 + skip_special_tokens: bool,
41 +) -> Tuple[str, str]:
42 + """Detokenizes the new token in conjuction with the previous output tokens.
43 +
44 + NOTE: This function does not update prev_output_tokens.
45 +
46 + Returns:
47 + new_token: The new token as a string.
48 + output_text: The new output text as a string.
49 + """
50 + new_token = tokenizer.convert_ids_to_tokens(
51 + new_token_id, skip_special_tokens=skip_special_tokens)
52 + output_tokens = prev_output_tokens + [new_token]
53 +
54 + # Convert the tokens to a string.
55 + # Optimization: If the tokenizer does not have `added_tokens_encoder`,
56 + # then we can directly use `convert_tokens_to_string`.
57 + if not getattr(tokenizer, "added_tokens_encoder", {}):
58 + output_text = tokenizer.convert_tokens_to_string(output_tokens)
59 + return new_token, output_text
60 +
61 + # Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
62 + # NOTE(woosuk): The following code is slow because it runs a for loop over
63 + # the output_tokens. In Python, running a for loop over a list can be slow
64 + # even when the loop body is very simple.
65 + sub_texts = []
66 + current_sub_text = []
67 + for token in output_tokens:
68 + if skip_special_tokens and token in tokenizer.all_special_ids:
69 + continue
70 + if token in tokenizer.added_tokens_encoder:
71 + if current_sub_text:
72 + sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
73 + sub_texts.append(sub_text)
74 + current_sub_text = []
75 + sub_texts.append(token)
76 + else:
77 + current_sub_text.append(token)
78 + if current_sub_text:
79 + sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
80 + sub_texts.append(sub_text)
81 + output_text = " ".join(sub_texts)
82 + return new_token, output_text