A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Incrementally decode output tokens (#121)
Woosuk Kwon committed
May 23, 2023 at 20:46 UTC
e86717833da1216222cf0d490c2e3ba198610b13
4 files changed
+83
-17
cacheflow/core/scheduler.py
+1
-1
@@ -291,7 +291,7 @@ class Scheduler:
291
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
292
# Append a new token to the sequence.
293
output = seq_outputs[seq.seq_id]
294
- seq.append_token(output.output_token, output.logprobs)
294
+ seq.append_token_id(output.output_token, output.logprobs)
295
return self.running.copy()
296
297
def free_seq(self, seq: Sequence) -> None:
cacheflow/sequence.py
+8
-3
@@ -24,7 +24,7 @@ class SequenceData:
24
self.output_token_ids: List[int] = []
25
self.cumulative_logprob = 0.0
26
27
- def append_token(self, token_id: int, logprob: float) -> None:
27
+ def append_token_id(self, token_id: int, logprob: float) -> None:
28
self.output_token_ids.append(token_id)
29
self.cumulative_logprob += logprob
30
@@ -64,6 +64,7 @@ class Sequence:
64
65
self.data = SequenceData(prompt_token_ids)
66
self.output_logprobs: List[Dict[int, float]] = []
67
+ self.output_tokens: List[str] = []
68
self.output_text = ""
69
70
self.logical_token_blocks: List[LogicalTokenBlock] = []
@@ -92,11 +93,15 @@ class Sequence:
93
last_block.append_tokens(token_ids[:num_empty_slots])
94
token_ids = token_ids[num_empty_slots:]
95
95
- def append_token(self, token_id: int, logprobs: Dict[int, float]) -> None:
96
+ def append_token_id(
97
+ self,
98
+ token_id: int,
99
+ logprobs: Dict[int, float],
100
+ ) -> None:
101
assert token_id in logprobs
102
self._append_tokens_to_blocks([token_id])
103
self.output_logprobs.append(logprobs)
99
- self.data.append_token(token_id, logprobs[token_id])
104
+ self.data.append_token_id(token_id, logprobs[token_id])
105
106
def get_len(self) -> int:
107
return self.data.get_len()
cacheflow/server/llm_server.py
+12
-12
@@ -14,7 +14,8 @@ from cacheflow.outputs import RequestOutput
14
from cacheflow.sampling_params import SamplingParams
15
from cacheflow.server.arg_utils import ServerArgs
16
from cacheflow.server.ray_utils import initialize_cluster
17
-from cacheflow.server.tokenizer_utils import get_tokenizer
17
+from cacheflow.server.tokenizer_utils import (get_tokenizer,
18
+ detokenize_incrementally)
19
from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
20
from cacheflow.utils import Counter
21
from cacheflow.worker.worker import Worker
@@ -185,18 +186,17 @@ class LLMServer:
186
return request_outputs
187
188
def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
188
- # Batch-decode the sequence outputs.
189
- seqs: List[Sequence] = []
189
+ # Decode the sequence outputs.
190
for seq_group in seq_groups:
191
- seqs.extend(seq_group.get_seqs(status=SequenceStatus.RUNNING))
192
- output_tokens_per_seq = []
193
- for seq in seqs:
194
- output_tokens_per_seq.append(seq.get_output_token_ids())
195
- output_texts = self.tokenizer.batch_decode(output_tokens_per_seq,
196
- skip_special_tokens=True)
197
- # Update the sequences with the output texts.
198
- for seq, output_text in zip(seqs, output_texts):
199
- seq.output_text = output_text
191
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
192
+ new_token, new_output_text = detokenize_incrementally(
193
+ self.tokenizer,
194
+ seq.output_tokens,
195
+ seq.get_last_token_id(),
196
+ skip_special_tokens=True,
197
+ )
198
+ seq.output_tokens.append(new_token)
199
+ seq.output_text = new_output_text
200
201
def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
202
# Stop the sequences.
cacheflow/server/tokenizer_utils.py
+62
-1
@@ -1,8 +1,12 @@
1
-from typing import Union
1
+from typing import List, Tuple, Union
2
3
from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
4
PreTrainedTokenizerFast)
5
6
+from cacheflow.logger import init_logger
7
+
8
+logger = init_logger(__name__)
9
+
10
_MODEL_TYPES_WITH_SLOW_TOKENIZER = [
11
# LLaMA fast tokenizer has a bug related to protobuf.
12
# See https://github.com/WoosukKwon/cacheflow/issues/80#issue-1698550554
@@ -17,5 +21,62 @@ def get_tokenizer(
21
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
22
config = AutoConfig.from_pretrained(model_name)
23
if config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
24
+ if getattr(kwargs, "use_fast", False) == True:
25
+ raise ValueError(
26
+ f"Cannot use the fast tokenizer for {config.model_type} due to "
27
+ "bugs in the fast tokenizer.")
28
+ logger.info(
29
+ f"Using the slow tokenizer for {config.model_type} due to bugs in "
30
+ "the fast tokenizer. This could potentially lead to performance "
31
+ "degradation.")
32
kwargs["use_fast"] = False
33
return AutoTokenizer.from_pretrained(model_name, *args, **kwargs)
34
+
35
+
36
+def detokenize_incrementally(
37
+ tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
38
+ prev_output_tokens: List[str],
39
+ new_token_id: int,
40
+ skip_special_tokens: bool,
41
+) -> Tuple[str, str]:
42
+ """Detokenizes the new token in conjuction with the previous output tokens.
43
+
44
+ NOTE: This function does not update prev_output_tokens.
45
+
46
+ Returns:
47
+ new_token: The new token as a string.
48
+ output_text: The new output text as a string.
49
+ """
50
+ new_token = tokenizer.convert_ids_to_tokens(
51
+ new_token_id, skip_special_tokens=skip_special_tokens)
52
+ output_tokens = prev_output_tokens + [new_token]
53
+
54
+ # Convert the tokens to a string.
55
+ # Optimization: If the tokenizer does not have `added_tokens_encoder`,
56
+ # then we can directly use `convert_tokens_to_string`.
57
+ if not getattr(tokenizer, "added_tokens_encoder", {}):
58
+ output_text = tokenizer.convert_tokens_to_string(output_tokens)
59
+ return new_token, output_text
60
+
61
+ # Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
62
+ # NOTE(woosuk): The following code is slow because it runs a for loop over
63
+ # the output_tokens. In Python, running a for loop over a list can be slow
64
+ # even when the loop body is very simple.
65
+ sub_texts = []
66
+ current_sub_text = []
67
+ for token in output_tokens:
68
+ if skip_special_tokens and token in tokenizer.all_special_ids:
69
+ continue
70
+ if token in tokenizer.added_tokens_encoder:
71
+ if current_sub_text:
72
+ sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
73
+ sub_texts.append(sub_text)
74
+ current_sub_text = []
75
+ sub_texts.append(token)
76
+ else:
77
+ current_sub_text.append(token)
78
+ if current_sub_text:
79
+ sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
80
+ sub_texts.append(sub_text)
81
+ output_text = " ".join(sub_texts)
82
+ return new_token, output_text