A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix detokenization leaving special tokens (#1044)
Signed-off-by: Antoni Baum <antoni.baum@protonmail.com>
Antoni Baum committed
Sep 14, 2023 at 16:37 UTC
dd54a4b026455f728f9d5945eca369b2be7b12f9
2 files changed
+14
-7
tests/engine/test_detokenize.py
+11
-4
@@ -23,7 +23,8 @@ TOKENIZERS = [
23
]
24
25
26
-def _run_incremental_decode(tokenizer, all_input_ids):
26
+def _run_incremental_decode(tokenizer, all_input_ids,
27
+ skip_special_tokens: bool):
28
decoded_text = ""
29
offset = 0
30
token_offset = 0
@@ -35,7 +36,7 @@ def _run_incremental_decode(tokenizer, all_input_ids):
36
prev_tokens,
37
offset,
38
token_offset,
38
- skip_special_tokens=False)
39
+ skip_special_tokens=skip_special_tokens)
40
decoded_text += text
41
if prev_tokens is None:
42
prev_tokens = new_tokens
@@ -46,10 +47,16 @@ def _run_incremental_decode(tokenizer, all_input_ids):
47
48
@pytest.mark.parametrize("truth", TRUTH)
49
@pytest.mark.parametrize("tokenizer_id", TOKENIZERS)
49
-def test_decode_streaming(tokenizer_id, truth):
50
+@pytest.mark.parametrize("skip_special_tokens", (True, False))
51
+def test_decode_streaming(tokenizer_id, truth, skip_special_tokens):
52
tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
53
all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
54
+ if skip_special_tokens:
55
+ all_input_ids = ([tokenizer.bos_token_id]
56
+ if tokenizer.bos_token_id is not None else
57
+ []) + all_input_ids + [tokenizer.eos_token_id]
58
53
- decoded_text = _run_incremental_decode(tokenizer, all_input_ids)
59
+ decoded_text = _run_incremental_decode(
60
+ tokenizer, all_input_ids, skip_special_tokens=skip_special_tokens)
61
62
assert decoded_text == truth
vllm/transformers_utils/tokenizer.py
+3
-3
@@ -119,9 +119,9 @@ def detokenize_incrementally(
119
prefix_offset = max(len(output_tokens) - 6, 0)
120
read_offset = max(len(output_tokens) - 1, 0)
121
else:
122
- new_token = tokenizer.convert_ids_to_tokens(
123
- new_token_id, skip_special_tokens=skip_special_tokens)
124
- new_tokens = [new_token]
122
+ # Put new_token_id in a list so skip_special_tokens is respected
123
+ new_tokens = tokenizer.convert_ids_to_tokens(
124
+ [new_token_id], skip_special_tokens=skip_special_tokens)
125
output_tokens = prev_tokens + new_tokens
126
127
# The prefix text is necessary only to defeat cleanup algorithms in