@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dd54a4b026
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix detokenization leaving special tokens (#1044)

Signed-off-by: Antoni Baum <antoni.baum@protonmail.com>

Antoni Baum committed Sep 14, 2023 at 16:37 UTC dd54a4b026455f728f9d5945eca369b2be7b12f9
2 files changed +14 -7
tests/engine/test_detokenize.py
+11 -4
@@ -23,7 +23,8 @@ TOKENIZERS = [
23 ]
24
25
26 -def _run_incremental_decode(tokenizer, all_input_ids):
26 +def _run_incremental_decode(tokenizer, all_input_ids,
27 + skip_special_tokens: bool):
28 decoded_text = ""
29 offset = 0
30 token_offset = 0
@@ -35,7 +36,7 @@ def _run_incremental_decode(tokenizer, all_input_ids):
36 prev_tokens,
37 offset,
38 token_offset,
38 - skip_special_tokens=False)
39 + skip_special_tokens=skip_special_tokens)
40 decoded_text += text
41 if prev_tokens is None:
42 prev_tokens = new_tokens
@@ -46,10 +47,16 @@ def _run_incremental_decode(tokenizer, all_input_ids):
47
48 @pytest.mark.parametrize("truth", TRUTH)
49 @pytest.mark.parametrize("tokenizer_id", TOKENIZERS)
49 -def test_decode_streaming(tokenizer_id, truth):
50 +@pytest.mark.parametrize("skip_special_tokens", (True, False))
51 +def test_decode_streaming(tokenizer_id, truth, skip_special_tokens):
52 tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
53 all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
54 + if skip_special_tokens:
55 + all_input_ids = ([tokenizer.bos_token_id]
56 + if tokenizer.bos_token_id is not None else
57 + []) + all_input_ids + [tokenizer.eos_token_id]
58
53 - decoded_text = _run_incremental_decode(tokenizer, all_input_ids)
59 + decoded_text = _run_incremental_decode(
60 + tokenizer, all_input_ids, skip_special_tokens=skip_special_tokens)
61
62 assert decoded_text == truth
vllm/transformers_utils/tokenizer.py
+3 -3
@@ -119,9 +119,9 @@ def detokenize_incrementally(
119 prefix_offset = max(len(output_tokens) - 6, 0)
120 read_offset = max(len(output_tokens) - 1, 0)
121 else:
122 - new_token = tokenizer.convert_ids_to_tokens(
123 - new_token_id, skip_special_tokens=skip_special_tokens)
124 - new_tokens = [new_token]
122 + # Put new_token_id in a list so skip_special_tokens is respected
123 + new_tokens = tokenizer.convert_ids_to_tokens(
124 + [new_token_id], skip_special_tokens=skip_special_tokens)
125 output_tokens = prev_tokens + new_tokens
126
127 # The prefix text is necessary only to defeat cleanup algorithms in