A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 06:00 UTC
[CI] Fix Bad_words test for tokenizer encode/decode asymmetry (#28193)
Signed-off-by: zhyajie <yajizhan@amd.com> Co-authored-by: zhyajie <yajizhan@amd.com>
杰兮 committed
Dec 2, 2025 at 16:02 UTC
48d15a32aa567dfc59ede46683b01cc2321579cb
1 file changed
+24
-3
tests/v1/sample/test_sampling_params_e2e.py
+24
-3
@@ -106,6 +106,25 @@ def test_detokenize_false(llm):
106
def test_bad_words(llm):
107
"""Check that we respect bad words."""
108
109
+ tokenizer = llm.get_tokenizer()
110
+
111
+ def contains_bad_word(text: str, tokens: list[int], bad_word: str) -> bool:
112
+ """Check if word appears in BOTH text and token sequence."""
113
+ if bad_word not in text:
114
+ return False
115
+
116
+ for add_prefix_space in [False, True]:
117
+ prefix = " " if add_prefix_space else ""
118
+ bad_words_token = tokenizer.encode(
119
+ prefix + bad_word.lstrip(), add_special_tokens=False
120
+ )
121
+ if not bad_words_token:
122
+ continue
123
+ for i in range(len(tokens) - len(bad_words_token) + 1):
124
+ if tokens[i : i + len(bad_words_token)] == bad_words_token:
125
+ return True
126
+ return False
127
+
128
output = llm.generate(PROMPT, SamplingParams(temperature=0))
129
split_text = output[0].outputs[0].text.split()
130
@@ -113,14 +132,16 @@ def test_bad_words(llm):
132
params = SamplingParams(temperature=0, bad_words=[bad_words_1])
133
output = llm.generate(PROMPT, params)
134
new_text = output[0].outputs[0].text
116
- assert bad_words_1 not in new_text
135
+ new_tokens = output[0].outputs[0].token_ids
136
+ assert not contains_bad_word(new_text, new_tokens, bad_words_1)
137
138
bad_words_2 = new_text.split()[-1]
139
params = SamplingParams(temperature=0, bad_words=[bad_words_1, bad_words_2])
140
output = llm.generate(PROMPT, params)
141
new_text = output[0].outputs[0].text
122
- assert bad_words_1 not in new_text
123
- assert bad_words_2 not in new_text
142
+ new_tokens = output[0].outputs[0].token_ids
143
+ assert not contains_bad_word(new_text, new_tokens, bad_words_1)
144
+ assert not contains_bad_word(new_text, new_tokens, bad_words_2)
145
146
147
def test_logits_processor(llm):