| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | import pytest_asyncio |
| 6 | import requests |
| 7 | |
| 8 | from tests.utils import RemoteOpenAIServer |
| 9 | from vllm.tokenizers import get_tokenizer |
| 10 | |
| 11 | # any model with a chat template should work here |
| 12 | MODEL_NAME = "HuggingFaceTB/SmolLM2-135M-Instruct" |
| 13 | |
| 14 | |
| 15 | @pytest.fixture(scope="module") |
| 16 | def server(): |
| 17 | args = [ |
| 18 | # use half precision for speed and memory savings in CI environment |
| 19 | "--dtype", |
| 20 | "bfloat16", |
| 21 | "--max-model-len", |
| 22 | "8192", |
| 23 | "--enforce-eager", |
| 24 | "--max-num-seqs", |
| 25 | "128", |
| 26 | "--enable-tokenizer-info-endpoint", |
| 27 | ] |
| 28 | |
| 29 | with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: |
| 30 | yield remote_server |
| 31 | |
| 32 | |
| 33 | @pytest.fixture(scope="module") |
| 34 | def tokenizer_name(model_name: str): |
| 35 | return model_name |
| 36 | |
| 37 | |
| 38 | @pytest_asyncio.fixture |
| 39 | async def client(server): |
| 40 | async with server.get_async_client() as async_client: |
| 41 | yield async_client |
| 42 | |
| 43 | |
| 44 | @pytest.mark.asyncio |
| 45 | @pytest.mark.parametrize( |
| 46 | "model_name,tokenizer_name", |
| 47 | [(MODEL_NAME, MODEL_NAME)], |
| 48 | indirect=["tokenizer_name"], |
| 49 | ) |
| 50 | async def test_tokenize_completions( |
| 51 | server: RemoteOpenAIServer, |
| 52 | model_name: str, |
| 53 | tokenizer_name: str, |
| 54 | ): |
| 55 | tokenizer = get_tokenizer(tokenizer_name=tokenizer_name) |
| 56 | |
| 57 | for add_special in [False, True]: |
| 58 | prompt = "vllm1 This is a test prompt." |
| 59 | tokens = tokenizer.encode(prompt, add_special_tokens=add_special) |
| 60 | |
| 61 | response = requests.post( |
| 62 | server.url_for("tokenize"), |
| 63 | json={ |
| 64 | "add_special_tokens": add_special, |
| 65 | "model": model_name, |
| 66 | "prompt": prompt, |
| 67 | }, |
| 68 | ) |
| 69 | response.raise_for_status() |
| 70 | |
| 71 | result = response.json() |
| 72 | assert result["tokens"] == tokens |
| 73 | assert result["count"] == len(tokens) |
| 74 | assert result["max_model_len"] == 8192 |
| 75 | assert result["token_strs"] is None |
| 76 | |
| 77 | |
| 78 | @pytest.mark.asyncio |
| 79 | @pytest.mark.parametrize( |
| 80 | "model_name,tokenizer_name", |
| 81 | [(MODEL_NAME, MODEL_NAME)], |
| 82 | indirect=["tokenizer_name"], |
| 83 | ) |
| 84 | async def test_tokenize_chat( |
| 85 | server: RemoteOpenAIServer, |
| 86 | model_name: str, |
| 87 | tokenizer_name: str, |
| 88 | ): |
| 89 | tokenizer = get_tokenizer(tokenizer_name=tokenizer_name) |
| 90 | |
| 91 | for add_generation in [False, True]: |
| 92 | for add_special in [False, True]: |
| 93 | conversation = [ |
| 94 | {"role": "user", "content": "Hi there!"}, |
| 95 | {"role": "assistant", "content": "Nice to meet you!"}, |
| 96 | {"role": "user", "content": "Can I ask a question? vllm1"}, |
| 97 | ] |
| 98 | for continue_final in [False, True]: |
| 99 | if add_generation and continue_final: |
| 100 | continue |
| 101 | if continue_final: |
| 102 | conversation.append({"role": "assistant", "content": "Sure,"}) |
| 103 | |
| 104 | prompt = tokenizer.apply_chat_template( # type: ignore[call-arg] |
| 105 | add_generation_prompt=add_generation, |
| 106 | continue_final_message=continue_final, |
| 107 | conversation=conversation, |
| 108 | tokenize=False, |
| 109 | ) |
| 110 | tokens = tokenizer.encode(prompt, add_special_tokens=add_special) |
| 111 | |
| 112 | response = requests.post( |
| 113 | server.url_for("tokenize"), |
| 114 | json={ |
| 115 | "add_generation_prompt": add_generation, |
| 116 | "continue_final_message": continue_final, |
| 117 | "add_special_tokens": add_special, |
| 118 | "messages": conversation, |
| 119 | "model": model_name, |
| 120 | }, |
| 121 | ) |
| 122 | response.raise_for_status() |
| 123 | |
| 124 | result = response.json() |
| 125 | assert result["tokens"] == tokens |
| 126 | assert result["count"] == len(tokens) |
| 127 | assert result["max_model_len"] == 8192 |
| 128 | assert result["token_strs"] is None |
| 129 | |
| 130 | |
| 131 | @pytest.mark.asyncio |
| 132 | @pytest.mark.parametrize( |
| 133 | "model_name,tokenizer_name", |
| 134 | [(MODEL_NAME, MODEL_NAME)], |
| 135 | indirect=["tokenizer_name"], |
| 136 | ) |
| 137 | async def test_tokenize_chat_with_tools( |
| 138 | server: RemoteOpenAIServer, |
| 139 | model_name: str, |
| 140 | tokenizer_name: str, |
| 141 | ): |
| 142 | tokenizer = get_tokenizer(tokenizer_name=tokenizer_name) |
| 143 | |
| 144 | for add_generation in [False, True]: |
| 145 | for add_special in [False, True]: |
| 146 | conversation = [ |
| 147 | { |
| 148 | "role": "user", |
| 149 | "content": "What's the weather like in Paris today?", |
| 150 | } |
| 151 | ] |
| 152 | |
| 153 | tools = [ |
| 154 | { |
| 155 | "type": "function", |
| 156 | "function": { |
| 157 | "name": "get_weather", |
| 158 | "parameters": { |
| 159 | "type": "object", |
| 160 | "properties": {"location": {"type": "string"}}, |
| 161 | }, |
| 162 | }, |
| 163 | } |
| 164 | ] |
| 165 | |
| 166 | for continue_final in [False, True]: |
| 167 | if add_generation and continue_final: |
| 168 | continue |
| 169 | if continue_final: |
| 170 | conversation.append({"role": "assistant", "content": "Sure,"}) |
| 171 | |
| 172 | prompt = tokenizer.apply_chat_template( # type: ignore[call-arg] |
| 173 | add_generation_prompt=add_generation, |
| 174 | continue_final_message=continue_final, |
| 175 | conversation=conversation, |
| 176 | tools=tools, |
| 177 | tokenize=False, |
| 178 | ) |
| 179 | tokens = tokenizer.encode(prompt, add_special_tokens=add_special) |
| 180 | |
| 181 | response = requests.post( |
| 182 | server.url_for("tokenize"), |
| 183 | json={ |
| 184 | "add_generation_prompt": add_generation, |
| 185 | "continue_final_message": continue_final, |
| 186 | "add_special_tokens": add_special, |
| 187 | "messages": conversation, |
| 188 | "model": model_name, |
| 189 | "tools": tools, |
| 190 | }, |
| 191 | ) |
| 192 | response.raise_for_status() |
| 193 | |
| 194 | result = response.json() |
| 195 | assert result["tokens"] == tokens |
| 196 | assert result["count"] == len(tokens) |
| 197 | assert result["max_model_len"] == 8192 |
| 198 | assert result["token_strs"] is None |
| 199 | |
| 200 | |
| 201 | @pytest.mark.asyncio |
| 202 | @pytest.mark.parametrize( |
| 203 | "model_name, tokenizer_name", |
| 204 | [(MODEL_NAME, MODEL_NAME)], |
| 205 | indirect=["tokenizer_name"], |
| 206 | ) |
| 207 | async def test_tokenize_with_return_token_strs( |
| 208 | server: RemoteOpenAIServer, |
| 209 | model_name: str, |
| 210 | tokenizer_name: str, |
| 211 | ): |
| 212 | tokenizer = get_tokenizer(tokenizer_name=tokenizer_name) |
| 213 | |
| 214 | prompt = "This is a token_strs test prompt! vllm1" |
| 215 | response = requests.post( |
| 216 | server.url_for("tokenize"), |
| 217 | json={"prompt": prompt, "model": model_name, "return_token_strs": True}, |
| 218 | ) |
| 219 | response.raise_for_status() |
| 220 | |
| 221 | tokens = tokenizer.encode(prompt, add_special_tokens=True) |
| 222 | tokens_str = tokenizer.convert_ids_to_tokens(tokens) |
| 223 | |
| 224 | result = response.json() |
| 225 | assert result["tokens"] == tokens |
| 226 | assert result["count"] == len(tokens) |
| 227 | assert result["max_model_len"] == 8192 |
| 228 | assert result["token_strs"] == tokens_str |
| 229 | |
| 230 | |
| 231 | @pytest.mark.asyncio |
| 232 | @pytest.mark.parametrize( |
| 233 | "model_name,tokenizer_name", |
| 234 | [(MODEL_NAME, MODEL_NAME)], |
| 235 | indirect=["tokenizer_name"], |
| 236 | ) |
| 237 | async def test_detokenize( |
| 238 | server: RemoteOpenAIServer, |
| 239 | model_name: str, |
| 240 | tokenizer_name: str, |
| 241 | ): |
| 242 | tokenizer = get_tokenizer(tokenizer_name=tokenizer_name) |
| 243 | |
| 244 | prompt = "This is a test prompt. vllm1" |
| 245 | tokens = tokenizer.encode(prompt, add_special_tokens=False) |
| 246 | |
| 247 | response = requests.post( |
| 248 | server.url_for("detokenize"), json={"model": model_name, "tokens": tokens} |
| 249 | ) |
| 250 | response.raise_for_status() |
| 251 | |
| 252 | assert response.json() == {"prompt": prompt} |
| 253 | |
| 254 | |
| 255 | @pytest.mark.asyncio |
| 256 | @pytest.mark.parametrize( |
| 257 | "model_name,tokenizer_name", |
| 258 | [(MODEL_NAME, MODEL_NAME)], |
| 259 | indirect=["tokenizer_name"], |
| 260 | ) |
| 261 | async def test_tokenizer_info_basic( |
| 262 | server: RemoteOpenAIServer, |
| 263 | model_name: str, |
| 264 | tokenizer_name: str, |
| 265 | ): |
| 266 | """Test basic tokenizer info endpoint functionality.""" |
| 267 | response = requests.get(server.url_for("tokenizer_info")) |
| 268 | response.raise_for_status() |
| 269 | result = response.json() |
| 270 | assert "tokenizer_class" in result |
| 271 | assert isinstance(result["tokenizer_class"], str) |
| 272 | assert result["tokenizer_class"] |
| 273 | |
| 274 | |
| 275 | @pytest.mark.asyncio |
| 276 | async def test_tokenizer_info_schema(server: RemoteOpenAIServer): |
| 277 | """Test that the response matches expected schema types.""" |
| 278 | response = requests.get(server.url_for("tokenizer_info")) |
| 279 | response.raise_for_status() |
| 280 | result = response.json() |
| 281 | field_types = { |
| 282 | "add_bos_token": bool, |
| 283 | "add_prefix_space": bool, |
| 284 | "clean_up_tokenization_spaces": bool, |
| 285 | "split_special_tokens": bool, |
| 286 | "bos_token": str, |
| 287 | "eos_token": str, |
| 288 | "pad_token": str, |
| 289 | "unk_token": str, |
| 290 | "chat_template": str, |
| 291 | "errors": str, |
| 292 | "model_max_length": int, |
| 293 | "additional_special_tokens": list, |
| 294 | "added_tokens_decoder": dict, |
| 295 | } |
| 296 | for field, expected_type in field_types.items(): |
| 297 | if field in result and result[field] is not None: |
| 298 | assert isinstance(result[field], expected_type), ( |
| 299 | f"{field} should be {expected_type.__name__}" |
| 300 | ) |
| 301 | |
| 302 | |
| 303 | @pytest.mark.asyncio |
| 304 | async def test_tokenizer_info_consistency_with_tokenize( |
| 305 | server: RemoteOpenAIServer, |
| 306 | ): |
| 307 | """Test that tokenizer info is consistent with tokenization endpoint.""" |
| 308 | info_response = requests.get(server.url_for("tokenizer_info")) |
| 309 | info_response.raise_for_status() |
| 310 | info = info_response.json() |
| 311 | tokenize_response = requests.post( |
| 312 | server.url_for("tokenize"), |
| 313 | json={"model": MODEL_NAME, "prompt": "Hello world!"}, |
| 314 | ) |
| 315 | tokenize_response.raise_for_status() |
| 316 | tokenize_result = tokenize_response.json() |
| 317 | info_max_len = info.get("model_max_length") |
| 318 | tokenize_max_len = tokenize_result.get("max_model_len") |
| 319 | if info_max_len and tokenize_max_len: |
| 320 | assert info_max_len >= tokenize_max_len, ( |
| 321 | "Info max length should be >= tokenize max length" |
| 322 | ) |
| 323 | |
| 324 | |
| 325 | @pytest.mark.asyncio |
| 326 | async def test_tokenizer_info_chat_template(server: RemoteOpenAIServer): |
| 327 | """Test chat template is properly included.""" |
| 328 | response = requests.get(server.url_for("tokenizer_info")) |
| 329 | response.raise_for_status() |
| 330 | result = response.json() |
| 331 | chat_template = result.get("chat_template") |
| 332 | if chat_template: |
| 333 | assert isinstance(chat_template, str), "Chat template should be a string" |
| 334 | assert chat_template.strip(), "Chat template should not be empty" |