@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a32cb49b60
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 19:00 UTC

feat(frontend): early-fail tokenization guard for user requests (#31366)

Signed-off-by: limingliang <limingliang@stepfun.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: limingliang <limingliang@stepfun.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Mingliang Li committed Feb 6, 2026 at 11:38 UTC a32cb49b60688fb64a6d3d7f86378b4d2fad06e6
7 files changed +308 -202
tests/renderers/test_completions.py
+230 -181
@@ -4,7 +4,6 @@
4 import io
5 from dataclasses import dataclass
6 from typing import Any
7 -from unittest.mock import AsyncMock
7
8 import pybase64
9 import pytest
@@ -28,7 +27,6 @@ class MockModelConfig:
27 model: str = MODEL_NAME
28 tokenizer: str = MODEL_NAME
29 trust_remote_code: bool = False
31 - max_model_len: int = 100
30 tokenizer_revision = None
31 tokenizer_mode = "auto"
32 hf_config = MockHFConfig()
@@ -37,25 +35,50 @@ class MockModelConfig:
35 skip_tokenizer_init: bool = False
36
37
40 -@pytest.fixture
41 -def mock_model_config():
42 - return MockModelConfig()
38 +@dataclass
39 +class DummyTokenizer:
40 + truncation_side: str = "left"
41 + max_chars_per_token: int = 1
42 +
43 + def __post_init__(self) -> None:
44 + self._captured_encode_kwargs: dict = {}
45 +
46 + def decode(self, tokens: list[int]):
47 + return str(tokens)
48 +
49 + def encode(self, text: str, **kwargs):
50 + self._captured_encode_kwargs = kwargs
51
52 + in_length = len(text)
53 + truncation = kwargs.get("truncation")
54 + max_length = kwargs.get("max_length")
55 + if truncation and max_length is not None:
56 + return list(range(min(in_length, max_length)))
57
45 -@pytest.fixture
46 -def mock_async_tokenizer():
47 - return AsyncMock()
58 + return list(range(in_length))
59
60
50 -@pytest.fixture
51 -def renderer(mock_model_config):
52 - _, tokenizer_name, _, kwargs = tokenizer_args_from_config(mock_model_config)
61 +def _build_renderer(
62 + model_config: MockModelConfig,
63 + *,
64 + truncation_side: str = "left",
65 + max_chars_per_token: int = 1,
66 +):
67 + _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
68
54 - return HfRenderer(
55 - mock_model_config,
69 + renderer = HfRenderer(
70 + model_config,
71 tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
72 )
73
74 + if not model_config.skip_tokenizer_init:
75 + renderer._tokenizer = DummyTokenizer(
76 + truncation_side=truncation_side,
77 + max_chars_per_token=max_chars_per_token,
78 + )
79 +
80 + return renderer
81 +
82
83 class TestValidatePrompt:
84 STRING_INPUTS = [
@@ -81,39 +104,50 @@ class TestValidatePrompt:
104 ]
105
106 # Test that a nested mixed-type list of lists raises a TypeError.
84 - def test_empty_input(self, renderer):
107 + def test_empty_input(self):
108 + renderer = _build_renderer(MockModelConfig())
109 +
110 with pytest.raises(ValueError, match="at least one prompt"):
111 renderer.render_completions([])
112
88 - def test_invalid_type(self, renderer):
113 + def test_invalid_type(self):
114 + renderer = _build_renderer(MockModelConfig())
115 +
116 with pytest.raises(TypeError, match="string or an array of tokens"):
117 renderer.render_completions([[1, 2], ["foo", "bar"]])
118
119 @pytest.mark.parametrize("string_input", STRING_INPUTS)
93 - def test_string_consistent(self, renderer, string_input: str):
120 + def test_string_consistent(self, string_input: str):
121 + renderer = _build_renderer(MockModelConfig())
122 +
123 assert renderer.render_completions(string_input) == renderer.render_completions(
124 [string_input]
125 )
126
127 @pytest.mark.parametrize("token_input", TOKEN_INPUTS)
99 - def test_token_consistent(self, renderer, token_input: list[int]):
128 + def test_token_consistent(self, token_input: list[int]):
129 + renderer = _build_renderer(MockModelConfig())
130 +
131 assert renderer.render_completions(token_input) == renderer.render_completions(
132 [token_input]
133 )
134
135 @pytest.mark.parametrize("inputs_slice", INPUTS_SLICES)
105 - def test_string_slice(self, renderer, inputs_slice: slice):
136 + def test_string_slice(self, inputs_slice: slice):
137 + renderer = _build_renderer(MockModelConfig())
138 +
139 assert renderer.render_completions(self.STRING_INPUTS)[
140 inputs_slice
141 ] == renderer.render_completions(self.STRING_INPUTS[inputs_slice])
142
143
144 class TestRenderPrompt:
112 - @pytest.mark.asyncio
113 - async def test_token_input(self, renderer):
145 + def test_token_input(self):
146 + renderer = _build_renderer(MockModelConfig())
147 +
148 tokens = [101, 7592, 2088]
115 - prompts = await renderer.render_completions_async(tokens)
116 - results = await renderer.tokenize_prompts_async(
149 + prompts = renderer.render_completions(tokens)
150 + results = renderer.tokenize_prompts(
151 prompts,
152 TokenizeParams(max_total_tokens=100),
153 )
@@ -121,11 +155,12 @@ class TestRenderPrompt:
155 assert len(results) == 1
156 assert results[0]["prompt_token_ids"] == tokens
157
124 - @pytest.mark.asyncio
125 - async def test_token_list_input(self, renderer):
158 + def test_token_list_input(self):
159 + renderer = _build_renderer(MockModelConfig())
160 +
161 token_lists = [[101, 7592, 2088], [102, 1234, 5678, 9012], [103, 4567]]
127 - prompts = await renderer.render_completions_async(token_lists)
128 - results = await renderer.tokenize_prompts_async(
162 + prompts = renderer.render_completions(token_lists)
163 + results = renderer.tokenize_prompts(
164 prompts,
165 TokenizeParams(max_total_tokens=100),
166 )
@@ -135,167 +170,178 @@ class TestRenderPrompt:
170 assert results[1]["prompt_token_ids"] == [102, 1234, 5678, 9012]
171 assert results[2]["prompt_token_ids"] == [103, 4567]
172
138 - @pytest.mark.asyncio
139 - async def test_text_input(self, renderer, mock_async_tokenizer):
140 - mock_async_tokenizer.encode.return_value = [101, 7592, 2088]
141 - renderer._async_tokenizer = mock_async_tokenizer
173 + def test_text_input(self):
174 + renderer = _build_renderer(MockModelConfig())
175
143 - prompts = await renderer.render_completions_async("Hello world")
144 - results = await renderer.tokenize_prompts_async(
176 + text_input = "x" * 10
177 + prompts = renderer.render_completions(text_input)
178 + results = renderer.tokenize_prompts(
179 prompts,
180 TokenizeParams(max_total_tokens=100),
181 )
182
183 assert len(results) == 1
150 - assert results[0]["prompt_token_ids"] == [101, 7592, 2088]
151 - mock_async_tokenizer.encode.assert_called_once()
184 + assert len(results[0]["prompt_token_ids"]) == 10
185
153 - @pytest.mark.asyncio
154 - async def test_text_list_input(self, renderer, mock_async_tokenizer):
155 - mock_async_tokenizer.encode.return_value = [101, 7592, 2088]
156 - renderer._async_tokenizer = mock_async_tokenizer
186 + def test_text_list_input(self):
187 + renderer = _build_renderer(MockModelConfig())
188
158 - text_list_input = ["Hello world", "How are you?", "Good morning"]
159 - prompts = await renderer.render_completions_async(text_list_input)
160 - results = await renderer.tokenize_prompts_async(
189 + text_list_input = ["x" * 10, "x" * 12, "x" * 14]
190 + prompts = renderer.render_completions(text_list_input)
191 + results = renderer.tokenize_prompts(
192 prompts,
193 TokenizeParams(max_total_tokens=100),
194 )
195
196 assert len(results) == 3
166 - for result in results:
167 - assert result["prompt_token_ids"] == [101, 7592, 2088]
168 - assert mock_async_tokenizer.encode.call_count == 3
197 + for text_input, result in zip(text_list_input, results):
198 + assert len(result["prompt_token_ids"]) == len(text_input)
199
170 - @pytest.mark.asyncio
171 - async def test_no_truncation(self, renderer, mock_async_tokenizer):
172 - mock_async_tokenizer.encode.return_value = [101, 7592, 2088]
173 - renderer._async_tokenizer = mock_async_tokenizer
200 + def test_zero_truncation(self):
201 + renderer = _build_renderer(MockModelConfig())
202
175 - prompts = await renderer.render_completions_async("Hello world")
176 - results = await renderer.tokenize_prompts_async(
203 + prompts = renderer.render_completions("x" * 200)
204 + results = renderer.tokenize_prompts(
205 prompts,
178 - TokenizeParams(max_total_tokens=100),
206 + TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=0),
207 )
208
209 assert len(results) == 1
182 - call_args = mock_async_tokenizer.encode.call_args
183 - assert (
184 - "truncation" not in call_args.kwargs
185 - or call_args.kwargs["truncation"] is False
186 - )
210 + assert len(results[0]["prompt_token_ids"]) == 0
211
188 - @pytest.mark.asyncio
189 - async def test_truncation_positive(self, renderer, mock_async_tokenizer):
190 - mock_async_tokenizer.encode.return_value = [101, 7592, 2088] # Truncated
191 - renderer._async_tokenizer = mock_async_tokenizer
212 + def test_pos_truncation(self):
213 + renderer = _build_renderer(MockModelConfig())
214
193 - prompts = await renderer.render_completions_async("Hello world")
194 - results = await renderer.tokenize_prompts_async(
215 + prompts = renderer.render_completions("x" * 200)
216 + results = renderer.tokenize_prompts(
217 prompts,
196 - TokenizeParams(
197 - max_total_tokens=200,
198 - truncate_prompt_tokens=50,
199 - ),
218 + TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=50),
219 )
220
221 assert len(results) == 1
203 - call_args = mock_async_tokenizer.encode.call_args
204 - assert call_args.kwargs["truncation"] is True
205 - assert call_args.kwargs["max_length"] == 50
206 -
207 - @pytest.mark.asyncio
208 - async def test_truncation_negative(self, renderer, mock_async_tokenizer):
209 - # Test that negative truncation uses model's max_model_len
210 - mock_async_tokenizer.encode.return_value = [
211 - 101,
212 - 7592,
213 - 2088,
214 - ] # Truncated to max_model_len
215 - renderer._async_tokenizer = mock_async_tokenizer
216 -
217 - prompts = await renderer.render_completions_async("Hello world")
218 - results = await renderer.tokenize_prompts_async(
222 + assert len(results[0]["prompt_token_ids"]) == 50
223 +
224 + def test_neg_truncation(self):
225 + renderer = _build_renderer(MockModelConfig())
226 +
227 + prompts = renderer.render_completions("x" * 200)
228 + results = renderer.tokenize_prompts(
229 prompts,
220 - TokenizeParams(
221 - max_total_tokens=200,
222 - truncate_prompt_tokens=-1,
223 - ),
230 + TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=-1),
231 )
232
233 assert len(results) == 1
227 - call_args = mock_async_tokenizer.encode.call_args
228 - assert call_args.kwargs["truncation"] is True
229 - assert call_args.kwargs["max_length"] == 200
234 + assert len(results[0]["prompt_token_ids"]) == 100 # max_total_tokens
235 +
236 + def test_truncation_left(self):
237 + renderer = _build_renderer(MockModelConfig(), truncation_side="left")
238
231 - @pytest.mark.asyncio
232 - async def test_token_truncation_last_elements(self, renderer):
233 - # Test that token truncation keeps the last N elements
239 long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens
235 - prompts = await renderer.render_completions_async(long_tokens)
236 - results = await renderer.tokenize_prompts_async(
240 + prompts = renderer.render_completions(long_tokens)
241 + results = renderer.tokenize_prompts(
242 prompts,
238 - TokenizeParams(
239 - max_total_tokens=100,
240 - truncate_prompt_tokens=5,
241 - ),
243 + TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=5),
244 )
245
246 assert len(results) == 1
247 # Should keep the last 5 tokens: [105, 106, 107, 108, 109]
248 assert results[0]["prompt_token_ids"] == [105, 106, 107, 108, 109]
249
248 - @pytest.mark.asyncio
249 - async def test_max_length_exceeded(self, renderer):
250 - long_tokens = list(range(150)) # Exceeds max_model_len=100
250 + def test_truncation_right(self):
251 + renderer = _build_renderer(MockModelConfig(), truncation_side="right")
252 +
253 + long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens
254 + prompts = renderer.render_completions(long_tokens)
255 + results = renderer.tokenize_prompts(
256 + prompts,
257 + TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=5),
258 + )
259
252 - prompts = await renderer.render_completions_async(long_tokens)
260 + assert len(results) == 1
261 + # Should keep the first 5 tokens: [100, 101, 102, 103, 104]
262 + assert results[0]["prompt_token_ids"] == [100, 101, 102, 103, 104]
263
254 - with pytest.raises(ValueError, match="context length is only"):
255 - await renderer.tokenize_prompts_async(
264 + def test_text_max_length_exceeded_obvious(self):
265 + renderer = _build_renderer(MockModelConfig(), max_chars_per_token=1)
266 +
267 + # Exceeds max_total_tokens and max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN
268 + long_tokens = "x" * 150
269 + prompts = renderer.render_completions(long_tokens)
270 +
271 + with pytest.raises(
272 + ValueError,
273 + match="input characters and requested .* context length is only",
274 + ):
275 + renderer.tokenize_prompts(
276 prompts,
277 TokenizeParams(max_total_tokens=100),
278 )
279
260 - @pytest.mark.asyncio
261 - async def test_no_tokenizer_for_text(self, renderer):
262 - renderer_no_tokenizer = HfRenderer.from_config(
263 - MockModelConfig(skip_tokenizer_init=True),
264 - tokenizer_kwargs={},
265 - )
280 + # Should not even attempt tokenization
281 + assert renderer._tokenizer._captured_encode_kwargs == {}
282 +
283 + def test_text_max_length_exceeded_nonobvious(self):
284 + renderer = _build_renderer(MockModelConfig(), max_chars_per_token=2)
285 +
286 + # Exceeds max_total_tokens but not max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN
287 + long_tokens = "x" * 150
288 + prompts = renderer.render_completions(long_tokens)
289 +
290 + with pytest.raises(
291 + ValueError,
292 + match="input tokens and requested .* context length is only",
293 + ):
294 + renderer.tokenize_prompts(
295 + prompts,
296 + TokenizeParams(max_total_tokens=100),
297 + )
298 +
299 + # Should only tokenize the first max_total_tokens + 1 tokens
300 + assert renderer._tokenizer._captured_encode_kwargs["truncation"] is True
301 + assert renderer._tokenizer._captured_encode_kwargs["max_length"] == 101
302 +
303 + def test_token_max_length_exceeded(self):
304 + renderer = _build_renderer(MockModelConfig())
305 +
306 + long_tokens = list(range(150)) # Exceeds max_total_tokens=100
307 + prompts = renderer.render_completions(long_tokens)
308 +
309 + with pytest.raises(
310 + ValueError,
311 + match="input tokens and requested .* context length is only",
312 + ):
313 + renderer.tokenize_prompts(
314 + prompts,
315 + TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=None),
316 + )
317 +
318 + def test_no_tokenizer_for_text(self):
319 + renderer = _build_renderer(MockModelConfig(skip_tokenizer_init=True))
320
267 - prompts = await renderer_no_tokenizer.render_completions_async("Hello world")
321 + prompts = renderer.render_completions("Hello world")
322
323 with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"):
270 - await renderer_no_tokenizer.tokenize_prompts_async(
324 + renderer.tokenize_prompts(
325 prompts,
326 TokenizeParams(max_total_tokens=100),
327 )
328
275 - @pytest.mark.asyncio
276 - async def test_token_input_with_needs_detokenization(
277 - self, renderer, mock_async_tokenizer
278 - ):
279 - # When needs_detokenization=True for token inputs, renderer should
280 - # use the async tokenizer to decode and include the original text
281 - # in the returned prompt object.
282 - mock_async_tokenizer.decode = AsyncMock(return_value="decoded text")
283 - renderer._async_tokenizer = mock_async_tokenizer
329 + def test_token_input_with_needs_detokenization(self):
330 + renderer = _build_renderer(MockModelConfig())
331
332 tokens = [1, 2, 3, 4]
286 - prompts = await renderer.render_completions_async(tokens)
287 - results = await renderer.tokenize_prompts_async(
333 + prompts = renderer.render_completions(tokens)
334 + results = renderer.tokenize_prompts(
335 prompts,
336 TokenizeParams(
290 - max_total_tokens=renderer.config.max_model_len,
337 + max_total_tokens=100,
338 needs_detokenization=True,
339 ),
340 )
341
342 assert len(results) == 1
343 assert results[0]["prompt_token_ids"] == tokens
297 - assert results[0]["prompt"] == "decoded text"
298 - mock_async_tokenizer.decode.assert_awaited_once()
344 + assert results[0]["prompt"] == "[1, 2, 3, 4]"
345
346
347 class TestRenderEmbedPrompt:
@@ -306,118 +352,121 @@ class TestRenderEmbedPrompt:
352 buffer.seek(0)
353 return pybase64.b64encode(buffer.read())
354
309 - @pytest.mark.asyncio
310 - async def test_single_prompt_embed(self, renderer):
355 + def test_single_prompt_embed(self):
356 + renderer = _build_renderer(MockModelConfig())
357 +
358 # Create a test tensor
312 - test_tensor = torch.randn(10, 768, dtype=torch.float32)
313 - embed_bytes = self._create_test_embed_bytes(test_tensor)
359 + tensor_input = torch.randn(10, 768, dtype=torch.float32)
360 + embed_bytes = self._create_test_embed_bytes(tensor_input)
361
315 - prompts = await renderer.render_completions_async(prompt_embeds=embed_bytes)
316 - results = await renderer.tokenize_prompts_async(
362 + prompts = renderer.render_completions(prompt_embeds=embed_bytes)
363 + results = renderer.tokenize_prompts(
364 prompts,
318 - TokenizeParams(max_total_tokens=renderer.config.max_model_len),
365 + TokenizeParams(max_total_tokens=100),
366 )
367
368 assert len(results) == 1
322 - assert torch.allclose(results[0]["prompt_embeds"], test_tensor)
369 + assert torch.equal(results[0]["prompt_embeds"], tensor_input)
370 +
371 + def test_multiple_prompt_embeds(self):
372 + renderer = _build_renderer(MockModelConfig())
373
324 - @pytest.mark.asyncio
325 - async def test_multiple_prompt_embeds(self, renderer):
374 # Create multiple test tensors
327 - test_tensors = [
375 + tensor_inputs = [
376 torch.randn(8, 512, dtype=torch.float32),
377 torch.randn(12, 512, dtype=torch.float32),
378 ]
331 - embed_bytes_list = [self._create_test_embed_bytes(t) for t in test_tensors]
379
333 - prompts = await renderer.render_completions_async(
334 - prompt_embeds=embed_bytes_list
380 + prompts = renderer.render_completions(
381 + prompt_embeds=[self._create_test_embed_bytes(t) for t in tensor_inputs],
382 )
336 - results = await renderer.tokenize_prompts_async(
383 + results = renderer.tokenize_prompts(
384 prompts,
338 - TokenizeParams(max_total_tokens=renderer.config.max_model_len),
385 + TokenizeParams(max_total_tokens=100),
386 )
387
388 assert len(results) == 2
389 for i, result in enumerate(results):
343 - assert torch.allclose(result["prompt_embeds"], test_tensors[i])
390 + assert torch.allclose(result["prompt_embeds"], tensor_inputs[i])
391 +
392 + def test_prompt_embed_truncation(self):
393 + renderer = _build_renderer(MockModelConfig())
394
345 - @pytest.mark.asyncio
346 - async def test_prompt_embed_truncation(self, renderer):
395 # Create tensor with more tokens than truncation limit
348 - test_tensor = torch.randn(20, 768, dtype=torch.float32)
349 - embed_bytes = self._create_test_embed_bytes(test_tensor)
396 + tensor_input = torch.randn(20, 768, dtype=torch.float32)
397
351 - prompts = await renderer.render_completions_async(prompt_embeds=embed_bytes)
352 - results = await renderer.tokenize_prompts_async(
398 + prompts = renderer.render_completions(
399 + prompt_embeds=self._create_test_embed_bytes(tensor_input),
400 + )
401 + results = renderer.tokenize_prompts(
402 prompts,
403 TokenizeParams(
355 - max_total_tokens=renderer.config.max_model_len,
404 + max_total_tokens=100,
405 truncate_prompt_tokens=10,
406 ),
407 )
408
409 assert len(results) == 1
410 # Should keep last 10 tokens
362 - expected = test_tensor[-10:]
363 - assert torch.allclose(results[0]["prompt_embeds"], expected)
411 + expected = tensor_input[-10:]
412 + assert torch.equal(results[0]["prompt_embeds"], expected)
413 +
414 + def test_prompt_embed_different_dtypes(self):
415 + renderer = _build_renderer(MockModelConfig())
416
365 - @pytest.mark.asyncio
366 - async def test_prompt_embed_different_dtypes(self, renderer):
417 # Test different supported dtypes
418 dtypes = [torch.float32, torch.float16, torch.bfloat16]
419
420 for dtype in dtypes:
371 - test_tensor = torch.randn(5, 256, dtype=dtype)
372 - embed_bytes = self._create_test_embed_bytes(test_tensor)
421 + tensor_input = torch.randn(5, 256, dtype=dtype)
422
374 - prompts = await renderer.render_completions_async(prompt_embeds=embed_bytes)
375 - results = await renderer.tokenize_prompts_async(
423 + prompts = renderer.render_completions(
424 + prompt_embeds=self._create_test_embed_bytes(tensor_input),
425 + )
426 + results = renderer.tokenize_prompts(
427 prompts,
377 - TokenizeParams(max_total_tokens=renderer.config.max_model_len),
428 + TokenizeParams(max_total_tokens=100),
429 )
430
431 assert len(results) == 1
432 assert results[0]["prompt_embeds"].dtype == dtype
433
383 - @pytest.mark.asyncio
384 - async def test_prompt_embed_squeeze_batch_dim(self, renderer):
434 + def test_prompt_embed_squeeze_batch_dim(self):
435 + renderer = _build_renderer(MockModelConfig())
436 +
437 # Test tensor with batch dimension gets squeezed
386 - test_tensor = torch.randn(1, 10, 768, dtype=torch.float32)
387 - embed_bytes = self._create_test_embed_bytes(test_tensor)
438 + tensor_input = torch.randn(1, 10, 768, dtype=torch.float32)
439
389 - prompts = await renderer.render_completions_async(prompt_embeds=embed_bytes)
390 - results = await renderer.tokenize_prompts_async(
440 + prompts = renderer.render_completions(
441 + prompt_embeds=self._create_test_embed_bytes(tensor_input),
442 + )
443 + results = renderer.tokenize_prompts(
444 prompts,
392 - TokenizeParams(max_total_tokens=renderer.config.max_model_len),
445 + TokenizeParams(max_total_tokens=100),
446 )
447
448 assert len(results) == 1
449 # Should be squeezed to 2D
450 assert results[0]["prompt_embeds"].shape == (10, 768)
451
399 - @pytest.mark.asyncio
400 - async def test_both_prompts_and_embeds(self, renderer, mock_async_tokenizer):
401 - # Set up text tokenization
402 - mock_async_tokenizer.encode.return_value = [101, 102, 103]
403 - renderer._async_tokenizer = mock_async_tokenizer
452 + def test_both_prompts_and_embeds(self):
453 + renderer = _build_renderer(MockModelConfig())
454
405 - # Create embed
406 - test_tensor = torch.randn(5, 256, dtype=torch.float32)
407 - embed_bytes = self._create_test_embed_bytes(test_tensor)
455 + text_input = "Hello world"
456 + tensor_input = torch.randn(5, 256, dtype=torch.float32)
457
409 - prompts = await renderer.render_completions_async(
410 - "Hello world",
411 - prompt_embeds=embed_bytes,
458 + prompts = renderer.render_completions(
459 + text_input,
460 + prompt_embeds=self._create_test_embed_bytes(tensor_input),
461 )
413 - results = await renderer.tokenize_prompts_async(
462 + results = renderer.tokenize_prompts(
463 prompts,
415 - TokenizeParams(max_total_tokens=renderer.config.max_model_len),
464 + TokenizeParams(max_total_tokens=100),
465 )
466
467 assert len(results) == 2
468 # First should be embed prompt
420 - assert torch.allclose(results[0]["prompt_embeds"], test_tensor)
469 + assert torch.equal(results[0]["prompt_embeds"], tensor_input)
470 # Second should be tokens prompt
471 assert "prompt_token_ids" in results[1]
423 - assert results[1]["prompt_token_ids"] == [101, 102, 103]
472 + assert len(results[1]["prompt_token_ids"]) == len(text_input)
vllm/renderers/params.py
+53 -21
@@ -229,23 +229,53 @@ class TokenizeParams:
229 max_length = self.truncate_prompt_tokens
230 if max_length is not None and max_length < 0:
231 max_length = self.max_input_tokens
232 + elif max_length is None and self.max_input_tokens is not None:
233 + # This prevents tokenization from taking up more resources than necessary
234 + # while still failing `self._token_len_check` as expected by users
235 + max_length = self.max_input_tokens + 1
236
237 return dict(
234 - truncation=self.truncate_prompt_tokens is not None,
238 + truncation=max_length is not None,
239 max_length=max_length,
240 add_special_tokens=self.add_special_tokens,
241 )
242
239 - def _apply_lowercase(self, tokenizer: TokenizerLike | None, text: str) -> str:
240 - if self.do_lower_case:
241 - text = text.lower()
243 + def _text_len_check(self, tokenizer: TokenizerLike | None, text: str) -> str:
244 + """Apply length checks to prompt text if necessary."""
245 + max_input_tokens = self.max_input_tokens
246 + if max_input_tokens is None:
247 + return text
248 +
249 + if self.truncate_prompt_tokens is None and tokenizer is not None:
250 + max_input_chars = max_input_tokens * tokenizer.max_chars_per_token
251 +
252 + if len(text) > max_input_chars:
253 + # To save resources, fail the request outright without even
254 + # attempting tokenization
255 + raise VLLMValidationError(
256 + f"You passed {len(text)} input characters "
257 + f"and requested {self.max_output_tokens} output tokens. "
258 + f"However, the model's context length is only "
259 + f"{self.max_total_tokens} tokens, resulting in a maximum "
260 + f"input length of {max_input_tokens} tokens "
261 + f"(at most {max_input_chars} characters). "
262 + f"Please reduce the length of the input prompt.",
263 + parameter="input_text",
264 + value=len(text),
265 + )
266
267 return text
268
269 + def _text_lowercase(self, tokenizer: TokenizerLike | None, text: str) -> str:
270 + """Apply lowercase to prompt text if necessary."""
271 + return text.lower() if self.do_lower_case else text
272 +
273 def _validate_text(self, tokenizer: TokenizerLike | None, text: str) -> str:
274 """Apply all validators to prompt text."""
247 - # TODO: Implement https://github.com/vllm-project/vllm/pull/31366
248 - for validator in (self._apply_lowercase,):
275 + for validator in (
276 + self._text_len_check,
277 + self._text_lowercase,
278 + ):
279 text = validator(tokenizer, text)
280
281 return text
@@ -265,8 +295,8 @@ class TokenizeParams:
295
296 return prompt
297
268 - def _apply_padding(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
269 - """Apply padding to a token sequence."""
298 + def _token_padding(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
299 + """Apply padding to prompt tokens if necessary."""
300 pad_length = self.pad_prompt_tokens
301 if pad_length is not None and pad_length < 0:
302 pad_length = self.max_input_tokens
@@ -281,8 +311,8 @@ class TokenizeParams:
311
312 return tokens + [tokenizer.pad_token_id] * (pad_length - len(tokens))
313
284 - def _apply_truncation(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
285 - """Apply truncation to a token sequence."""
314 + def _token_truncation(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
315 + """Apply truncation to prompt tokens if necessary."""
316 max_length = self.truncate_prompt_tokens
317 if max_length is not None and max_length < 0:
318 max_length = self.max_input_tokens
@@ -297,18 +327,20 @@ class TokenizeParams:
327
328 return tokens[:max_length]
329
300 - def _apply_length_check(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
301 - """Apply length checks to a token sequence."""
330 + def _token_len_check(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
331 + """Apply length checks to prompt tokens if necessary."""
332 max_input_tokens = self.max_input_tokens
333 + if max_input_tokens is None:
334 + return tokens
335
304 - if max_input_tokens is not None and len(tokens) > max_input_tokens:
336 + if len(tokens) > max_input_tokens:
337 raise VLLMValidationError(
306 - f"You passed {len(tokens)} input tokens and "
307 - f"requested {self.max_output_tokens} output tokens. "
338 + f"You passed {len(tokens)} input tokens "
339 + f"and requested {self.max_output_tokens} output tokens. "
340 f"However, the model's context length is only "
309 - f"{self.max_total_tokens}, resulting in a maximum "
310 - f"input length of {max_input_tokens}. "
311 - f"Please reduce the length of the input messages.",
341 + f"{self.max_total_tokens} tokens, resulting in a maximum "
342 + f"input length of {max_input_tokens} tokens. "
343 + f"Please reduce the length of the input prompt.",
344 parameter="input_tokens",
345 value=len(tokens),
346 )
@@ -318,9 +350,9 @@ class TokenizeParams:
350 def _validate_tokens(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
351 """Apply all validators to a token sequence."""
352 for validator in (
321 - self._apply_padding,
322 - self._apply_truncation,
323 - self._apply_length_check,
353 + self._token_padding,
354 + self._token_truncation,
355 + self._token_len_check,
356 ):
357 tokens = validator(tokenizer, tokens)
358
vllm/tokenizers/deepseek_v32.py
+4
@@ -115,6 +115,10 @@ class DeepseekV32Tokenizer(CachedHfTokenizer):
115 def max_token_id(self) -> int:
116 return self.tokenizer.max_token_id
117
118 + @property
119 + def max_chars_per_token(self) -> int:
120 + return self.tokenizer.max_chars_per_token
121 +
122 @property
123 def truncation_side(self) -> str:
124 return self.tokenizer.truncation_side
vllm/tokenizers/grok2.py
+6
@@ -277,6 +277,8 @@ class Grok2Tokenizer(TokenizerLike):
277 self._pad_token_id = self._special_tokens.get(PAD, self._eos_token_id)
278 self._unk_token_id = self._pad_token_id
279
280 + self._max_chars_per_token = max(len(tok) for tok in self._token_to_id)
281 +
282 def num_special_tokens_to_add(self) -> int:
283 return 0
284
@@ -312,6 +314,10 @@ class Grok2Tokenizer(TokenizerLike):
314 def max_token_id(self) -> int:
315 return self._tokenizer.n_vocab - 1
316
317 + @property
318 + def max_chars_per_token(self) -> int:
319 + return self._max_chars_per_token
320 +
321 @property
322 def truncation_side(self) -> str:
323 return self._truncation_side
vllm/tokenizers/hf.py
+6
@@ -28,6 +28,8 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
28 tokenizer_len = len(tokenizer)
29
30 max_token_id = max(tokenizer_vocab.values())
31 + max_chars_per_token = max(len(tok) for tok in tokenizer_vocab)
32 +
33 # Some tokenizers (e.g., QwenTokenizer) have special tokens that
34 # are added and included in the implementation of the vocab_size
35 # property, but not in get_vocab(); if there is an implementation
@@ -49,6 +51,10 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
51 def max_token_id(self) -> int:
52 return max_token_id
53
54 + @property
55 + def max_chars_per_token(self) -> int:
56 + return max_chars_per_token
57 +
58 def get_vocab(self) -> dict[str, int]:
59 return tokenizer_vocab
60
vllm/tokenizers/mistral.py
+5
@@ -272,6 +272,7 @@ class MistralTokenizer(TokenizerLike):
272 # Vocab sorted by token id.
273 self._vocab = self.tokenizer.vocab()
274 self._max_token_id = self.vocab_size - 1
275 + self._max_chars_per_token = max(len(tok) for tok in self._vocab)
276
277 # Cache special tokens for faster access.
278 self._special_token_ids = self._get_special_token_ids()
@@ -325,6 +326,10 @@ class MistralTokenizer(TokenizerLike):
326 def max_token_id(self) -> int:
327 return self._max_token_id
328
329 + @property
330 + def max_chars_per_token(self) -> int:
331 + return self._max_chars_per_token
332 +
333 @property
334 def truncation_side(self) -> str:
335 return self.transformers_tokenizer.truncation_side
vllm/tokenizers/protocol.py
+4
@@ -57,6 +57,10 @@ class TokenizerLike(Protocol):
57 def max_token_id(self) -> int:
58 raise NotImplementedError
59
60 + @property
61 + def max_chars_per_token(self) -> int:
62 + raise NotImplementedError
63 +
64 @property
65 def truncation_side(self) -> str:
66 raise NotImplementedError