@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f0a1c8453a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 06:15 UTC

[Frontend] Use new Renderer for Completions and Tokenize API (#32863)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 31, 2026 at 20:51 UTC f0a1c8453ad1c664c8a04c83fe545195fcd556eb

64 files changed +2117 -2004

examples/online_serving/prompt_embed_inference_with_openai_client.py

+1 -3

@@ -60,9 +60,7 @@ def main():

61 completion = client.completions.create(

62 model=model_name,

63 - # NOTE: The OpenAI client does not allow `None` as an input to

64 - # `prompt`. Use an empty string if you have no text prompts.

65 - prompt="",

63 + prompt=None,

64 max_tokens=5,

65 temperature=0.0,

66 # NOTE: The OpenAI client allows passing in extra JSON body via the

tests/engine/test_short_mm_context.py

+5 -1

@@ -22,7 +22,11 @@ def test_context_length_too_short(vllm_runner, image_assets, model):

22 with pytest.raises(ValueError, match="longer than the maximum model length"):

23 vllm_model = vllm_runner(

24 model,

25 - max_model_len=128, # LLaVA has a feature size of 576

25 + # LLaVA has a feature size of 576

26 + # For the HF processor to execute successfully but still

27 + # failing the overall context length check, we need the

28 + # max_model_len to at least contain all image tokens

29 + max_model_len=579,

30 enforce_eager=True,

31 load_format="dummy",

32 )

tests/entrypoints/llm/test_chat.py

+1 -1

@@ -205,7 +205,7 @@ def test_chat_batch_failure_cleanup(llm_for_failure_test):

205 valid_msg,

206 ]

207 sampling_params = SamplingParams(temperature=0, max_tokens=10)

208 - with pytest.raises(ValueError, match="longer than the maximum model length"):

208 + with pytest.raises(ValueError, match="context length is only"):

209 llm.chat(batch_1, sampling_params=sampling_params)

210 outputs_2 = llm.chat(batch_2, sampling_params=sampling_params)

211 assert len(outputs_2) == len(batch_2)

tests/entrypoints/openai/test_chat_error.py

+13 -16

@@ -15,7 +15,8 @@ from vllm.entrypoints.openai.engine.protocol import ErrorResponse

15 from vllm.entrypoints.openai.models.protocol import BaseModelPath

16 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

17 from vllm.outputs import CompletionOutput, RequestOutput

18 -from vllm.tokenizers import get_tokenizer

18 +from vllm.renderers.hf import HfRenderer

19 +from vllm.tokenizers.registry import tokenizer_args_from_config

20 from vllm.v1.engine.async_llm import AsyncLLM

22 MODEL_NAME = "openai-community/gpt2"

@@ -57,6 +58,15 @@ class MockModelConfig:

58 return self.diff_sampling_param or {}

61 +def _build_renderer(model_config: MockModelConfig):

62 + _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)

63 +

64 + return HfRenderer(

65 + model_config,

66 + tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

67 + )

68 +

69 +

70 def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:

71 models = OpenAIServingModels(

72 engine_client=engine,

@@ -71,18 +81,6 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:

81 chat_template_content_format="auto",

82 )

74 - async def _fake_process_inputs(

75 - request_id,

76 - engine_prompt,

77 - sampling_params,

78 - *,

79 - lora_request,

80 - trace_headers,

81 - priority,

82 - data_parallel_rank,

83 - ):

84 - return dict(engine_prompt), {}

85 -

84 async def _fake_preprocess_chat(*args, **kwargs):

85 # return conversation, engine_prompts

86 return (

@@ -90,7 +88,6 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:

88 [{"prompt_token_ids": [1, 2, 3]}],

89 )

93 - serving_chat._process_inputs = AsyncMock(side_effect=_fake_process_inputs)

91 serving_chat._preprocess_chat = AsyncMock(side_effect=_fake_preprocess_chat)

92 return serving_chat

@@ -99,11 +96,11 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:

96 async def test_chat_error_non_stream():

97 """test finish_reason='error' returns 500 InternalServerError (non-streaming)"""

98 mock_engine = MagicMock(spec=AsyncLLM)

102 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

99 mock_engine.errored = False

100 mock_engine.model_config = MockModelConfig()

101 mock_engine.input_processor = MagicMock()

102 mock_engine.io_processor = MagicMock()

103 + mock_engine.renderer = _build_renderer(mock_engine.model_config)

104

105 serving_chat = _build_serving_chat(mock_engine)

106

@@ -153,11 +150,11 @@ async def test_chat_error_non_stream():

150 async def test_chat_error_stream():

151 """test finish_reason='error' returns 500 InternalServerError (streaming)"""

152 mock_engine = MagicMock(spec=AsyncLLM)

156 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

153 mock_engine.errored = False

154 mock_engine.model_config = MockModelConfig()

155 mock_engine.input_processor = MagicMock()

156 mock_engine.io_processor = MagicMock()

157 + mock_engine.renderer = _build_renderer(mock_engine.model_config)

158

159 serving_chat = _build_serving_chat(mock_engine)

160

tests/entrypoints/openai/test_completion_error.py

+13 -18

@@ -4,7 +4,7 @@

4 from dataclasses import dataclass, field

5 from http import HTTPStatus

6 from typing import Any

7 -from unittest.mock import AsyncMock, MagicMock

7 +from unittest.mock import MagicMock

9 import pytest

@@ -15,7 +15,8 @@ from vllm.entrypoints.openai.engine.protocol import ErrorResponse

15 from vllm.entrypoints.openai.models.protocol import BaseModelPath

16 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

17 from vllm.outputs import CompletionOutput, RequestOutput

18 -from vllm.tokenizers import get_tokenizer

18 +from vllm.renderers.hf import HfRenderer

19 +from vllm.tokenizers.registry import tokenizer_args_from_config

20 from vllm.v1.engine.async_llm import AsyncLLM

22 MODEL_NAME = "openai-community/gpt2"

@@ -61,37 +62,31 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:

62 engine_client=engine,

63 base_model_paths=BASE_MODEL_PATHS,

64 )

64 - serving_completion = OpenAIServingCompletion(

65 + return OpenAIServingCompletion(

66 engine,

67 models,

68 request_logger=None,

69 )

70 - async def _fake_process_inputs(

71 - request_id,

72 - engine_prompt,

73 - sampling_params,

74 - *,

75 - lora_request,

76 - trace_headers,

77 - priority,

78 - data_parallel_rank,

79 - ):

80 - return dict(engine_prompt), {}

82 - serving_completion._process_inputs = AsyncMock(side_effect=_fake_process_inputs)

83 - return serving_completion

72 +def _build_renderer(model_config: MockModelConfig):

73 + _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)

74 +

75 + return HfRenderer(

76 + model_config,

77 + tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

78 + )

81 @pytest.mark.asyncio

82 async def test_completion_error_non_stream():

83 """test finish_reason='error' returns 500 InternalServerError (non-streaming)"""

84 mock_engine = MagicMock(spec=AsyncLLM)

90 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

85 mock_engine.errored = False

86 mock_engine.model_config = MockModelConfig()

87 mock_engine.input_processor = MagicMock()

88 mock_engine.io_processor = MagicMock()

89 + mock_engine.renderer = _build_renderer(mock_engine.model_config)

91 serving_completion = _build_serving_completion(mock_engine)

@@ -141,11 +136,11 @@ async def test_completion_error_non_stream():

136 async def test_completion_error_stream():

137 """test finish_reason='error' returns 500 InternalServerError (streaming)"""

138 mock_engine = MagicMock(spec=AsyncLLM)

144 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

139 mock_engine.errored = False

140 mock_engine.model_config = MockModelConfig()

141 mock_engine.input_processor = MagicMock()

142 mock_engine.io_processor = MagicMock()

143 + mock_engine.renderer = _build_renderer(mock_engine.model_config)

144

145 serving_completion = _build_serving_completion(mock_engine)

146

tests/entrypoints/openai/test_completion_with_prompt_embeds.py

+10 -10

@@ -110,7 +110,7 @@ async def test_completions_with_prompt_embeds(

110 # Test case: Single prompt embeds input

111 completion = await client_with_prompt_embeds.completions.create(

112 model=model_name,

113 - prompt="", # Add empty prompt as required parameter

113 + prompt=None,

114 max_tokens=5,

115 temperature=0.0,

116 extra_body={"prompt_embeds": encoded_embeds},

@@ -121,7 +121,7 @@ async def test_completions_with_prompt_embeds(

121 # Test case: batch completion with prompt_embeds

122 completion = await client_with_prompt_embeds.completions.create(

123 model=model_name,

124 - prompt="", # Add empty prompt as required parameter

124 + prompt=None,

125 max_tokens=5,

126 temperature=0.0,

127 extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},

@@ -133,7 +133,7 @@ async def test_completions_with_prompt_embeds(

133 # Test case: streaming with prompt_embeds

134 single_completion = await client_with_prompt_embeds.completions.create(

135 model=model_name,

136 - prompt="", # Add empty prompt as required parameter

136 + prompt=None,

137 max_tokens=5,

138 temperature=0.0,

139 extra_body={"prompt_embeds": encoded_embeds},

@@ -142,7 +142,7 @@ async def test_completions_with_prompt_embeds(

142

143 stream = await client_with_prompt_embeds.completions.create(

144 model=model_name,

145 - prompt="", # Add empty prompt as required parameter

145 + prompt=None,

146 max_tokens=5,

147 temperature=0.0,

148 stream=True,

@@ -162,7 +162,7 @@ async def test_completions_with_prompt_embeds(

162 # Test case: batch streaming with prompt_embeds

163 stream = await client_with_prompt_embeds.completions.create(

164 model=model_name,

165 - prompt="", # Add empty prompt as required parameter

165 + prompt=None,

166 max_tokens=5,

167 temperature=0.0,

168 stream=True,

@@ -197,7 +197,7 @@ async def test_completions_with_prompt_embeds(

197 )

198 completion_embeds_only = await client_with_prompt_embeds.completions.create(

199 model=model_name,

200 - prompt="",

200 + prompt=None,

201 max_tokens=5,

202 temperature=0.0,

203 extra_body={"prompt_embeds": encoded_embeds},

@@ -215,7 +215,7 @@ async def test_completions_errors_with_prompt_embeds(

215 # Test error case: invalid prompt_embeds

216 with pytest.raises(BadRequestError):

217 await client_with_prompt_embeds.completions.create(

218 - prompt="",

218 + prompt=None,

219 model=model_name,

220 max_tokens=5,

221 temperature=0.0,

@@ -237,7 +237,7 @@ async def test_completions_with_logprobs_and_prompt_embeds(

237 # Test case: Logprobs using prompt_embeds

238 completion = await client_with_prompt_embeds.completions.create(

239 model=model_name,

240 - prompt="", # Add empty prompt as required parameter

240 + prompt=None,

241 max_tokens=5,

242 temperature=0.0,

243 echo=False,

@@ -257,7 +257,7 @@ async def test_completions_with_logprobs_and_prompt_embeds(

257 # Test case: Log probs with batch completion and prompt_embeds

258 completion = await client_with_prompt_embeds.completions.create(

259 model=model_name,

260 - prompt="", # Add empty prompt as required parameter

260 + prompt=None,

261 max_tokens=5,

262 temperature=0.0,

263 echo=False,

@@ -287,7 +287,7 @@ async def test_prompt_logprobs_raises_error(

287 with pytest.raises(BadRequestError, match="not compatible"):

288 await client_with_prompt_embeds.completions.create(

289 model=MODEL_NAME,

290 - prompt="",

290 + prompt=None,

291 max_tokens=5,

292 temperature=0.0,

293 extra_body={"prompt_embeds": encoded_embeds, "prompt_logprobs": True},

tests/entrypoints/openai/test_embedding_shape_validation.py

+1 -1

@@ -7,7 +7,7 @@ Tests verify that embeddings with correct ndim but incorrect hidden_size

7 are rejected before they can cause crashes during model inference.

9 Validation is performed by the parser (MultiModalDataParser) and EmbeddingItems

10 -classes, not by CompletionRenderer or MediaIO classes.

10 +classes, not by MediaIO classes.

11 """

13 import pytest

tests/entrypoints/openai/test_lora_resolvers.py

+13 -8

@@ -16,7 +16,8 @@ from vllm.entrypoints.openai.models.protocol import BaseModelPath

16 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

17 from vllm.lora.request import LoRARequest

18 from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry

19 -from vllm.tokenizers import get_tokenizer

19 +from vllm.renderers.hf import HfRenderer

20 +from vllm.tokenizers.registry import tokenizer_args_from_config

21 from vllm.v1.engine.async_llm import AsyncLLM

23 MODEL_NAME = "openai-community/gpt2"

@@ -35,6 +36,7 @@ class MockModelConfig:

36 """Minimal mock ModelConfig for testing."""

38 model: str = MODEL_NAME

39 + runner_type = "generate"

40 tokenizer: str = MODEL_NAME

41 trust_remote_code: bool = False

42 tokenizer_mode: str = "auto"

@@ -85,15 +87,21 @@ def register_mock_resolver():

87 del LoRAResolverRegistry.resolvers[MOCK_RESOLVER_NAME]

90 +def _build_renderer(model_config: MockModelConfig):

91 + _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)

92 +

93 + return HfRenderer(

94 + model_config,

95 + tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

96 + )

97 +

98 +

99 @pytest.fixture

100 def mock_serving_setup():

101 """Provides a mocked engine and serving completion instance."""

102 mock_engine = MagicMock(spec=AsyncLLM)

103 mock_engine.errored = False

104

94 - tokenizer = get_tokenizer(MODEL_NAME)

95 - mock_engine.get_tokenizer = AsyncMock(return_value=tokenizer)

96 -

105 async def mock_add_lora_side_effect(lora_request: LoRARequest):

106 """Simulate engine behavior when adding LoRAs."""

107 if lora_request.lora_name == "test-lora":

@@ -118,6 +126,7 @@ def mock_serving_setup():

126 mock_engine.model_config = MockModelConfig()

127 mock_engine.input_processor = MagicMock()

128 mock_engine.io_processor = MagicMock()

129 + mock_engine.renderer = _build_renderer(mock_engine.model_config)

130

131 models = OpenAIServingModels(

132 engine_client=mock_engine,

@@ -128,10 +137,6 @@ def mock_serving_setup():

137 mock_engine, models, request_logger=None

138 )

139

131 - serving_completion._process_inputs = AsyncMock(

132 - return_value=(MagicMock(name="engine_request"), {})

133 - )

134 -

140 return mock_engine, serving_completion

141

142

tests/entrypoints/openai/test_prompt_validation.py

+4 -8

@@ -12,7 +12,7 @@ import regex as re

12 import torch

14 from vllm.config import ModelConfig

15 -from vllm.entrypoints.renderer import CompletionRenderer

15 +from vllm.renderers.embed_utils import safe_load_prompt_embeds

17 from ...utils import RemoteOpenAIServer

@@ -30,7 +30,7 @@ async def test_empty_prompt():

30 ):

31 await client.completions.create(

32 model=model_name,

33 - prompt="",

33 + prompt=None,

34 max_tokens=5,

35 temperature=0.0,

36 extra_body={"prompt_embeds": []},

@@ -63,7 +63,6 @@ def test_load_prompt_embeds(

63 ):

64 model_config = Mock(spec=ModelConfig)

65 model_config.enable_prompt_embeds = True

66 - renderer = CompletionRenderer(model_config, tokenizer=None)

67 # construct arbitrary tensors of various dtypes, layouts, and sizes.

68 # We need to check against different layouts to make sure that if a user

@@ -89,9 +88,7 @@ def test_load_prompt_embeds(

88 buffer.seek(0)

89 encoded_tensor = pybase64.b64encode(buffer.getvalue())

92 - loaded_prompt_embeds = renderer.load_prompt_embeds(encoded_tensor)

93 - assert len(loaded_prompt_embeds) == 1

94 - loaded_tensor = loaded_prompt_embeds[0]["prompt_embeds"]

91 + loaded_tensor = safe_load_prompt_embeds(model_config, encoded_tensor)

92 assert loaded_tensor.device.type == "cpu"

93 assert loaded_tensor.layout == torch.strided

94 torch.testing.assert_close(

@@ -105,7 +102,6 @@ def test_load_prompt_embeds(

102 def test_disable_prompt_embeds(dtype: torch.dtype, seq_len: int, hidden_size: int):

103 model_config = Mock(spec=ModelConfig)

104 model_config.enable_prompt_embeds = False

108 - renderer = CompletionRenderer(model_config, tokenizer=None)

105

106 tensor = torch.randn((seq_len, hidden_size), dtype=dtype)

107

@@ -115,4 +111,4 @@ def test_disable_prompt_embeds(dtype: torch.dtype, seq_len: int, hidden_size: in

111 encoded_tensor = pybase64.b64encode(buffer.getvalue())

112

113 with pytest.raises(ValueError, match="--enable-prompt-embeds"):

118 - renderer.load_prompt_embeds(encoded_tensor)

114 + safe_load_prompt_embeds(model_config, encoded_tensor)

tests/entrypoints/openai/test_serving_chat.py

+26 -21

@@ -556,19 +556,6 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:

556 request_logger=None,

557 )

558

559 - async def _fake_process_inputs(

560 - request_id,

561 - engine_prompt,

562 - sampling_params,

563 - *,

564 - lora_request,

565 - trace_headers,

566 - priority,

567 - data_parallel_rank,

568 - ):

569 - return dict(engine_prompt), {}

570 -

571 - serving_chat._process_inputs = AsyncMock(side_effect=_fake_process_inputs)

559 return serving_chat

560

561

@@ -784,7 +771,7 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():

771

772 resp = await serving_chat.create_chat_completion(req)

773 assert isinstance(resp, ErrorResponse)

787 - assert "max_tokens" in resp.error.message

774 + assert "context length is only" in resp.error.message

775

776

777 @pytest.mark.asyncio

@@ -824,7 +811,7 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():

811

812 resp = await serving_chat.create_chat_completion(req)

813 assert isinstance(resp, ErrorResponse)

827 - assert "maximum context length" in resp.error.message

814 + assert "context length is only" in resp.error.message

815

816

817 @pytest.mark.asyncio

@@ -890,6 +877,20 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):

877

878 serving_chat = _build_serving_chat(mock_engine)

879

880 + orig_render_chat_request = serving_chat.render_chat_request

881 + captured_prompts = []

882 +

883 + async def render_chat_request(request):

884 + result = await orig_render_chat_request(request)

885 +

886 + assert isinstance(result, tuple)

887 + conversation, engine_prompts = result

888 + captured_prompts.extend(engine_prompts)

889 +

890 + return result

891 +

892 + serving_chat.render_chat_request = render_chat_request

893 +

894 # Test cache_salt

895 req = ChatCompletionRequest(

896 model=MODEL_NAME,

@@ -899,15 +900,19 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):

900 # By default, cache_salt in the engine prompt is not set

901 with suppress(Exception):

902 await serving_chat.create_chat_completion(req)

902 - engine_prompt = serving_chat._process_inputs.await_args_list[0].args[1]

903 - assert "cache_salt" not in engine_prompt

903 +

904 + assert len(captured_prompts) == 1

905 + assert "cache_salt" not in captured_prompts[0]

906 +

907 + captured_prompts.clear()

908

909 # Test with certain cache_salt

910 req.cache_salt = "test_salt"

911 with suppress(Exception):

912 await serving_chat.create_chat_completion(req)

909 - engine_prompt = serving_chat._process_inputs.await_args_list[1].args[1]

910 - assert engine_prompt.get("cache_salt") == "test_salt"

913 +

914 + assert len(captured_prompts) == 1

915 + assert captured_prompts[0]["cache_salt"] == "test_salt"

916

917

918 @pytest.mark.asyncio

@@ -1007,11 +1012,11 @@ class TestServingChatWithHarmony:

1012 @pytest.fixture()

1013 def mock_engine(self) -> AsyncLLM:

1014 mock_engine = MagicMock(spec=AsyncLLM)

1010 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

1015 mock_engine.errored = False

1016 mock_engine.model_config = MockModelConfig()

1017 mock_engine.input_processor = MagicMock()

1018 mock_engine.io_processor = MagicMock()

1019 + mock_engine.renderer = _build_renderer(mock_engine.model_config)

1020 return mock_engine

1021

1022 @pytest.fixture()

@@ -1618,11 +1623,11 @@ async def test_tool_choice_validation_without_parser():

1623 """Test that tool_choice='required' or named tool without tool_parser

1624 returns an appropriate error message."""

1625 mock_engine = MagicMock(spec=AsyncLLM)

1621 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

1626 mock_engine.errored = False

1627 mock_engine.model_config = MockModelConfig()

1628 mock_engine.input_processor = MagicMock()

1629 mock_engine.io_processor = MagicMock()

1630 + mock_engine.renderer = _build_renderer(mock_engine.model_config)

1631

1632 models = OpenAIServingModels(

1633 engine_client=mock_engine,

tests/entrypoints/pooling/basic/test_truncation.py

-14

@@ -67,20 +67,6 @@ async def test_smaller_truncation_size(client: openai.AsyncOpenAI):

67 assert response["usage"]["prompt_tokens"] == truncation_size

70 -@pytest.mark.asyncio

71 -async def test_zero_truncation_size(client: openai.AsyncOpenAI):

72 - truncation_size = 0

73 - kwargs: dict[str, Any] = {

74 - "model": MODEL_NAME,

75 - "input": input,

76 - "truncate_prompt_tokens": truncation_size,

77 - }

78 -

79 - response = await client.post(path="embeddings", cast_to=object, body={**kwargs})

80 -

81 - assert response["usage"]["prompt_tokens"] == truncation_size

82 -

83 -

70 @pytest.mark.asyncio

71 async def test_bigger_truncation_size(client: openai.AsyncOpenAI):

72 truncation_size = max_model_len + 1

tests/entrypoints/pooling/classify/test_online.py

+3 -5

@@ -128,12 +128,10 @@ def test_empty_input_error(server: RemoteOpenAIServer, model_name: str):

128 server.url_for("classify"),

129 json={"model": model_name, "input": []},

130 )

131 - classification_response.raise_for_status()

132 - output = ClassificationResponse.model_validate(classification_response.json())

131

134 - assert output.object == "list"

135 - assert isinstance(output.data, list)

136 - assert len(output.data) == 0

132 + error = classification_response.json()

133 + assert classification_response.status_code == 400

134 + assert "error" in error

135

136

137 @pytest.mark.parametrize("model_name", [MODEL_NAME])

tests/entrypoints/pooling/score/test_online_score.py

+1 -1

@@ -247,7 +247,7 @@ class TestModel:

247 },

248 )

249 assert score_response.status_code == 400

250 - assert "Please, select a smaller truncation size." in score_response.text

250 + assert "Please request a smaller truncation size." in score_response.text

251

252 def test_invocations(self, server: RemoteOpenAIServer, model: dict[str, Any]):

253 queries = "What is the capital of France?"

tests/entrypoints/test_renderer.py deleted

-325

@@ -1,325 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import io

5 -from dataclasses import dataclass

6 -from unittest.mock import AsyncMock, MagicMock

7 -

8 -import pybase64

9 -import pytest

10 -import torch

11 -

12 -from vllm.entrypoints.renderer import CompletionRenderer, RenderConfig

13 -from vllm.inputs.data import is_embeds_prompt

14 -

15 -

16 -@dataclass

17 -class MockModelConfig:

18 - max_model_len: int = 100

19 - encoder_config: dict | None = None

20 - enable_prompt_embeds: bool = True

21 -

22 -

23 -class MockTokenizerResult:

24 - def __init__(self, input_ids):

25 - self.input_ids = input_ids

26 -

27 -

28 -@pytest.fixture

29 -def mock_model_config():

30 - return MockModelConfig()

31 -

32 -

33 -@pytest.fixture

34 -def mock_tokenizer():

35 - tokenizer = MagicMock()

36 - return tokenizer

37 -

38 -

39 -@pytest.fixture

40 -def mock_async_tokenizer():

41 - async_tokenizer = AsyncMock()

42 - return async_tokenizer

43 -

44 -

45 -@pytest.fixture

46 -def renderer(mock_model_config, mock_tokenizer):

47 - return CompletionRenderer(

48 - model_config=mock_model_config,

49 - tokenizer=mock_tokenizer,

50 - async_tokenizer_pool={},

51 - )

52 -

53 -

54 -class TestRenderPrompt:

55 - """Test Category A: Basic Functionality Tests"""

56 -

57 - @pytest.mark.asyncio

58 - async def test_token_input(self, renderer):

59 - tokens = [101, 7592, 2088]

60 - results = await renderer.render_prompt(

61 - prompt_or_prompts=tokens, config=RenderConfig(max_length=100)

62 - )

63 -

64 - assert len(results) == 1

65 - assert results[0]["prompt_token_ids"] == tokens

66 -

67 - @pytest.mark.asyncio

68 - async def test_token_list_input(self, renderer):

69 - token_lists = [[101, 7592, 2088], [102, 1234, 5678, 9012], [103, 4567]]

70 - results = await renderer.render_prompt(

71 - prompt_or_prompts=token_lists, config=RenderConfig(max_length=100)

72 - )

73 -

74 - assert len(results) == 3

75 - assert results[0]["prompt_token_ids"] == [101, 7592, 2088]

76 - assert results[1]["prompt_token_ids"] == [102, 1234, 5678, 9012]

77 - assert results[2]["prompt_token_ids"] == [103, 4567]

78 -

79 - @pytest.mark.asyncio

80 - async def test_text_input(self, renderer, mock_async_tokenizer):

81 - mock_async_tokenizer.return_value = MockTokenizerResult([101, 7592, 2088])

82 - renderer.async_tokenizer_pool[renderer.tokenizer] = mock_async_tokenizer

83 -

84 - results = await renderer.render_prompt(

85 - prompt_or_prompts="Hello world", config=RenderConfig(max_length=100)

86 - )

87 -

88 - assert len(results) == 1

89 - assert results[0]["prompt_token_ids"] == [101, 7592, 2088]

90 - mock_async_tokenizer.assert_called_once()

91 -

92 - @pytest.mark.asyncio

93 - async def test_text_list_input(self, renderer, mock_async_tokenizer):

94 - mock_async_tokenizer.return_value = MockTokenizerResult([101, 7592, 2088])

95 - renderer.async_tokenizer_pool[renderer.tokenizer] = mock_async_tokenizer

96 -

97 - text_list_input = ["Hello world", "How are you?", "Good morning"]

98 - results = await renderer.render_prompt(

99 - prompt_or_prompts=text_list_input, config=RenderConfig(max_length=100)

100 - )

101 -

102 - assert len(results) == 3

103 - for result in results:

104 - assert result["prompt_token_ids"] == [101, 7592, 2088]

105 - assert mock_async_tokenizer.call_count == 3

106 -

107 - @pytest.mark.asyncio

108 - async def test_no_truncation(self, renderer, mock_async_tokenizer):

109 - mock_async_tokenizer.return_value = MockTokenizerResult([101, 7592, 2088])

110 - renderer.async_tokenizer_pool[renderer.tokenizer] = mock_async_tokenizer

111 -

112 - results = await renderer.render_prompt(

113 - prompt_or_prompts="Hello world", config=RenderConfig(max_length=100)

114 - )

115 -

116 - assert len(results) == 1

117 - call_args = mock_async_tokenizer.call_args

118 - assert (

119 - "truncation" not in call_args.kwargs

120 - or call_args.kwargs["truncation"] is False

121 - )

122 -

123 - @pytest.mark.asyncio

124 - async def test_truncation_positive(self, renderer, mock_async_tokenizer):

125 - mock_async_tokenizer.return_value = MockTokenizerResult(

126 - [101, 7592, 2088]

127 - ) # Truncated

128 - renderer.async_tokenizer_pool[renderer.tokenizer] = mock_async_tokenizer

129 -

130 - results = await renderer.render_prompt(

131 - prompt_or_prompts="Hello world",

132 - config=RenderConfig(max_length=100, truncate_prompt_tokens=50),

133 - )

134 -

135 - assert len(results) == 1

136 - call_args = mock_async_tokenizer.call_args

137 - assert call_args.kwargs["truncation"] is True

138 - assert call_args.kwargs["max_length"] == 50

139 -

140 - @pytest.mark.asyncio

141 - async def test_truncation_negative(self, renderer, mock_async_tokenizer):

142 - # Test that negative truncation uses model's max_model_len

143 - mock_async_tokenizer.return_value = MockTokenizerResult(

144 - [101, 7592, 2088]

145 - ) # Truncated to max_model_len

146 - renderer.async_tokenizer_pool[renderer.tokenizer] = mock_async_tokenizer

147 -

148 - results = await renderer.render_prompt(

149 - prompt_or_prompts="Hello world",

150 - config=RenderConfig(max_length=200, truncate_prompt_tokens=-1),

151 - )

152 -

153 - assert len(results) == 1

154 - call_args = mock_async_tokenizer.call_args

155 - assert call_args.kwargs["truncation"] is True

156 - assert call_args.kwargs["max_length"] == 100 # model's max_model_len

157 -

158 - @pytest.mark.asyncio

159 - async def test_token_truncation_last_elements(self, renderer):

160 - # Test that token truncation keeps the last N elements

161 - long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens

162 - results = await renderer.render_prompt(

163 - prompt_or_prompts=long_tokens,

164 - config=RenderConfig(max_length=100, truncate_prompt_tokens=5),

165 - )

166 -

167 - assert len(results) == 1

168 - # Should keep the last 5 tokens: [105, 106, 107, 108, 109]

169 - assert results[0]["prompt_token_ids"] == [105, 106, 107, 108, 109]

170 -

171 - @pytest.mark.asyncio

172 - async def test_max_length_exceeded(self, renderer):

173 - long_tokens = list(range(150)) # Exceeds max_model_len=100

174 -

175 - with pytest.raises(ValueError, match="maximum context length"):

176 - await renderer.render_prompt(

177 - prompt_or_prompts=long_tokens, config=RenderConfig(max_length=100)

178 - )

179 -

180 - @pytest.mark.asyncio

181 - async def test_no_tokenizer_for_text(self, mock_model_config):

182 - renderer_no_tokenizer = CompletionRenderer(

183 - model_config=mock_model_config, tokenizer=None, async_tokenizer_pool={}

184 - )

185 -

186 - with pytest.raises(ValueError, match="No tokenizer available"):

187 - await renderer_no_tokenizer.render_prompt(

188 - prompt_or_prompts="Hello world", config=RenderConfig(max_length=100)

189 - )

190 -

191 - @pytest.mark.asyncio

192 - async def test_token_input_with_needs_detokenization(

193 - self, renderer, mock_async_tokenizer

194 - ):

195 - # When needs_detokenization=True for token inputs, renderer should

196 - # use the async tokenizer to decode and include the original text

197 - # in the returned prompt object.

198 - mock_async_tokenizer.decode = AsyncMock(return_value="decoded text")

199 - renderer.async_tokenizer_pool[renderer.tokenizer] = mock_async_tokenizer

200 -

201 - tokens = [1, 2, 3, 4]

202 - results = await renderer.render_prompt(

203 - prompt_or_prompts=tokens,

204 - config=RenderConfig(needs_detokenization=True),

205 - )

206 -

207 - assert len(results) == 1

208 - assert results[0]["prompt_token_ids"] == tokens

209 - assert results[0]["prompt"] == "decoded text"

210 - mock_async_tokenizer.decode.assert_awaited_once()

211 -

212 -

213 -class TestRenderEmbedPrompt:

214 - def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes:

215 - """Helper to create base64-encoded tensor bytes"""

216 - buffer = io.BytesIO()

217 - torch.save(tensor, buffer)

218 - buffer.seek(0)

219 - return pybase64.b64encode(buffer.read())

220 -

221 - @pytest.mark.asyncio

222 - async def test_single_prompt_embed(self, renderer):

223 - # Create a test tensor

224 - test_tensor = torch.randn(10, 768, dtype=torch.float32)

225 - embed_bytes = self._create_test_embed_bytes(test_tensor)

226 -

227 - results = await renderer.render_prompt_and_embeds(

228 - prompt_embeds=embed_bytes,

229 - config=RenderConfig(cache_salt="test_salt"),

230 - )

231 -

232 - assert len(results) == 1

233 - assert is_embeds_prompt(results[0])

234 - assert torch.allclose(results[0]["prompt_embeds"], test_tensor)

235 - assert results[0]["cache_salt"] == "test_salt"

236 -

237 - @pytest.mark.asyncio

238 - async def test_multiple_prompt_embeds(self, renderer):

239 - # Create multiple test tensors

240 - test_tensors = [

241 - torch.randn(8, 512, dtype=torch.float32),

242 - torch.randn(12, 512, dtype=torch.float32),

243 - ]

244 - embed_bytes_list = [self._create_test_embed_bytes(t) for t in test_tensors]

245 -

246 - results = await renderer.render_prompt_and_embeds(

247 - prompt_embeds=embed_bytes_list,

248 - config=RenderConfig(),

249 - )

250 -

251 - assert len(results) == 2

252 - for i, result in enumerate(results):

253 - assert is_embeds_prompt(result)

254 - assert torch.allclose(result["prompt_embeds"], test_tensors[i])

255 -

256 - @pytest.mark.asyncio

257 - async def test_prompt_embed_truncation(self, renderer):

258 - # Create tensor with more tokens than truncation limit

259 - test_tensor = torch.randn(20, 768, dtype=torch.float32)

260 - embed_bytes = self._create_test_embed_bytes(test_tensor)

261 -

262 - results = await renderer.render_prompt_and_embeds(

263 - prompt_embeds=embed_bytes,

264 - config=RenderConfig(truncate_prompt_tokens=10),

265 - )

266 -

267 - assert len(results) == 1

268 - # Should keep last 10 tokens

269 - expected = test_tensor[-10:]

270 - assert torch.allclose(results[0]["prompt_embeds"], expected)

271 -

272 - @pytest.mark.asyncio

273 - async def test_prompt_embed_different_dtypes(self, renderer):

274 - # Test different supported dtypes

275 - dtypes = [torch.float32, torch.float16, torch.bfloat16]

276 -

277 - for dtype in dtypes:

278 - test_tensor = torch.randn(5, 256, dtype=dtype)

279 - embed_bytes = self._create_test_embed_bytes(test_tensor)

280 -

281 - results = await renderer.render_prompt_and_embeds(

282 - prompt_embeds=embed_bytes,

283 - config=RenderConfig(),

284 - )

285 -

286 - assert len(results) == 1

287 - assert results[0]["prompt_embeds"].dtype == dtype

288 -

289 - @pytest.mark.asyncio

290 - async def test_prompt_embed_squeeze_batch_dim(self, renderer):

291 - # Test tensor with batch dimension gets squeezed

292 - test_tensor = torch.randn(1, 10, 768, dtype=torch.float32)

293 - embed_bytes = self._create_test_embed_bytes(test_tensor)

294 -

295 - results = await renderer.render_prompt_and_embeds(

296 - prompt_embeds=embed_bytes,

297 - config=RenderConfig(),

298 - )

299 -

300 - assert len(results) == 1

301 - # Should be squeezed to 2D

302 - assert results[0]["prompt_embeds"].shape == (10, 768)

303 -

304 - @pytest.mark.asyncio

305 - async def test_both_prompts_and_embeds(self, renderer, mock_async_tokenizer):

306 - # Set up text tokenization

307 - mock_async_tokenizer.return_value = MockTokenizerResult([101, 102, 103])

308 - renderer.async_tokenizer_pool[renderer.tokenizer] = mock_async_tokenizer

309 -

310 - # Create embed

311 - test_tensor = torch.randn(5, 256, dtype=torch.float32)

312 - embed_bytes = self._create_test_embed_bytes(test_tensor)

313 -

314 - results = await renderer.render_prompt_and_embeds(

315 - prompt_or_prompts="Hello world",

316 - prompt_embeds=embed_bytes,

317 - config=RenderConfig(),

318 - )

319 -

320 - assert len(results) == 2

321 - # First should be embed prompt

322 - assert is_embeds_prompt(results[0])

323 - # Second should be tokens prompt

324 - assert "prompt_token_ids" in results[1]

325 - assert results[1]["prompt_token_ids"] == [101, 102, 103]

tests/models/language/pooling/test_mm_classifier_conversion.py

+1 -1

@@ -96,7 +96,7 @@ def test_gemma_multimodal(

96 dtype="bfloat16",

97 ) as vllm_model:

98 llm = vllm_model.get_llm()

99 - prompts = llm.preprocess_chat(messages)

99 + prompts = llm._preprocess_chat([messages])

100

101 result = llm.classify(prompts)

102 assert result[0].outputs.probs[0] > 0.95

tests/models/language/pooling/test_truncation_control.py

+6 -3

@@ -29,7 +29,8 @@ def test_smaller_truncation_size(

29 model_name, runner="pooling", max_model_len=max_model_len

30 ) as vllm_model:

31 vllm_output = vllm_model.llm.embed(

32 - input_str, truncate_prompt_tokens=truncate_prompt_tokens

32 + input_str,

33 + tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),

34 )

36 prompt_tokens = vllm_output[0].prompt_token_ids

@@ -44,7 +45,8 @@ def test_max_truncation_size(vllm_runner, model_name=MODEL_NAME, input_str=input

45 model_name, runner="pooling", max_model_len=max_model_len

46 ) as vllm_model:

47 vllm_output = vllm_model.llm.embed(

47 - input_str, truncate_prompt_tokens=truncate_prompt_tokens

48 + input_str,

49 + tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),

50 )

52 prompt_tokens = vllm_output[0].prompt_token_ids

@@ -64,7 +66,8 @@ def test_bigger_truncation_size(

66 ) as vllm_model,

67 ):

68 llm_output = vllm_model.llm.embed(

67 - input_str, truncate_prompt_tokens=truncate_prompt_tokens

69 + input_str,

70 + tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),

71 )

73 assert (

tests/models/language/pooling_mteb_test/mteb_embed_utils.py

+4 -1

@@ -187,7 +187,10 @@ def mteb_test_embed_models(

187 head_dtype = model_config.head_dtype

188

189 # Test embedding_size, isnan and whether to use normalize

190 - vllm_outputs = vllm_model.embed(example_prompts, truncate_prompt_tokens=-1)

190 + vllm_outputs = vllm_model.embed(

191 + example_prompts,

192 + tokenization_kwargs=dict(truncate_prompt_tokens=-1),

193 + )

194 outputs_tensor = torch.tensor(vllm_outputs)

195 assert not torch.any(torch.isnan(outputs_tensor))

196 embedding_size = model_config.embedding_size

tests/models/language/pooling_mteb_test/mteb_score_utils.py

+1 -1

@@ -79,9 +79,9 @@ class VllmMtebCrossEncoder(MtebCrossEncoderMixin):

79 outputs = self.llm.score(

80 queries,

81 corpus,

82 - truncate_prompt_tokens=-1,

82 use_tqdm=False,

83 chat_template=self.chat_template,

84 + tokenization_kwargs={"truncate_prompt_tokens": -1},

85 )

86 scores = np.array(outputs)

87 scores = scores[np.argsort(r)]

tests/renderers/test_completions.py new

+426

@@ -0,0 +1,426 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import io

5 +from dataclasses import dataclass

6 +from typing import Any

7 +from unittest.mock import AsyncMock

8 +

9 +import pybase64

10 +import pytest

11 +import torch

12 +

13 +from vllm.inputs.data import is_embeds_prompt

14 +from vllm.renderers import TokenizeParams

15 +from vllm.renderers.hf import HfRenderer

16 +from vllm.tokenizers.registry import tokenizer_args_from_config

17 +

18 +MODEL_NAME = "openai-community/gpt2"

19 +

20 +

21 +@dataclass

22 +class MockHFConfig:

23 + model_type: str = "any"

24 +

25 +

26 +@dataclass

27 +class MockModelConfig:

28 + runner_type = "generate"

29 + model: str = MODEL_NAME

30 + tokenizer: str = MODEL_NAME

31 + trust_remote_code: bool = False

32 + max_model_len: int = 100

33 + tokenizer_revision = None

34 + tokenizer_mode = "auto"

35 + hf_config = MockHFConfig()

36 + encoder_config: dict[str, Any] | None = None

37 + enable_prompt_embeds: bool = True

38 + skip_tokenizer_init: bool = False

39 +

40 +

41 +@pytest.fixture

42 +def mock_model_config():

43 + return MockModelConfig()

44 +

45 +

46 +@pytest.fixture

47 +def mock_async_tokenizer():

48 + return AsyncMock()

49 +

50 +

51 +@pytest.fixture

52 +def renderer(mock_model_config):

53 + _, tokenizer_name, _, kwargs = tokenizer_args_from_config(mock_model_config)

54 +

55 + return HfRenderer(

56 + mock_model_config,

57 + tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

58 + )

59 +

60 +

61 +class TestValidatePrompt:

62 + STRING_INPUTS = [

63 + "",

64 + "foo",

65 + "foo bar",

66 + "foo baz bar",

67 + "foo bar qux baz",

68 + ]

69 +

70 + TOKEN_INPUTS = [

71 + [-1],

72 + [1],

73 + [1, 2],

74 + [1, 3, 4],

75 + [1, 2, 4, 3],

76 + ]

77 +

78 + INPUTS_SLICES = [

79 + slice(None, None, -1),

80 + slice(None, None, 2),

81 + slice(None, None, -2),

82 + ]

83 +

84 + # Test that a nested mixed-type list of lists raises a TypeError.

85 + def test_empty_input(self, renderer):

86 + with pytest.raises(ValueError, match="at least one prompt"):

87 + renderer.render_completions([])

88 +

89 + def test_invalid_type(self, renderer):

90 + with pytest.raises(TypeError, match="string or an array of tokens"):

91 + renderer.render_completions([[1, 2], ["foo", "bar"]])

92 +

93 + @pytest.mark.parametrize("string_input", STRING_INPUTS)

94 + def test_string_consistent(self, renderer, string_input: str):

95 + assert renderer.render_completions(string_input) == renderer.render_completions(

96 + [string_input]

97 + )

98 +

99 + @pytest.mark.parametrize("token_input", TOKEN_INPUTS)

100 + def test_token_consistent(self, renderer, token_input: list[int]):

101 + assert renderer.render_completions(token_input) == renderer.render_completions(

102 + [token_input]

103 + )

104 +

105 + @pytest.mark.parametrize("inputs_slice", INPUTS_SLICES)

106 + def test_string_slice(self, renderer, inputs_slice: slice):

107 + assert renderer.render_completions(self.STRING_INPUTS)[

108 + inputs_slice

109 + ] == renderer.render_completions(self.STRING_INPUTS[inputs_slice])

110 +

111 +

112 +class TestRenderPrompt:

113 + @pytest.mark.asyncio

114 + async def test_token_input(self, renderer):

115 + tokens = [101, 7592, 2088]

116 + prompts = await renderer.render_completions_async(tokens)

117 + results = await renderer.tokenize_prompts_async(

118 + prompts,

119 + TokenizeParams(max_total_tokens=100),

120 + )

121 +

122 + assert len(results) == 1

123 + assert results[0]["prompt_token_ids"] == tokens

124 +

125 + @pytest.mark.asyncio

126 + async def test_token_list_input(self, renderer):

127 + token_lists = [[101, 7592, 2088], [102, 1234, 5678, 9012], [103, 4567]]

128 + prompts = await renderer.render_completions_async(token_lists)

129 + results = await renderer.tokenize_prompts_async(

130 + prompts,

131 + TokenizeParams(max_total_tokens=100),

132 + )

133 +

134 + assert len(results) == 3

135 + assert results[0]["prompt_token_ids"] == [101, 7592, 2088]

136 + assert results[1]["prompt_token_ids"] == [102, 1234, 5678, 9012]

137 + assert results[2]["prompt_token_ids"] == [103, 4567]

138 +

139 + @pytest.mark.asyncio

140 + async def test_text_input(self, renderer, mock_async_tokenizer):

141 + mock_async_tokenizer.encode.return_value = [101, 7592, 2088]

142 + renderer._async_tokenizer = mock_async_tokenizer

143 +

144 + prompts = await renderer.render_completions_async("Hello world")

145 + results = await renderer.tokenize_prompts_async(

146 + prompts,

147 + TokenizeParams(max_total_tokens=100),

148 + )

149 +

150 + assert len(results) == 1

151 + assert results[0]["prompt_token_ids"] == [101, 7592, 2088]

152 + mock_async_tokenizer.encode.assert_called_once()

153 +

154 + @pytest.mark.asyncio

155 + async def test_text_list_input(self, renderer, mock_async_tokenizer):

156 + mock_async_tokenizer.encode.return_value = [101, 7592, 2088]

157 + renderer._async_tokenizer = mock_async_tokenizer

158 +

159 + text_list_input = ["Hello world", "How are you?", "Good morning"]

160 + prompts = await renderer.render_completions_async(text_list_input)

161 + results = await renderer.tokenize_prompts_async(

162 + prompts,

163 + TokenizeParams(max_total_tokens=100),

164 + )

165 +

166 + assert len(results) == 3

167 + for result in results:

168 + assert result["prompt_token_ids"] == [101, 7592, 2088]

169 + assert mock_async_tokenizer.encode.call_count == 3

170 +

171 + @pytest.mark.asyncio

172 + async def test_no_truncation(self, renderer, mock_async_tokenizer):

173 + mock_async_tokenizer.encode.return_value = [101, 7592, 2088]

174 + renderer._async_tokenizer = mock_async_tokenizer

175 +

176 + prompts = await renderer.render_completions_async("Hello world")

177 + results = await renderer.tokenize_prompts_async(

178 + prompts,

179 + TokenizeParams(max_total_tokens=100),

180 + )

181 +

182 + assert len(results) == 1

183 + call_args = mock_async_tokenizer.encode.call_args

184 + assert (

185 + "truncation" not in call_args.kwargs

186 + or call_args.kwargs["truncation"] is False

187 + )

188 +

189 + @pytest.mark.asyncio

190 + async def test_truncation_positive(self, renderer, mock_async_tokenizer):

191 + mock_async_tokenizer.encode.return_value = [101, 7592, 2088] # Truncated

192 + renderer._async_tokenizer = mock_async_tokenizer

193 +

194 + prompts = await renderer.render_completions_async("Hello world")

195 + results = await renderer.tokenize_prompts_async(

196 + prompts,

197 + TokenizeParams(

198 + max_total_tokens=200,

199 + truncate_prompt_tokens=50,

200 + ),

201 + )

202 +

203 + assert len(results) == 1

204 + call_args = mock_async_tokenizer.encode.call_args

205 + assert call_args.kwargs["truncation"] is True

206 + assert call_args.kwargs["max_length"] == 50

207 +

208 + @pytest.mark.asyncio

209 + async def test_truncation_negative(self, renderer, mock_async_tokenizer):

210 + # Test that negative truncation uses model's max_model_len

211 + mock_async_tokenizer.encode.return_value = [

212 + 101,

213 + 7592,

214 + 2088,

215 + ] # Truncated to max_model_len

216 + renderer._async_tokenizer = mock_async_tokenizer

217 +

218 + prompts = await renderer.render_completions_async("Hello world")

219 + results = await renderer.tokenize_prompts_async(

220 + prompts,

221 + TokenizeParams(

222 + max_total_tokens=200,

223 + truncate_prompt_tokens=-1,

224 + ),

225 + )

226 +

227 + assert len(results) == 1

228 + call_args = mock_async_tokenizer.encode.call_args

229 + assert call_args.kwargs["truncation"] is True

230 + assert call_args.kwargs["max_length"] == 200

231 +

232 + @pytest.mark.asyncio

233 + async def test_token_truncation_last_elements(self, renderer):

234 + # Test that token truncation keeps the last N elements

235 + long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens

236 + prompts = await renderer.render_completions_async(long_tokens)

237 + results = await renderer.tokenize_prompts_async(

238 + prompts,

239 + TokenizeParams(

240 + max_total_tokens=100,

241 + truncate_prompt_tokens=5,

242 + ),

243 + )

244 +

245 + assert len(results) == 1

246 + # Should keep the last 5 tokens: [105, 106, 107, 108, 109]

247 + assert results[0]["prompt_token_ids"] == [105, 106, 107, 108, 109]

248 +

249 + @pytest.mark.asyncio

250 + async def test_max_length_exceeded(self, renderer):

251 + long_tokens = list(range(150)) # Exceeds max_model_len=100

252 +

253 + prompts = await renderer.render_completions_async(long_tokens)

254 +

255 + with pytest.raises(ValueError, match="context length is only"):

256 + await renderer.tokenize_prompts_async(

257 + prompts,

258 + TokenizeParams(max_total_tokens=100),

259 + )

260 +

261 + @pytest.mark.asyncio

262 + async def test_no_tokenizer_for_text(self, renderer):

263 + renderer_no_tokenizer = HfRenderer.from_config(

264 + MockModelConfig(skip_tokenizer_init=True),

265 + tokenizer_kwargs={},

266 + )

267 +

268 + prompts = await renderer_no_tokenizer.render_completions_async("Hello world")

269 +

270 + with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"):

271 + await renderer_no_tokenizer.tokenize_prompts_async(

272 + prompts,

273 + TokenizeParams(max_total_tokens=100),

274 + )

275 +

276 + @pytest.mark.asyncio

277 + async def test_token_input_with_needs_detokenization(

278 + self, renderer, mock_async_tokenizer

279 + ):

280 + # When needs_detokenization=True for token inputs, renderer should

281 + # use the async tokenizer to decode and include the original text

282 + # in the returned prompt object.

283 + mock_async_tokenizer.decode = AsyncMock(return_value="decoded text")

284 + renderer._async_tokenizer = mock_async_tokenizer

285 +

286 + tokens = [1, 2, 3, 4]

287 + prompts = await renderer.render_completions_async(tokens)

288 + results = await renderer.tokenize_prompts_async(

289 + prompts,

290 + TokenizeParams(

291 + max_total_tokens=renderer.config.max_model_len,

292 + needs_detokenization=True,

293 + ),

294 + )

295 +

296 + assert len(results) == 1

297 + assert results[0]["prompt_token_ids"] == tokens

298 + assert results[0]["prompt"] == "decoded text"

299 + mock_async_tokenizer.decode.assert_awaited_once()

300 +

301 +

302 +class TestRenderEmbedPrompt:

303 + def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes:

304 + """Helper to create base64-encoded tensor bytes"""

305 + buffer = io.BytesIO()

306 + torch.save(tensor, buffer)

307 + buffer.seek(0)

308 + return pybase64.b64encode(buffer.read())

309 +

310 + @pytest.mark.asyncio

311 + async def test_single_prompt_embed(self, renderer):

312 + # Create a test tensor

313 + test_tensor = torch.randn(10, 768, dtype=torch.float32)

314 + embed_bytes = self._create_test_embed_bytes(test_tensor)

315 +

316 + prompts = await renderer.render_completions_async(prompt_embeds=embed_bytes)

317 + results = await renderer.tokenize_prompts_async(

318 + prompts,

319 + TokenizeParams(max_total_tokens=renderer.config.max_model_len),

320 + )

321 +

322 + assert len(results) == 1

323 + assert is_embeds_prompt(results[0])

324 + assert torch.allclose(results[0]["prompt_embeds"], test_tensor)

325 +

326 + @pytest.mark.asyncio

327 + async def test_multiple_prompt_embeds(self, renderer):

328 + # Create multiple test tensors

329 + test_tensors = [

330 + torch.randn(8, 512, dtype=torch.float32),

331 + torch.randn(12, 512, dtype=torch.float32),

332 + ]

333 + embed_bytes_list = [self._create_test_embed_bytes(t) for t in test_tensors]

334 +

335 + prompts = await renderer.render_completions_async(

336 + prompt_embeds=embed_bytes_list

337 + )

338 + results = await renderer.tokenize_prompts_async(

339 + prompts,

340 + TokenizeParams(max_total_tokens=renderer.config.max_model_len),

341 + )

342 +

343 + assert len(results) == 2

344 + for i, result in enumerate(results):

345 + assert is_embeds_prompt(result)

346 + assert torch.allclose(result["prompt_embeds"], test_tensors[i])

347 +

348 + @pytest.mark.asyncio

349 + async def test_prompt_embed_truncation(self, renderer):

350 + # Create tensor with more tokens than truncation limit

351 + test_tensor = torch.randn(20, 768, dtype=torch.float32)

352 + embed_bytes = self._create_test_embed_bytes(test_tensor)

353 +

354 + prompts = await renderer.render_completions_async(prompt_embeds=embed_bytes)

355 + results = await renderer.tokenize_prompts_async(

356 + prompts,

357 + TokenizeParams(

358 + max_total_tokens=renderer.config.max_model_len,

359 + truncate_prompt_tokens=10,

360 + ),

361 + )

362 +

363 + assert len(results) == 1

364 + # Should keep last 10 tokens

365 + expected = test_tensor[-10:]

366 + assert torch.allclose(results[0]["prompt_embeds"], expected)

367 +

368 + @pytest.mark.asyncio

369 + async def test_prompt_embed_different_dtypes(self, renderer):

370 + # Test different supported dtypes

371 + dtypes = [torch.float32, torch.float16, torch.bfloat16]

372 +

373 + for dtype in dtypes:

374 + test_tensor = torch.randn(5, 256, dtype=dtype)

375 + embed_bytes = self._create_test_embed_bytes(test_tensor)

376 +

377 + prompts = await renderer.render_completions_async(prompt_embeds=embed_bytes)

378 + results = await renderer.tokenize_prompts_async(

379 + prompts,

380 + TokenizeParams(max_total_tokens=renderer.config.max_model_len),

381 + )

382 +

383 + assert len(results) == 1

384 + assert results[0]["prompt_embeds"].dtype == dtype

385 +

386 + @pytest.mark.asyncio

387 + async def test_prompt_embed_squeeze_batch_dim(self, renderer):

388 + # Test tensor with batch dimension gets squeezed

389 + test_tensor = torch.randn(1, 10, 768, dtype=torch.float32)

390 + embed_bytes = self._create_test_embed_bytes(test_tensor)

391 +

392 + prompts = await renderer.render_completions_async(prompt_embeds=embed_bytes)

393 + results = await renderer.tokenize_prompts_async(

394 + prompts,

395 + TokenizeParams(max_total_tokens=renderer.config.max_model_len),

396 + )

397 +

398 + assert len(results) == 1

399 + # Should be squeezed to 2D

400 + assert results[0]["prompt_embeds"].shape == (10, 768)

401 +

402 + @pytest.mark.asyncio

403 + async def test_both_prompts_and_embeds(self, renderer, mock_async_tokenizer):

404 + # Set up text tokenization

405 + mock_async_tokenizer.encode.return_value = [101, 102, 103]

406 + renderer._async_tokenizer = mock_async_tokenizer

407 +

408 + # Create embed

409 + test_tensor = torch.randn(5, 256, dtype=torch.float32)

410 + embed_bytes = self._create_test_embed_bytes(test_tensor)

411 +

412 + prompts = await renderer.render_completions_async(

413 + "Hello world",

414 + prompt_embeds=embed_bytes,

415 + )

416 + results = await renderer.tokenize_prompts_async(

417 + prompts,

418 + TokenizeParams(max_total_tokens=renderer.config.max_model_len),

419 + )

420 +

421 + assert len(results) == 2

422 + # First should be embed prompt

423 + assert is_embeds_prompt(results[0])

424 + # Second should be tokens prompt

425 + assert "prompt_token_ids" in results[1]

426 + assert results[1]["prompt_token_ids"] == [101, 102, 103]

tests/renderers/test_mistral.py

+2 -1

@@ -9,6 +9,7 @@ import pytest

9 from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy

11 from vllm.config import ModelConfig

12 +from vllm.renderers import ChatParams

13 from vllm.renderers.mistral import MistralRenderer, safe_apply_chat_template

14 from vllm.tokenizers.mistral import MistralTokenizer

@@ -27,7 +28,7 @@ async def test_async_mistral_tokenizer_does_not_block_event_loop():

28 mock_renderer = MistralRenderer(Mock(spec=ModelConfig), tokenizer_kwargs={})

29 mock_renderer._tokenizer = mock_tokenizer

30 - task = mock_renderer.render_messages_async([])

31 + task = mock_renderer.render_messages_async([], ChatParams())

33 # Ensure the event loop is not blocked

34 blocked_count = 0

tests/renderers/test_sparse_tensor_validation.py renamed

+23 -74

@@ -1,8 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 """

4 -Sparse tensor validation in embedding APIs.

5 -

4 Tests verify that malicious sparse tensors are rejected before they can trigger

5 out-of-bounds memory writes during to_dense() operations.

6 """

@@ -13,8 +11,24 @@ import io

11 import pytest

12 import torch

16 -from vllm.entrypoints.renderer import CompletionRenderer

14 from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO

15 +from vllm.renderers.embed_utils import safe_load_prompt_embeds

16 +

17 +

18 +@pytest.fixture

19 +def model_config():

20 + """Mock ModelConfig for testing."""

21 + from vllm.config import ModelConfig

22 +

23 + return ModelConfig(

24 + model="facebook/opt-125m",

25 + tokenizer="facebook/opt-125m",

26 + tokenizer_mode="auto",

27 + trust_remote_code=False,

28 + dtype="float32",

29 + seed=0,

30 + enable_prompt_embeds=True, # Required for prompt embeds tests

31 + )

34 def _encode_tensor(tensor: torch.Tensor) -> bytes:

@@ -63,15 +77,12 @@ class TestPromptEmbedsValidation:

78 def test_valid_dense_tensor_accepted(self, model_config):

79 """Baseline: Valid dense tensors should work normally."""

66 - renderer = CompletionRenderer(model_config)

67 -

80 valid_tensor = _create_valid_dense_tensor()

81 encoded = _encode_tensor(valid_tensor)

83 # Should not raise any exception

72 - result = renderer.load_prompt_embeds(encoded)

73 - assert len(result) == 1

74 - assert result[0]["prompt_embeds"].shape == valid_tensor.shape

84 + result = safe_load_prompt_embeds(model_config, encoded)

85 + assert result.shape == valid_tensor.shape

87 def test_valid_sparse_tensor_accepted(self):

88 """Baseline: Valid sparse tensors should load successfully."""

@@ -86,14 +97,12 @@ class TestPromptEmbedsValidation:

98 def test_malicious_sparse_tensor_rejected(self, model_config):

99 """Security: Malicious sparse tensors should be rejected."""

89 - renderer = CompletionRenderer(model_config)

90 -

100 malicious_tensor = _create_malicious_sparse_tensor()

101 encoded = _encode_tensor(malicious_tensor)

102

103 # Should raise RuntimeError due to invalid sparse tensor

104 with pytest.raises((RuntimeError, ValueError)) as exc_info:

96 - renderer.load_prompt_embeds(encoded)

105 + safe_load_prompt_embeds(model_config, encoded)

106

107 # Error should indicate sparse tensor validation failure

108 error_msg = str(exc_info.value).lower()

@@ -101,8 +110,6 @@ class TestPromptEmbedsValidation:

110

111 def test_extremely_large_indices_rejected(self, model_config):

112 """Security: Sparse tensors with extremely large indices should be rejected."""

104 - renderer = CompletionRenderer(model_config)

105 -

113 # Create tensor with indices far beyond reasonable bounds

114 indices = torch.tensor([[999999], [999999]])

115 values = torch.tensor([1.0])

@@ -114,12 +121,10 @@ class TestPromptEmbedsValidation:

121 encoded = _encode_tensor(malicious_tensor)

122

123 with pytest.raises((RuntimeError, ValueError)):

117 - renderer.load_prompt_embeds(encoded)

124 + safe_load_prompt_embeds(model_config, encoded)

125

126 def test_negative_indices_rejected(self, model_config):

127 """Security: Sparse tensors with negative indices should be rejected."""

121 - renderer = CompletionRenderer(model_config)

122 -

128 # Create tensor with negative indices

129 indices = torch.tensor([[-1], [-1]])

130 values = torch.tensor([1.0])

@@ -131,7 +136,7 @@ class TestPromptEmbedsValidation:

136 encoded = _encode_tensor(malicious_tensor)

137

138 with pytest.raises((RuntimeError, ValueError)):

134 - renderer.load_prompt_embeds(encoded)

139 + safe_load_prompt_embeds(model_config, encoded)

140

141

142 class TestImageEmbedsValidation:

@@ -253,14 +258,12 @@ class TestSparseTensorValidationIntegration:

258 3. Sends to /v1/completions with prompt_embeds parameter

259 4. Server should reject before memory corruption occurs

260 """

256 - renderer = CompletionRenderer(model_config)

257 -

261 # Step 1-2: Attacker creates malicious payload

262 attack_payload = _encode_tensor(_create_malicious_sparse_tensor())

263

264 # Step 3-4: Server processes and should reject

265 with pytest.raises((RuntimeError, ValueError)):

263 - renderer.load_prompt_embeds(attack_payload)

266 + safe_load_prompt_embeds(model_config, attack_payload)

267

268 def test_attack_scenario_chat_api_image(self):

269 """

@@ -285,57 +288,3 @@ class TestSparseTensorValidationIntegration:

288

289 with pytest.raises((RuntimeError, ValueError)):

290 io_handler.load_base64("", attack_payload.decode("utf-8"))

288 -

289 - def test_multiple_valid_embeddings_in_batch(self, model_config):

290 - """

291 - Regression test: Multiple valid embeddings should still work.

292 -

293 - Ensures the fix doesn't break legitimate batch processing.

294 - """

295 - renderer = CompletionRenderer(model_config)

296 -

297 - valid_tensors = [

298 - _encode_tensor(_create_valid_dense_tensor()),

299 - _encode_tensor(_create_valid_dense_tensor()),

300 - _encode_tensor(_create_valid_dense_tensor()),

301 - ]

302 -

303 - # Should process all without error

304 - result = renderer.load_prompt_embeds(valid_tensors)

305 - assert len(result) == 3

306 -

307 - def test_mixed_valid_and_malicious_rejected(self, model_config):

308 - """

309 - Security: Batch with one malicious tensor should be rejected.

310 -

311 - Even if most tensors are valid, a single malicious one should

312 - cause rejection of the entire batch.

313 - """

314 - renderer = CompletionRenderer(model_config)

315 -

316 - mixed_batch = [

317 - _encode_tensor(_create_valid_dense_tensor()),

318 - _encode_tensor(_create_malicious_sparse_tensor()), # Malicious

319 - _encode_tensor(_create_valid_dense_tensor()),

320 - ]

321 -

322 - # Should fail on the malicious tensor

323 - with pytest.raises((RuntimeError, ValueError)):

324 - renderer.load_prompt_embeds(mixed_batch)

325 -

326 -

327 -# Pytest fixtures

328 -@pytest.fixture

329 -def model_config():

330 - """Mock ModelConfig for testing."""

331 - from vllm.config import ModelConfig

332 -

333 - return ModelConfig(

334 - model="facebook/opt-125m",

335 - tokenizer="facebook/opt-125m",

336 - tokenizer_mode="auto",

337 - trust_remote_code=False,

338 - dtype="float32",

339 - seed=0,

340 - enable_prompt_embeds=True, # Required for prompt embeds tests

341 - )

tests/test_inputs.py

-55

@@ -5,65 +5,10 @@ import pytest

6 from vllm.config import ModelConfig

7 from vllm.inputs import zip_enc_dec_prompts

8 -from vllm.inputs.parse import parse_raw_prompts

8 from vllm.inputs.preprocess import InputPreprocessor

10 pytestmark = pytest.mark.cpu_test

13 -STRING_INPUTS = [

14 - "",

15 - "foo",

16 - "foo bar",

17 - "foo baz bar",

18 - "foo bar qux baz",

19 -]

20 -

21 -TOKEN_INPUTS = [

22 - [-1],

23 - [1],

24 - [1, 2],

25 - [1, 3, 4],

26 - [1, 2, 4, 3],

27 -]

28 -

29 -INPUTS_SLICES = [

30 - slice(None, None, -1),

31 - slice(None, None, 2),

32 - slice(None, None, -2),

33 -]

34 -

35 -

36 -# Test that a nested mixed-type list of lists raises a TypeError.

37 -@pytest.mark.parametrize("invalid_input", [[[1, 2], ["foo", "bar"]]])

38 -def test_invalid_input_raise_type_error(invalid_input):

39 - with pytest.raises(TypeError):

40 - parse_raw_prompts(invalid_input)

41 -

42 -

43 -def test_parse_raw_single_batch_empty():

44 - with pytest.raises(ValueError, match="at least one prompt"):

45 - parse_raw_prompts([])

46 -

47 - with pytest.raises(ValueError, match="at least one prompt"):

48 - parse_raw_prompts([[]])

49 -

50 -

51 -@pytest.mark.parametrize("string_input", STRING_INPUTS)

52 -def test_parse_raw_single_batch_string_consistent(string_input: str):

53 - assert parse_raw_prompts(string_input) == parse_raw_prompts([string_input])

54 -

55 -

56 -@pytest.mark.parametrize("token_input", TOKEN_INPUTS)

57 -def test_parse_raw_single_batch_token_consistent(token_input: list[int]):

58 - assert parse_raw_prompts(token_input) == parse_raw_prompts([token_input])

59 -

60 -

61 -@pytest.mark.parametrize("inputs_slice", INPUTS_SLICES)

62 -def test_parse_raw_single_batch_string_slice(inputs_slice: slice):

63 - assert parse_raw_prompts(STRING_INPUTS)[inputs_slice] == parse_raw_prompts(

64 - STRING_INPUTS[inputs_slice]

65 - )

66 -

13 @pytest.mark.parametrize(

14 "mm_processor_kwargs,expected_mm_kwargs",

vllm/config/model.py

+2 -2

@@ -768,7 +768,7 @@ class ModelConfig:

768 )

769 self.tokenizer = object_storage_tokenizer.dir

770

771 - def _get_encoder_config(self):

771 + def _get_encoder_config(self) -> dict[str, Any] | None:

772 model = self.model

773 if is_remote_gguf(model):

774 model, _ = split_remote_gguf(model)

@@ -1918,7 +1918,7 @@ def _get_and_verify_max_len(

1918 disable_sliding_window: bool,

1919 sliding_window: int | None,

1920 spec_target_max_model_len: int | None = None,

1921 - encoder_config: Any | None = None,

1921 + encoder_config: dict[str, Any] | None = None,

1922 ) -> int:

1923 """Get and verify the model's maximum length."""

1924 (derived_max_model_len, max_len_key) = (

vllm/engine/protocol.py

+1 -6

@@ -72,14 +72,9 @@ class EngineClient(ABC):

72 lora_request: LoRARequest | None = None,

73 trace_headers: Mapping[str, str] | None = None,

74 priority: int = 0,

75 - truncate_prompt_tokens: int | None = None,

75 tokenization_kwargs: dict[str, Any] | None = None,

76 ) -> AsyncGenerator[PoolingRequestOutput, None]:

78 - """Generate outputs for a request from a pooling model.

79 -

80 - NOTE: truncate_prompt_tokens is deprecated in v0.14.

81 - TODO: Remove this argument in v0.15.

82 - """

77 + """Generate outputs for a request from a pooling model."""

78 ...

80 @abstractmethod

vllm/entrypoints/llm.py

+339 -199

@@ -2,8 +2,9 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 import itertools

5 +import warnings

6 from collections.abc import Callable, Sequence

6 -from typing import TYPE_CHECKING, Any, cast

7 +from typing import TYPE_CHECKING, Any, TypeAlias, cast

9 import cloudpickle

10 import torch.nn as nn

@@ -46,15 +47,17 @@ from vllm.entrypoints.pooling.score.utils import (

47 compress_token_type_ids,

48 get_score_prompt,

49 )

49 -from vllm.entrypoints.utils import _validate_truncation_size, log_non_default_args

50 +from vllm.entrypoints.utils import log_non_default_args

51 from vllm.inputs import (

52 DataPrompt,

53 + EmbedsPrompt,

54 + ExplicitEncoderDecoderPrompt,

55 PromptType,

56 SingletonPrompt,

57 TextPrompt,

58 TokensPrompt,

59 )

57 -from vllm.inputs.parse import get_prompt_components

60 +from vllm.inputs.parse import get_prompt_components, is_explicit_encoder_decoder_prompt

61 from vllm.logger import init_logger

62 from vllm.lora.request import LoRARequest

63 from vllm.model_executor.layers.quantization import QuantizationMethods

@@ -67,6 +70,7 @@ from vllm.outputs import (

70 )

71 from vllm.platforms import current_platform

72 from vllm.pooling_params import PoolingParams

73 +from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs

74 from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams

75 from vllm.tasks import PoolingTask

76 from vllm.tokenizers import TokenizerLike

@@ -74,7 +78,6 @@ from vllm.tokenizers.mistral import MistralTokenizer

78 from vllm.usage.usage_lib import UsageContext

79 from vllm.utils.collection_utils import as_iter, is_list_of

80 from vllm.utils.counter import Counter

77 -from vllm.v1.engine import EngineCoreRequest

81 from vllm.v1.engine.llm_engine import LLMEngine

82 from vllm.v1.sample.logits_processor import LogitsProcessor

@@ -85,6 +88,9 @@ logger = init_logger(__name__)

89 _R = TypeVar("_R", default=Any)

91 +EnginePrompt: TypeAlias = TextPrompt | TokensPrompt | EmbedsPrompt

92 +EngineEncDecPrompt: TypeAlias = ExplicitEncoderDecoderPrompt[EnginePrompt, EnginePrompt]

93 +

95 class LLM:

96 """An LLM for generating texts from given prompts and sampling parameters.

@@ -372,6 +378,7 @@ class LLM:

378 use_tqdm: bool | Callable[..., tqdm] = True,

379 lora_request: list[LoRARequest] | LoRARequest | None = None,

380 priority: list[int] | None = None,

381 + tokenization_kwargs: dict[str, Any] | None = None,

382 ) -> list[RequestOutput]:

383 """Generates the completions for the input prompts.

384

@@ -398,15 +405,11 @@ class LLM:

405 If provided, must be a list of integers matching the length

406 of `prompts`, where each priority value corresponds to the prompt

407 at the same index.

408 + tokenization_kwargs: Overrides for `tokenizer.encode`.

409

410 Returns:

411 A list of `RequestOutput` objects containing the

412 generated completions in the same order as the input prompts.

405 -

406 - Note:

407 - Using `prompts` and `prompt_token_ids` as keyword parameters is

408 - considered legacy and may be deprecated in the future. You should

409 - instead pass them via the `inputs` parameter.

413 """

414 model_config = self.model_config

415 runner_type = model_config.runner_type

@@ -418,17 +421,14 @@ class LLM:

421 )

422

423 if sampling_params is None:

421 - # Use default sampling params.

424 sampling_params = self.get_default_sampling_params()

425

424 - # Add any modality specific loras to the corresponding prompts

425 - lora_request = self._get_modality_specific_lora_reqs(prompts, lora_request)

426 -

426 self._validate_and_add_requests(

427 prompts=prompts,

428 params=sampling_params,

429 use_tqdm=use_tqdm,

431 - lora_request=lora_request,

430 + lora_request=self._get_modality_specific_lora_reqs(prompts, lora_request),

431 + tokenization_kwargs=tokenization_kwargs,

432 priority=priority,

433 )

434

@@ -771,65 +771,169 @@ class LLM:

771

772 return outputs

773

774 - def preprocess_chat(

774 + def _get_cmpl_tok_params(self, tokenization_kwargs: dict[str, Any] | None):

775 + model_config = self.model_config

776 + encoder_config = model_config.encoder_config or {}

777 +

778 + return TokenizeParams(

779 + max_total_tokens=model_config.max_model_len,

780 + do_lower_case=encoder_config.get("do_lower_case", False),

781 + # For Whisper, special tokens should be provided by the user based

782 + # on the task and language of their request. Also needed to avoid

783 + # appending an EOS token to the prompt which disrupts generation.

784 + add_special_tokens=not model_config.is_encoder_decoder,

785 + ).with_kwargs(tokenization_kwargs)

786 +

787 + def _normalize_prompts(

788 self,

776 - messages: list[ChatCompletionMessageParam]

789 + prompts: PromptType | Sequence[PromptType],

790 + ) -> list[EnginePrompt | EngineEncDecPrompt]:

791 + if isinstance(prompts, str):

792 + prompts = TextPrompt(prompt=prompts)

793 +

794 + return prompts if isinstance(prompts, Sequence) else [prompts] # type: ignore[return-value]

795 +

796 + def _preprocess_cmpl_singleton(

797 + self,

798 + prompt: SingletonPrompt,

799 + tok_params: TokenizeParams,

800 + *,

801 + tokenize: bool,

802 + ) -> EnginePrompt:

803 + renderer = self.llm_engine.renderer

804 +

805 + if not isinstance(prompt, dict):

806 + prompt = renderer.render_completion(prompt)

807 +

808 + return renderer.tokenize_prompt(prompt, tok_params) if tokenize else prompt

809 +

810 + def _preprocess_cmpl_enc_dec(

811 + self,

812 + prompt: ExplicitEncoderDecoderPrompt,

813 + tok_params: TokenizeParams,

814 + ) -> EngineEncDecPrompt:

815 + enc_prompt = prompt["encoder_prompt"]

816 + dec_prompt = prompt["decoder_prompt"]

817 +

818 + return EngineEncDecPrompt(

819 + encoder_prompt=self._preprocess_cmpl_singleton(

820 + enc_prompt,

821 + tok_params,

822 + # TODO: Move multi-modal processor into tokenization

823 + tokenize=not self.model_config.is_multimodal_model,

824 + ),

825 + decoder_prompt=(

826 + None

827 + if dec_prompt is None

828 + else self._preprocess_cmpl_singleton(

829 + dec_prompt,

830 + tok_params,

831 + # TODO: Move multi-modal processor into tokenization

832 + tokenize=not self.model_config.is_multimodal_model,

833 + )

834 + ),

835 + )

836 +

837 + def _preprocess_completion(

838 + self,

839 + prompts: PromptType | Sequence[PromptType],

840 + tokenization_kwargs: dict[str, Any] | None = None,

841 + ) -> list[EnginePrompt | EngineEncDecPrompt]:

842 + """

843 + Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into

844 + a format that can be passed to `_add_request`.

845 +

846 + Refer to [LLM.generate][] for a complete description of the arguments.

847 +

848 + Returns:

849 + A list of `TokensPrompts` objects containing the tokenized prompt

850 + after chat template interpolation, and the raw multi-modal inputs.

851 + """

852 + tok_params = self._get_cmpl_tok_params(tokenization_kwargs)

853 +

854 + engine_prompts = list[EnginePrompt | EngineEncDecPrompt]()

855 + for prompt in self._normalize_prompts(prompts):

856 + if is_explicit_encoder_decoder_prompt(prompt):

857 + engine_prompts.append(self._preprocess_cmpl_enc_dec(prompt, tok_params))

858 + else:

859 + # Some MM models have non-default `add_special_tokens`

860 + # TODO: Move multi-modal processor into tokenization

861 + engine_prompts.append(

862 + self._preprocess_cmpl_singleton(

863 + prompt,

864 + tok_params,

865 + tokenize=not self.model_config.is_multimodal_model,

866 + )

867 + )

868 +

869 + return engine_prompts

870 +

871 + def _normalize_conversations(

872 + self,

873 + conversations: list[ChatCompletionMessageParam]

874 + | list[list[ChatCompletionMessageParam]],

875 + ) -> list[list[ChatCompletionMessageParam]]:

876 + return conversations if is_list_of(conversations, list) else [conversations] # type: ignore[list-item,return-value]

877 +

878 + def _get_chat_tok_params(self, tokenization_kwargs: dict[str, Any] | None):

879 + model_config = self.model_config

880 + encoder_config = model_config.encoder_config or {}

881 +

882 + return TokenizeParams(

883 + max_total_tokens=model_config.max_model_len,

884 + do_lower_case=encoder_config.get("do_lower_case", False),

885 + add_special_tokens=False,

886 + ).with_kwargs(tokenization_kwargs)

887 +

888 + def _preprocess_chat(

889 + self,

890 + conversations: list[ChatCompletionMessageParam]

891 | list[list[ChatCompletionMessageParam]],

892 chat_template: str | None = None,

893 chat_template_content_format: ChatTemplateContentFormatOption = "auto",

894 + chat_template_kwargs: dict[str, Any] | None = None,

895 add_generation_prompt: bool = True,

896 continue_final_message: bool = False,

897 tools: list[dict[str, Any]] | None = None,

783 - chat_template_kwargs: dict[str, Any] | None = None,

898 + tokenization_kwargs: dict[str, Any] | None = None,

899 mm_processor_kwargs: dict[str, Any] | None = None,

785 - ) -> list[TextPrompt | TokensPrompt]:

900 + ) -> list[EnginePrompt]:

901 """

787 - Generate prompt for a chat conversation. The pre-processed

788 - prompt can then be used as input for the other LLM methods.

902 + Convert a list of conversations into prompts so that they can then

903 + be used as input for other LLM APIs.

904 +

905 + Refer to [LLM.chat][] for a complete description of the arguments.

906

790 - Refer to `chat` for a complete description of the arguments.

907 Returns:

792 - A list of `TokensPrompts` objects containing the tokenized

793 - prompt after chat template interpolation, and the

794 - pre-processed multi-modal inputs.

908 + A list of `TokensPrompts` objects containing the tokenized prompt

909 + after chat template interpolation, and the raw multi-modal inputs.

910 """

796 - list_of_messages: list[list[ChatCompletionMessageParam]]

797 -

798 - # Handle multi and single conversations

799 - if is_list_of(messages, list):

800 - # messages is list[list[...]]

801 - list_of_messages = cast(list[list[ChatCompletionMessageParam]], messages)

802 - else:

803 - # messages is list[...]

804 - list_of_messages = [cast(list[ChatCompletionMessageParam], messages)]

805 -

911 renderer = self.llm_engine.renderer

912

808 - chat_template_kwargs = {

809 - "chat_template": chat_template,

810 - "add_generation_prompt": add_generation_prompt,

811 - "continue_final_message": continue_final_message,

812 - "tools": tools,

813 - **(chat_template_kwargs or {}),

814 - }

815 -

816 - prompts = list[TextPrompt | TokensPrompt]()

817 -

818 - for msgs in list_of_messages:

819 - # NOTE: renderer.render_messages() currently doesn't

820 - # handle mm_processor_kwargs, since there is no implementation in

821 - # the chat message parsing for it.

822 - _, prompt = renderer.render_messages(

823 - msgs,

824 - chat_template_content_format=chat_template_content_format,

825 - **chat_template_kwargs,

826 - )

913 + chat_params = ChatParams(

914 + chat_template=chat_template,

915 + chat_template_content_format=chat_template_content_format,

916 + chat_template_kwargs=merge_kwargs(

917 + chat_template_kwargs,

918 + dict(

919 + add_generation_prompt=add_generation_prompt,

920 + continue_final_message=continue_final_message,

921 + tools=tools,

922 + tokenize=isinstance(renderer.tokenizer, MistralTokenizer),

923 + ),

924 + ),

925 + )

926 + tok_params = self._get_chat_tok_params(tokenization_kwargs)

927 +

928 + engine_prompts = list[EnginePrompt]()

929 + for conversation in self._normalize_conversations(conversations):

930 + _, in_prompt = renderer.render_messages(conversation, chat_params)

931 if mm_processor_kwargs is not None:

828 - prompt["mm_processor_kwargs"] = mm_processor_kwargs

932 + in_prompt["mm_processor_kwargs"] = mm_processor_kwargs

933

830 - prompts.append(prompt)

934 + engine_prompts.append(renderer.tokenize_prompt(in_prompt, tok_params))

935

832 - return prompts

936 + return engine_prompts

937

938 def chat(

939 self,

@@ -844,6 +948,7 @@ class LLM:

948 continue_final_message: bool = False,

949 tools: list[dict[str, Any]] | None = None,

950 chat_template_kwargs: dict[str, Any] | None = None,

951 + tokenization_kwargs: dict[str, Any] | None = None,

952 mm_processor_kwargs: dict[str, Any] | None = None,

953 ) -> list[RequestOutput]:

954 """

@@ -889,22 +994,22 @@ class LLM:

994 `True` if `add_generation_prompt` is also `True`.

995 chat_template_kwargs: Additional kwargs to pass to the chat

996 template.

892 - mm_processor_kwargs: Multimodal processor kwarg overrides for this

893 - chat request. Only used for offline requests.

997 + tokenization_kwargs: Overrides for `tokenizer.encode`.

998 + mm_processor_kwargs: Overrides for `processor.__call__`.

999

1000 Returns:

1001 A list of `RequestOutput` objects containing the generated

1002 responses in the same order as the input messages.

1003 """

899 -

900 - prompts = self.preprocess_chat(

901 - messages=messages,

1004 + prompts = self._preprocess_chat(

1005 + messages,

1006 chat_template=chat_template,

1007 chat_template_content_format=chat_template_content_format,

1008 + chat_template_kwargs=chat_template_kwargs,

1009 add_generation_prompt=add_generation_prompt,

1010 continue_final_message=continue_final_message,

1011 tools=tools,

907 - chat_template_kwargs=chat_template_kwargs,

1012 + tokenization_kwargs=tokenization_kwargs,

1013 mm_processor_kwargs=mm_processor_kwargs,

1014 )

1015

@@ -913,6 +1018,7 @@ class LLM:

1018 sampling_params=sampling_params,

1019 use_tqdm=use_tqdm,

1020 lora_request=lora_request,

1021 + tokenization_kwargs=tokenization_kwargs,

1022 )

1023

1024 def encode(

@@ -945,37 +1051,29 @@ class LLM:

1051 If `False`, no progress bar is created.

1052 lora_request: LoRA request to use for generation, if any.

1053 pooling_task: Override the pooling task to use.

948 - tokenization_kwargs: overrides tokenization_kwargs set in

949 - pooling_params

1054 + tokenization_kwargs: Overrides for `tokenizer.encode`.

1055

1056 Returns:

1057 A list of `PoolingRequestOutput` objects containing the

1058 pooled hidden states in the same order as the input prompts.

954 -

955 - Note:

956 - Using `prompts` and `prompt_token_ids` as keyword parameters is

957 - considered legacy and may be deprecated in the future. You should

958 - instead pass them via the `inputs` parameter.

1059 """

1060

961 - error_str = (

962 - "pooling_task required for `LLM.encode`\n"

963 - "Please use one of the more specific methods or set the "

964 - "pooling_task when using `LLM.encode`:\n"

965 - " - For embeddings, use `LLM.embed(...)` "

966 - 'or `pooling_task="embed"`.\n'

967 - " - For classification logits, use `LLM.classify(...)` "

968 - 'or `pooling_task="classify"`.\n'

969 - " - For similarity scores, use `LLM.score(...)`.\n"

970 - " - For rewards, use `LLM.reward(...)` "

971 - 'or `pooling_task="token_classify"`\n'

972 - " - For token classification, "

973 - 'use `pooling_task="token_classify"`\n'

974 - ' - For multi-vector retrieval, use `pooling_task="token_embed"`'

975 - )

976 -

1061 if pooling_task is None:

978 - raise ValueError(error_str)

1062 + raise ValueError(

1063 + "pooling_task required for `LLM.encode`\n"

1064 + "Please use one of the more specific methods or set the "

1065 + "pooling_task when using `LLM.encode`:\n"

1066 + " - For embeddings, use `LLM.embed(...)` "

1067 + 'or `pooling_task="embed"`.\n'

1068 + " - For classification logits, use `LLM.classify(...)` "

1069 + 'or `pooling_task="classify"`.\n'

1070 + " - For similarity scores, use `LLM.score(...)`.\n"

1071 + " - For rewards, use `LLM.reward(...)` "

1072 + 'or `pooling_task="token_classify"`\n'

1073 + " - For token classification, "

1074 + 'use `pooling_task="token_classify"`\n'

1075 + ' - For multi-vector retrieval, use `pooling_task="token_embed"`'

1076 + )

1077

1078 model_config = self.model_config

1079 runner_type = model_config.runner_type

@@ -986,6 +1084,20 @@ class LLM:

1084 "pooling model."

1085 )

1086

1087 + if truncate_prompt_tokens is not None:

1088 + warnings.warn(

1089 + "The `truncate_prompt_tokens` parameter in `LLM.encode()` "

1090 + "is deprecated and will be removed in v0.16. "

1091 + "Please pass it via `tokenization_kwargs` instead.",

1092 + DeprecationWarning,

1093 + stacklevel=2,

1094 + )

1095 +

1096 + tokenization_kwargs = merge_kwargs(

1097 + tokenization_kwargs,

1098 + dict(truncate_prompt_tokens=truncate_prompt_tokens),

1099 + )

1100 +

1101 io_processor_prompt = False

1102 if isinstance(prompts, dict) and "data" in prompts:

1103 io_processor_prompt = True

@@ -1017,19 +1129,16 @@ class LLM:

1129 pooling_params = self.io_processor.validate_or_generate_params(

1130 pooling_params

1131 )

1020 - else:

1021 - if pooling_params is None:

1022 - # Use default pooling params.

1023 - pooling_params = PoolingParams()

1132 +

1133 + if pooling_params is None:

1134 + # Use default pooling params.

1135 + pooling_params = PoolingParams()

1136

1137 if pooling_task not in self.supported_tasks:

1138 raise ValueError(f"pooling_task must be one of {self.supported_tasks}.")

1139

1140 for param in as_iter(pooling_params):

1141 param.verify(pooling_task, model_config)

1030 - # for backwards compatibility

1031 - if truncate_prompt_tokens is not None:

1032 - param.truncate_prompt_tokens = truncate_prompt_tokens

1142

1143 self._validate_and_add_requests(

1144 prompts=prompts,

@@ -1094,6 +1203,7 @@ class LLM:

1203 it is used to create the progress bar.

1204 If `False`, no progress bar is created.

1205 lora_request: LoRA request to use for generation, if any.

1206 + tokenization_kwargs: Overrides for `tokenizer.encode`.

1207

1208 Returns:

1209 A list of `EmbeddingRequestOutput` objects containing the

@@ -1105,9 +1215,14 @@ class LLM:

1215 "Try converting the model using `--convert embed`."

1216 )

1217

1218 + if truncate_prompt_tokens is not None:

1219 + tokenization_kwargs = merge_kwargs(

1220 + tokenization_kwargs,

1221 + dict(truncate_prompt_tokens=truncate_prompt_tokens),

1222 + )

1223 +

1224 items = self.encode(

1225 prompts,

1110 - truncate_prompt_tokens=truncate_prompt_tokens,

1226 use_tqdm=use_tqdm,

1227 pooling_params=pooling_params,

1228 lora_request=lora_request,

@@ -1121,8 +1236,8 @@ class LLM:

1236 self,

1237 prompts: PromptType | Sequence[PromptType],

1238 *,

1124 - use_tqdm: bool | Callable[..., tqdm] = True,

1239 pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,

1240 + use_tqdm: bool | Callable[..., tqdm] = True,

1241 lora_request: list[LoRARequest] | LoRARequest | None = None,

1242 tokenization_kwargs: dict[str, Any] | None = None,

1243 ) -> list[ClassificationRequestOutput]:

@@ -1137,13 +1252,15 @@ class LLM:

1252 prompts: The prompts to the LLM. You may pass a sequence of prompts

1253 for batch inference. See [PromptType][vllm.inputs.PromptType]

1254 for more details about the format of each prompt.

1255 + pooling_params: The pooling parameters for pooling. If None, we

1256 + use the default pooling parameters.

1257 use_tqdm: If `True`, shows a tqdm progress bar.

1258 If a callable (e.g., `functools.partial(tqdm, leave=False)`),

1259 it is used to create the progress bar.

1260 If `False`, no progress bar is created.

1261 lora_request: LoRA request to use for generation, if any.

1145 - pooling_params: The pooling parameters for pooling. If None, we

1146 - use the default pooling parameters.

1262 + tokenization_kwargs: Overrides for `tokenizer.encode`.

1263 +

1264 Returns:

1265 A list of `ClassificationRequestOutput` objects containing the

1266 embedding vectors in the same order as the input prompts.

@@ -1170,9 +1287,9 @@ class LLM:

1287 prompts: PromptType | Sequence[PromptType],

1288 /,

1289 *,

1290 + pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,

1291 truncate_prompt_tokens: int | None = None,

1292 use_tqdm: bool | Callable[..., tqdm] = True,

1175 - pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,

1293 lora_request: list[LoRARequest] | LoRARequest | None = None,

1294 tokenization_kwargs: dict[str, Any] | None = None,

1295 ) -> list[PoolingRequestOutput]:

@@ -1183,13 +1300,15 @@ class LLM:

1300 prompts: The prompts to the LLM. You may pass a sequence of prompts

1301 for batch inference. See [PromptType][vllm.inputs.PromptType]

1302 for more details about the format of each prompt.

1303 + pooling_params: The pooling parameters for pooling. If None, we

1304 + use the default pooling parameters.

1305 use_tqdm: If `True`, shows a tqdm progress bar.

1306 If a callable (e.g., `functools.partial(tqdm, leave=False)`),

1307 it is used to create the progress bar.

1308 If `False`, no progress bar is created.

1309 lora_request: LoRA request to use for generation, if any.

1191 - pooling_params: The pooling parameters for pooling. If None, we

1192 - use the default pooling parameters.

1310 + tokenization_kwargs: Overrides for `tokenizer.encode`.

1311 +

1312 Returns:

1313 A list of `PoolingRequestOutput` objects containing the

1314 pooled hidden states in the same order as the input prompts.

@@ -1207,18 +1326,18 @@ class LLM:

1326

1327 def _embedding_score(

1328 self,

1210 - tokenizer: TokenizerLike,

1211 - text_1: list[str | TextPrompt | TokensPrompt],

1212 - text_2: list[str | TextPrompt | TokensPrompt],

1213 - truncate_prompt_tokens: int | None = None,

1214 - use_tqdm: bool | Callable[..., tqdm] = True,

1215 - pooling_params: PoolingParams | None = None,

1216 - lora_request: list[LoRARequest] | LoRARequest | None = None,

1217 - tokenization_kwargs: dict[str, Any] | None = None,

1329 + text_1: list[SingletonPrompt],

1330 + text_2: list[SingletonPrompt],

1331 + *,

1332 + use_tqdm: bool | Callable[..., tqdm],

1333 + pooling_params: PoolingParams | None,

1334 + lora_request: list[LoRARequest] | LoRARequest | None,

1335 + tokenization_kwargs: dict[str, Any],

1336 ) -> list[ScoringRequestOutput]:

1219 - encoded_output: list[PoolingRequestOutput] = self.encode(

1337 + tokenizer = self.get_tokenizer()

1338 +

1339 + encoded_output = self.encode(

1340 text_1 + text_2,

1221 - truncate_prompt_tokens=truncate_prompt_tokens,

1341 use_tqdm=use_tqdm,

1342 lora_request=lora_request,

1343 pooling_params=pooling_params,

@@ -1226,14 +1345,16 @@ class LLM:

1345 tokenization_kwargs=tokenization_kwargs,

1346 )

1347

1229 - encoded_output_1: list[PoolingRequestOutput] = encoded_output[0 : len(text_1)]

1230 - encoded_output_2: list[PoolingRequestOutput] = encoded_output[len(text_1) :]

1348 + encoded_output_1 = encoded_output[0 : len(text_1)]

1349 + encoded_output_2 = encoded_output[len(text_1) :]

1350

1351 if len(encoded_output_1) == 1:

1352 encoded_output_1 = encoded_output_1 * len(encoded_output_2)

1353

1354 scores = _cosine_similarity(

1236 - tokenizer=tokenizer, embed_1=encoded_output_1, embed_2=encoded_output_2

1355 + tokenizer=tokenizer,

1356 + embed_1=encoded_output_1,

1357 + embed_2=encoded_output_2,

1358 )

1359

1360 items = self.engine_class.validate_outputs(scores, PoolingRequestOutput)

@@ -1241,17 +1362,17 @@ class LLM:

1362

1363 def _cross_encoding_score(

1364 self,

1244 - tokenizer: TokenizerLike,

1365 data_1: list[str] | list[ScoreContentPartParam],

1366 data_2: list[str] | list[ScoreContentPartParam],

1247 - truncate_prompt_tokens: int | None = None,

1248 - use_tqdm: bool | Callable[..., tqdm] = True,

1249 - pooling_params: PoolingParams | None = None,

1250 - lora_request: list[LoRARequest] | LoRARequest | None = None,

1251 - tokenization_kwargs: dict[str, Any] | None = None,

1252 - score_template: str | None = None,

1367 + *,

1368 + use_tqdm: bool | Callable[..., tqdm],

1369 + pooling_params: PoolingParams | None,

1370 + lora_request: list[LoRARequest] | LoRARequest | None,

1371 + tokenization_kwargs: dict[str, Any],

1372 + score_template: str | None,

1373 ) -> list[ScoringRequestOutput]:

1374 model_config = self.model_config

1375 + tokenizer = self.get_tokenizer()

1376

1377 if isinstance(tokenizer, MistralTokenizer):

1378 raise ValueError("Score API is not supported for Mistral tokenizer")

@@ -1265,13 +1386,6 @@ class LLM:

1386 pooling_params.verify("score", model_config)

1387 pooling_params_list = list[PoolingParams]()

1388

1268 - local_kwargs = tokenization_kwargs or {}

1269 - tokenization_kwargs = local_kwargs.copy()

1270 -

1271 - _validate_truncation_size(

1272 - model_config.max_model_len, truncate_prompt_tokens, tokenization_kwargs

1273 - )

1274 -

1389 prompts = list[PromptType]()

1390

1391 input_pairs = [(t1, t2) for t1, t2 in zip(data_1, data_2)]

@@ -1314,10 +1428,10 @@ class LLM:

1428 data_2: SingletonPrompt | Sequence[SingletonPrompt] | ScoreMultiModalParam,

1429 /,

1430 *,

1317 - truncate_prompt_tokens: int | None = None,

1431 use_tqdm: bool | Callable[..., tqdm] = True,

1432 pooling_params: PoolingParams | None = None,

1433 lora_request: list[LoRARequest] | LoRARequest | None = None,

1434 + tokenization_kwargs: dict[str, Any] | None = None,

1435 chat_template: str | None = None,

1436 ) -> list[ScoringRequestOutput]:

1437 """Generate similarity scores for all pairs `<text,text_pair>` or

@@ -1344,20 +1458,22 @@ class LLM:

1458 the LLM. Can be text or multi-modal data. See [PromptType]

1459 [vllm.inputs.PromptType] for more details about the format of

1460 each prompt.

1461 + pooling_params: The pooling parameters for pooling. If None, we

1462 + use the default pooling parameters.

1463 use_tqdm: If `True`, shows a tqdm progress bar.

1464 If a callable (e.g., `functools.partial(tqdm, leave=False)`),

1465 it is used to create the progress bar.

1466 If `False`, no progress bar is created.

1467 lora_request: LoRA request to use for generation, if any.

1352 - pooling_params: The pooling parameters for pooling. If None, we

1353 - use the default pooling parameters.

1468 chat_template: The chat template to use for the scoring. If None, we

1469 use the model's default chat template.

1470 + tokenization_kwargs: Overrides for `tokenizer.encode`.

1471 Returns:

1472 A list of `ScoringRequestOutput` objects containing the

1473 generated scores in the same order as the input prompts.

1474 """

1475 model_config = self.model_config

1476 +

1477 runner_type = model_config.runner_type

1478 if runner_type != "pooling":

1479 raise ValueError(

@@ -1445,26 +1561,27 @@ class LLM:

1561

1562 _validate_score_input_lens(data_1, data_2) # type: ignore[arg-type]

1563

1564 + tok_params = self._get_cmpl_tok_params(tokenization_kwargs)

1565 + encode_kwargs = tok_params.get_encode_kwargs()

1566 +

1567 if model_config.is_cross_encoder:

1568 return self._cross_encoding_score(

1450 - tokenizer,

1569 data_1, # type: ignore[arg-type]

1570 data_2, # type: ignore[arg-type]

1453 - truncate_prompt_tokens,

1454 - use_tqdm,

1455 - pooling_params,

1456 - lora_request,

1571 + use_tqdm=use_tqdm,

1572 + pooling_params=pooling_params,

1573 + lora_request=lora_request,

1574 + tokenization_kwargs=encode_kwargs,

1575 score_template=chat_template,

1576 )

1577 else:

1578 return self._embedding_score(

1461 - tokenizer,

1579 data_1, # type: ignore[arg-type]

1580 data_2, # type: ignore[arg-type]

1464 - truncate_prompt_tokens,

1465 - use_tqdm,

1466 - pooling_params,

1467 - lora_request,

1581 + use_tqdm=use_tqdm,

1582 + pooling_params=pooling_params,

1583 + lora_request=lora_request,

1584 + tokenization_kwargs=encode_kwargs,

1585 )

1586

1587 def start_profile(self) -> None:

@@ -1530,42 +1647,79 @@ class LLM:

1647

1648 def _validate_and_add_requests(

1649 self,

1533 - prompts: PromptType | Sequence[PromptType] | DataPrompt,

1650 + prompts: PromptType | Sequence[PromptType],

1651 params: SamplingParams

1652 | Sequence[SamplingParams]

1653 | PoolingParams

1654 | Sequence[PoolingParams],

1655 *,

1656 use_tqdm: bool | Callable[..., tqdm] = True,

1540 - lora_request: Sequence[LoRARequest] | LoRARequest | None,

1541 - priority: list[int] | None = None,

1657 + lora_request: Sequence[LoRARequest | None] | LoRARequest | None,

1658 tokenization_kwargs: dict[str, Any] | None = None,

1659 + priority: list[int] | None = None,

1660 ) -> None:

1544 - if isinstance(prompts, (str, dict)):

1545 - # Convert a single prompt to a list.

1546 - prompts = [prompts] # type: ignore[list-item]

1547 -

1548 - num_requests = len(prompts)

1549 - if isinstance(params, Sequence) and len(params) != num_requests:

1550 - raise ValueError("The lengths of prompts and params must be the same.")

1551 - if isinstance(lora_request, Sequence) and len(lora_request) != num_requests:

1552 - raise ValueError(

1553 - "The lengths of prompts and lora_request must be the same."

1554 - )

1555 - if priority is not None and len(priority) != num_requests:

1556 - raise ValueError(

1557 - "The lengths of prompts "

1558 - f"({num_requests}) and priority ({len(priority)}) "

1559 - "must be the same."

1661 + in_prompts = self._normalize_prompts(prompts)

1662 + num_requests = len(in_prompts)

1663 +

1664 + if isinstance(params, Sequence):

1665 + if len(params) != num_requests:

1666 + raise ValueError(

1667 + f"The lengths of prompts ({params}) "

1668 + f"and lora_request ({len(params)}) must be the same."

1669 + )

1670 +

1671 + engine_params = params

1672 + else:

1673 + engine_params = [params] * num_requests

1674 +

1675 + if isinstance(lora_request, Sequence):

1676 + if len(lora_request) != num_requests:

1677 + raise ValueError(

1678 + f"The lengths of prompts ({num_requests}) "

1679 + f"and lora_request ({len(lora_request)}) must be the same."

1680 + )

1681 +

1682 + engine_lora_requests: Sequence[LoRARequest | None] = lora_request

1683 + else:

1684 + engine_lora_requests = [lora_request] * num_requests

1685 +

1686 + if priority is not None:

1687 + if len(priority) != num_requests:

1688 + raise ValueError(

1689 + f"The lengths of prompts ({num_requests}) "

1690 + f"and priority ({len(priority)}) must be the same."

1691 + )

1692 + else:

1693 + priority = [0] * num_requests

1694 +

1695 + if any(param.truncate_prompt_tokens is not None for param in engine_params):

1696 + # TODO: Remove this after deprecating `param.truncate_prompt_tokens`

1697 + # Then, move the code from the `else` block to the top and let

1698 + # `self._preprocess_completion` handle prompt normalization

1699 + engine_prompts = [

1700 + engine_prompt

1701 + for in_prompt, param in zip(in_prompts, engine_params)

1702 + for engine_prompt in self._preprocess_completion(

1703 + [in_prompt],

1704 + tokenization_kwargs=merge_kwargs(

1705 + tokenization_kwargs,

1706 + dict(truncate_prompt_tokens=param.truncate_prompt_tokens),

1707 + ),

1708 + )

1709 + ]

1710 + else:

1711 + engine_prompts = self._preprocess_completion(

1712 + in_prompts,

1713 + tokenization_kwargs=tokenization_kwargs,

1714 )

1715

1562 - for sp in params if isinstance(params, Sequence) else (params,):

1716 + for sp in engine_params:

1717 if isinstance(sp, SamplingParams):

1718 # We only care about the final output

1719 sp.output_kind = RequestOutputKind.FINAL_ONLY

1720

1721 # Add requests to the engine.

1568 - it = prompts

1722 + it = engine_prompts

1723 if use_tqdm:

1724 tqdm_func = use_tqdm if callable(use_tqdm) else tqdm

1725 it = tqdm_func(it, desc="Adding requests")

@@ -1576,12 +1730,10 @@ class LLM:

1730 for i, prompt in enumerate(it):

1731 request_id = self._add_request(

1732 prompt,

1579 - params[i] if isinstance(params, Sequence) else params,

1580 - lora_request=lora_request[i]

1581 - if isinstance(lora_request, Sequence)

1582 - else lora_request,

1583 - priority=priority[i] if priority else 0,

1733 + engine_params[i],

1734 + lora_request=engine_lora_requests[i],

1735 tokenization_kwargs=tokenization_kwargs,

1736 + priority=priority[i],

1737 )

1738 added_request_ids.append(request_id)

1739 except Exception as e:

@@ -1589,54 +1741,42 @@ class LLM:

1741 self.llm_engine.abort_request(added_request_ids, internal=True)

1742 raise e

1743

1592 - def _process_inputs(

1593 - self,

1594 - request_id: str,

1595 - engine_prompt: PromptType,

1596 - params: SamplingParams | PoolingParams,

1597 - *,

1598 - lora_request: LoRARequest | None,

1599 - priority: int,

1600 - tokenization_kwargs: dict[str, Any] | None = None,

1601 - ) -> tuple[EngineCoreRequest, dict[str, Any]]:

1602 - """Use the Processor to process inputs for LLMEngine."""

1603 -

1604 - local_kwargs = tokenization_kwargs or {}

1605 - tokenization_kwargs = local_kwargs.copy()

1606 - _validate_truncation_size(

1607 - self.model_config.max_model_len,

1608 - params.truncate_prompt_tokens,

1609 - tokenization_kwargs,

1610 - )

1611 -

1612 - engine_request = self.input_processor.process_inputs(

1613 - request_id,

1614 - engine_prompt,

1615 - params,

1616 - lora_request=lora_request,

1617 - tokenization_kwargs=tokenization_kwargs,

1618 - priority=priority,

1619 - )

1620 - return engine_request, tokenization_kwargs

1621 -

1744 def _add_request(

1745 self,

1746 prompt: PromptType,

1747 params: SamplingParams | PoolingParams,

1748 lora_request: LoRARequest | None = None,

1627 - priority: int = 0,

1749 tokenization_kwargs: dict[str, Any] | None = None,

1750 + priority: int = 0,

1751 ) -> str:

1752 prompt_text, _, _ = get_prompt_components(prompt)

1753 request_id = str(next(self.request_counter))

1754

1633 - engine_request, tokenization_kwargs = self._process_inputs(

1755 + if params.truncate_prompt_tokens is not None:

1756 + params_type = type(params).__name__

1757 + warnings.warn(

1758 + f"The `truncate_prompt_tokens` parameter in `{params_type}` "

1759 + "is deprecated and will be removed in v0.16. "

1760 + "Please pass it via `tokenization_kwargs` instead.",

1761 + DeprecationWarning,

1762 + stacklevel=2,

1763 + )

1764 +

1765 + tokenization_kwargs = merge_kwargs(

1766 + tokenization_kwargs,

1767 + dict(truncate_prompt_tokens=params.truncate_prompt_tokens),

1768 + )

1769 +

1770 + tok_params = self._get_cmpl_tok_params(tokenization_kwargs)

1771 +

1772 + tokenization_kwargs = tok_params.get_encode_kwargs()

1773 + engine_request = self.input_processor.process_inputs(

1774 request_id,

1775 prompt,

1776 params,

1777 lora_request=lora_request,

1638 - priority=priority,

1778 tokenization_kwargs=tokenization_kwargs,

1779 + priority=priority,

1780 )

1781

1782 self.llm_engine.add_request(

vllm/entrypoints/openai/chat_completion/protocol.py

+44 -5

@@ -13,12 +13,13 @@ from openai.types.chat.chat_completion_audio import (

13 ChatCompletionAudio as OpenAIChatCompletionAudio,

14 )

15 from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation

16 -from pydantic import (

17 - Field,

18 - model_validator,

19 -)

16 +from pydantic import Field, model_validator

21 -from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

18 +from vllm.config import ModelConfig

19 +from vllm.entrypoints.chat_utils import (

20 + ChatCompletionMessageParam,

21 + ChatTemplateContentFormatOption,

22 +)

23 from vllm.entrypoints.openai.engine.protocol import (

24 AnyResponseFormat,

25 DeltaMessage,

@@ -36,6 +37,7 @@ from vllm.entrypoints.openai.engine.protocol import (

37 from vllm.exceptions import VLLMValidationError

38 from vllm.logger import init_logger

39 from vllm.logprobs import Logprob

40 +from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs

41 from vllm.sampling_params import (

42 BeamSearchParams,

43 RequestOutputKind,

@@ -348,6 +350,43 @@ class ChatCompletionRequest(OpenAIBaseModel):

350

351 # --8<-- [end:chat-completion-extra-params]

352

353 + def build_chat_params(

354 + self,

355 + default_template: str | None,

356 + default_template_content_format: ChatTemplateContentFormatOption,

357 + ) -> ChatParams:

358 + return ChatParams(

359 + chat_template=self.chat_template or default_template,

360 + chat_template_content_format=default_template_content_format,

361 + chat_template_kwargs=merge_kwargs(

362 + self.chat_template_kwargs,

363 + dict(

364 + add_generation_prompt=self.add_generation_prompt,

365 + continue_final_message=self.continue_final_message,

366 + documents=self.documents,

367 + reasoning_effort=self.reasoning_effort,

368 + ),

369 + ),

370 + )

371 +

372 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

373 + if self.max_completion_tokens is not None:

374 + max_output_tokens: int | None = self.max_completion_tokens

375 + max_output_tokens_param = "max_completion_tokens"

376 + else:

377 + max_output_tokens = self.max_tokens

378 + max_output_tokens_param = "max_tokens"

379 +

380 + return TokenizeParams(

381 + max_total_tokens=model_config.max_model_len,

382 + max_output_tokens=max_output_tokens or 0,

383 + truncate_prompt_tokens=self.truncate_prompt_tokens,

384 + add_special_tokens=self.add_special_tokens,

385 + needs_detokenization=bool(self.echo and not self.return_token_ids),

386 + max_total_tokens_param="max_model_len",

387 + max_output_tokens_param=max_output_tokens_param,

388 + )

389 +

390 # Default sampling parameters for chat completion requests

391 _DEFAULT_SAMPLING_PARAMS: dict = {

392 "repetition_penalty": 1.0,

vllm/entrypoints/openai/chat_completion/serving.py

+16 -28

@@ -67,7 +67,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (

67 )

68 from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls

69 from vllm.entrypoints.utils import get_max_tokens, should_include_usage

70 -from vllm.inputs.data import TokensPrompt

70 +from vllm.inputs.data import EmbedsPrompt, TokensPrompt

71 from vllm.inputs.parse import get_prompt_components

72 from vllm.logger import init_logger

73 from vllm.logprobs import Logprob

@@ -185,8 +185,6 @@ class OpenAIServingChat(OpenAIServing):

185 start_time = time.perf_counter()

186

187 try:

188 - renderer = self.engine_client.renderer

189 -

188 # Create a minimal dummy request

189 dummy_request = ChatCompletionRequest(

190 messages=[{"role": "user", "content": "warmup"}],

@@ -201,18 +199,10 @@ class OpenAIServingChat(OpenAIServing):

199 # 3. Tokenizer initialization for chat

200 await self._preprocess_chat(

201 dummy_request,

204 - renderer,

202 dummy_request.messages,

206 - chat_template=self.chat_template,

207 - chat_template_content_format=self.chat_template_content_format,

208 - add_generation_prompt=True,

209 - continue_final_message=False,

210 - tool_dicts=None,

211 - documents=None,

212 - chat_template_kwargs=None,

213 - default_chat_template_kwargs=self.default_chat_template_kwargs,

214 - tool_parser=None,

215 - add_special_tokens=False,

203 + default_template=self.chat_template,

204 + default_template_content_format=self.chat_template_content_format,

205 + default_template_kwargs=self.default_chat_template_kwargs,

206 )

207

208 elapsed = (time.perf_counter() - start_time) * 1000

@@ -225,7 +215,10 @@ class OpenAIServingChat(OpenAIServing):

215 async def render_chat_request(

216 self,

217 request: ChatCompletionRequest,

228 - ) -> tuple[list[ConversationMessage], list[Any]] | ErrorResponse:

218 + ) -> (

219 + tuple[list[ConversationMessage], list[TokensPrompt | EmbedsPrompt]]

220 + | ErrorResponse

221 + ):

222 """

223 render chat request by validating and preprocessing inputs.

224

@@ -302,23 +295,14 @@ class OpenAIServingChat(OpenAIServing):

295 if error_check_ret is not None:

296 return error_check_ret

297

305 - chat_template_kwargs = request.chat_template_kwargs or {}

306 - chat_template_kwargs.update(reasoning_effort=request.reasoning_effort)

307 -

298 conversation, engine_prompts = await self._preprocess_chat(

299 request,

310 - renderer,

300 request.messages,

312 - chat_template=request.chat_template or self.chat_template,

313 - chat_template_content_format=self.chat_template_content_format,

314 - add_generation_prompt=request.add_generation_prompt,

315 - continue_final_message=request.continue_final_message,

301 + default_template=self.chat_template,

302 + default_template_content_format=self.chat_template_content_format,

303 + default_template_kwargs=self.default_chat_template_kwargs,

304 tool_dicts=tool_dicts,

317 - documents=request.documents,

318 - chat_template_kwargs=chat_template_kwargs,

319 - default_chat_template_kwargs=self.default_chat_template_kwargs,

305 tool_parser=tool_parser,

321 - add_special_tokens=request.add_special_tokens,

306 )

307 else:

308 # For GPT-OSS.

@@ -428,11 +412,15 @@ class OpenAIServingChat(OpenAIServing):

412 trace_headers=trace_headers,

413 )

414 else:

431 - engine_request, tokenization_kwargs = await self._process_inputs(

415 + tok_params = request.build_tok_params(self.model_config)

416 + tokenization_kwargs = tok_params.get_encode_kwargs()

417 +

418 + engine_request = self.input_processor.process_inputs(

419 sub_request_id,

420 engine_prompt,

421 sampling_params,

422 lora_request=lora_request,

423 + tokenization_kwargs=tokenization_kwargs,

424 trace_headers=trace_headers,

425 priority=request.priority,

426 data_parallel_rank=data_parallel_rank,

vllm/entrypoints/openai/completion/protocol.py

+14 -4

@@ -9,11 +9,9 @@ from dataclasses import replace

9 from typing import Annotated, Any, Literal

11 import torch

12 -from pydantic import (

13 - Field,

14 - model_validator,

15 -)

12 +from pydantic import Field, model_validator

14 +from vllm.config import ModelConfig

15 from vllm.entrypoints.openai.engine.protocol import (

16 AnyResponseFormat,

17 LegacyStructuralTagResponseFormat,

@@ -27,6 +25,7 @@ from vllm.entrypoints.openai.engine.protocol import (

25 from vllm.exceptions import VLLMValidationError

26 from vllm.logger import init_logger

27 from vllm.logprobs import Logprob

28 +from vllm.renderers import TokenizeParams

29 from vllm.sampling_params import (

30 BeamSearchParams,

31 RequestOutputKind,

@@ -178,6 +177,17 @@ class CompletionRequest(OpenAIBaseModel):

177

178 # --8<-- [end:completion-extra-params]

179

180 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

181 + return TokenizeParams(

182 + max_total_tokens=model_config.max_model_len,

183 + max_output_tokens=self.max_tokens or 0,

184 + truncate_prompt_tokens=self.truncate_prompt_tokens,

185 + add_special_tokens=self.add_special_tokens,

186 + needs_detokenization=bool(self.echo and not self.return_token_ids),

187 + max_total_tokens_param="max_model_len",

188 + max_output_tokens_param="max_tokens",

189 + )

190 +

191 # Default sampling parameters for completion requests

192 _DEFAULT_SAMPLING_PARAMS: dict = {

193 "repetition_penalty": 1.0,

vllm/entrypoints/openai/completion/serving.py

+8 -33

@@ -32,7 +32,6 @@ from vllm.entrypoints.openai.engine.serving import (

32 clamp_prompt_logprobs,

33 )

34 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

35 -from vllm.entrypoints.renderer import RenderConfig

35 from vllm.entrypoints.utils import get_max_tokens, should_include_usage

36 from vllm.exceptions import VLLMValidationError

37 from vllm.inputs.data import EmbedsPrompt, TokensPrompt, is_embeds_prompt

@@ -111,11 +110,10 @@ class OpenAIServingCompletion(OpenAIServing):

110 )

111

112 try:

114 - renderer = self._get_completion_renderer()

115 - engine_prompts = await renderer.render_prompt_and_embeds(

116 - prompt_or_prompts=request.prompt,

113 + engine_prompts = await self._preprocess_completion(

114 + request,

115 + prompt_input=request.prompt,

116 prompt_embeds=request.prompt_embeds,

118 - config=self._build_render_config(request),

117 )

118 except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:

119 logger.exception("Error in preprocessing prompt inputs")

@@ -203,10 +201,6 @@ class OpenAIServingCompletion(OpenAIServing):

201 else await self._get_trace_headers(raw_request.headers)

202 )

203

206 - # Mypy inconsistently requires this second cast in different

207 - # environments. It shouldn't be necessary (redundant from above)

208 - # but pre-commit in CI fails without it.

209 - engine_prompt = cast(EmbedsPrompt | TokensPrompt, engine_prompt)

204 if isinstance(sampling_params, BeamSearchParams):

205 generator = self.beam_search(

206 prompt=engine_prompt,

@@ -216,11 +210,15 @@ class OpenAIServingCompletion(OpenAIServing):

210 trace_headers=trace_headers,

211 )

212 else:

219 - engine_request, tokenization_kwargs = await self._process_inputs(

213 + tok_params = request.build_tok_params(self.model_config)

214 + tokenization_kwargs = tok_params.get_encode_kwargs()

215 +

216 + engine_request = self.input_processor.process_inputs(

217 request_id_item,

218 engine_prompt,

219 sampling_params,

220 lora_request=lora_request,

221 + tokenization_kwargs=tokenization_kwargs,

222 trace_headers=trace_headers,

223 priority=request.priority,

224 data_parallel_rank=data_parallel_rank,

@@ -709,26 +707,3 @@ class OpenAIServingCompletion(OpenAIServing):

707 tokens=out_tokens,

708 top_logprobs=out_top_logprobs,

709 )

712 -

713 - def _build_render_config(

714 - self,

715 - request: CompletionRequest,

716 - max_input_length: int | None = None,

717 - ) -> RenderConfig:

718 - # Validate max_tokens before using it

719 - if request.max_tokens is not None and request.max_tokens > self.max_model_len:

720 - raise VLLMValidationError(

721 - f"'max_tokens' ({request.max_tokens}) cannot be greater than "

722 - f"the model's maximum context length ({self.max_model_len}).",

723 - parameter="max_tokens",

724 - value=request.max_tokens,

725 - )

726 -

727 - max_input_tokens_len = self.max_model_len - (request.max_tokens or 0)

728 - return RenderConfig(

729 - max_length=max_input_tokens_len,

730 - truncate_prompt_tokens=request.truncate_prompt_tokens,

731 - add_special_tokens=request.add_special_tokens,

732 - cache_salt=request.cache_salt,

733 - needs_detokenization=bool(request.echo and not request.return_token_ids),

734 - )

vllm/entrypoints/openai/engine/protocol.py

+1 -3

@@ -16,9 +16,7 @@ from pydantic import (

17 from vllm.entrypoints.chat_utils import make_tool_call_id

18 from vllm.logger import init_logger

19 -from vllm.sampling_params import (

20 - SamplingParams,

21 -)

19 +from vllm.sampling_params import SamplingParams

20 from vllm.utils import random_uuid

21 from vllm.utils.import_utils import resolve_obj_by_qualname

vllm/entrypoints/openai/engine/serving.py

+99 -267

@@ -5,10 +5,10 @@ import json

5 import sys

6 import time

7 import traceback

8 -from collections.abc import AsyncGenerator, Callable, Iterable, Mapping

8 +from collections.abc import AsyncGenerator, Callable, Mapping

9 from dataclasses import dataclass, field

10 from http import HTTPStatus

11 -from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast

11 +from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar

13 import numpy as np

14 from fastapi import Request

@@ -20,6 +20,7 @@ from starlette.datastructures import Headers

21 import vllm.envs as envs

22 from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function

23 +from vllm.config import ModelConfig

24 from vllm.engine.protocol import EngineClient

25 from vllm.entrypoints.chat_utils import (

26 ChatCompletionMessageParam,

@@ -86,7 +87,6 @@ from vllm.entrypoints.pooling.score.protocol import (

87 ScoreResponse,

88 ScoreTextRequest,

89 )

89 -from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig

90 from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse

91 from vllm.entrypoints.serve.tokenize.protocol import (

92 DetokenizeRequest,

@@ -94,13 +94,9 @@ from vllm.entrypoints.serve.tokenize.protocol import (

94 TokenizeCompletionRequest,

95 TokenizeResponse,

96 )

97 -from vllm.entrypoints.utils import (

98 - _validate_truncation_size,

99 - get_max_tokens,

100 - sanitize_message,

101 -)

97 +from vllm.entrypoints.utils import get_max_tokens, sanitize_message

98 from vllm.exceptions import VLLMValidationError

103 -from vllm.inputs.data import PromptType, TokensPrompt

99 +from vllm.inputs.data import EmbedsPrompt, PromptType, TokensPrompt

100 from vllm.inputs.parse import (

101 get_prompt_components,

102 is_explicit_encoder_decoder_prompt,

@@ -112,7 +108,7 @@ from vllm.multimodal import MultiModalDataDict

108 from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput

109 from vllm.pooling_params import PoolingParams

110 from vllm.reasoning import ReasoningParser, ReasoningParserManager

115 -from vllm.renderers import RendererLike

111 +from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs

112 from vllm.sampling_params import BeamSearchParams, SamplingParams

113 from vllm.tokenizers import TokenizerLike

114 from vllm.tool_parsers import ToolParser, ToolParserManager

@@ -123,11 +119,9 @@ from vllm.tracing import (

119 )

120 from vllm.utils import random_uuid

121 from vllm.utils.async_utils import (

126 - AsyncMicrobatchTokenizer,

122 collect_from_async_generator,

123 merge_async_iterators,

124 )

130 -from vllm.v1.engine import EngineCoreRequest

125

126

127 class GenerationError(Exception):

@@ -140,6 +134,21 @@ class GenerationError(Exception):

134

135 logger = init_logger(__name__)

136

137 +

138 +class RendererRequest(Protocol):

139 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

140 + raise NotImplementedError

141 +

142 +

143 +class RendererChatRequest(RendererRequest, Protocol):

144 + def build_chat_params(

145 + self,

146 + default_template: str | None,

147 + default_template_content_format: ChatTemplateContentFormatOption,

148 + ) -> ChatParams:

149 + raise NotImplementedError

150 +

151 +

152 CompletionLikeRequest: TypeAlias = (

153 CompletionRequest

154 | TokenizeCompletionRequest

@@ -158,7 +167,9 @@ ChatLikeRequest: TypeAlias = (

167 | ClassificationChatRequest

168 | PoolingChatRequest

169 )

170 +

171 SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest

172 +

173 AnyRequest: TypeAlias = (

174 CompletionLikeRequest

175 | ChatLikeRequest

@@ -193,7 +204,7 @@ class ServeContext(Generic[RequestT]):

204 request_id: str

205 created_time: int = field(default_factory=lambda: int(time.time()))

206 lora_request: LoRARequest | None = None

196 - engine_prompts: list[TokensPrompt] | None = None

207 + engine_prompts: list[TokensPrompt | EmbedsPrompt] | None = None

208

209 result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (

210 None

@@ -227,7 +238,6 @@ class OpenAIServing:

238 self.request_logger = request_logger

239 self.return_tokens_as_token_ids = return_tokens_as_token_ids

240

230 - self._async_tokenizer_pool: dict[TokenizerLike, AsyncMicrobatchTokenizer] = {}

241 self.log_error_stack = log_error_stack

242

243 self.input_processor = self.models.input_processor

@@ -519,41 +529,6 @@ class OpenAIServing:

529 prompt_logprobs=None,

530 )

531

522 - def _get_completion_renderer(self) -> BaseRenderer:

523 - """

524 - Get a Renderer instance with the provided tokenizer.

525 - Uses shared async tokenizer pool for efficiency.

526 - """

527 - return CompletionRenderer(

528 - model_config=self.model_config,

529 - tokenizer=self.renderer.tokenizer,

530 - async_tokenizer_pool=self._async_tokenizer_pool,

531 - )

532 -

533 - def _build_render_config(

534 - self,

535 - request: Any,

536 - ) -> RenderConfig:

537 - """

538 - Build and return a `RenderConfig` for an endpoint.

539 -

540 - Used by the renderer to control how prompts are prepared

541 - (e.g., tokenization and length handling). Endpoints should

542 - implement this with logic appropriate to their request type.

543 - """

544 - raise NotImplementedError

545 -

546 - def _get_async_tokenizer(self, tokenizer) -> AsyncMicrobatchTokenizer:

547 - """

548 - Return (and cache) an `AsyncMicrobatchTokenizer` bound to the

549 - given tokenizer.

550 - """

551 - async_tokenizer = self._async_tokenizer_pool.get(tokenizer)

552 - if async_tokenizer is None:

553 - async_tokenizer = AsyncMicrobatchTokenizer(tokenizer)

554 - self._async_tokenizer_pool[tokenizer] = async_tokenizer

555 - return async_tokenizer

556 -

532 async def _preprocess(

533 self,

534 ctx: ServeContext,

@@ -912,71 +887,6 @@ class OpenAIServing:

887 message_types.add(content_dict["type"].split("_")[0])

888 return message_types

889

915 - async def _normalize_prompt_text_to_input(

916 - self,

917 - request: AnyRequest,

918 - prompt: str,

919 - tokenizer: TokenizerLike,

920 - add_special_tokens: bool,

921 - ) -> TokensPrompt:

922 - async_tokenizer = self._get_async_tokenizer(tokenizer)

923 -

924 - if (

925 - self.model_config.encoder_config is not None

926 - and self.model_config.encoder_config.get("do_lower_case", False)

927 - ):

928 - prompt = prompt.lower()

929 -

930 - truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)

931 -

932 - if truncate_prompt_tokens is None:

933 - encoded = await async_tokenizer(

934 - prompt, add_special_tokens=add_special_tokens

935 - )

936 - elif truncate_prompt_tokens < 0:

937 - # Negative means we cap at the model's max length

938 - encoded = await async_tokenizer(

939 - prompt,

940 - add_special_tokens=add_special_tokens,

941 - truncation=True,

942 - max_length=self.max_model_len,

943 - )

944 - else:

945 - encoded = await async_tokenizer(

946 - prompt,

947 - add_special_tokens=add_special_tokens,

948 - truncation=True,

949 - max_length=truncate_prompt_tokens,

950 - )

951 -

952 - input_ids = encoded.input_ids

953 - input_text = prompt

954 -

955 - return self._validate_input(request, input_ids, input_text)

956 -

957 - async def _normalize_prompt_tokens_to_input(

958 - self,

959 - request: AnyRequest,

960 - prompt_ids: list[int],

961 - tokenizer: TokenizerLike | None,

962 - ) -> TokensPrompt:

963 - truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)

964 -

965 - if truncate_prompt_tokens is None:

966 - input_ids = prompt_ids

967 - elif truncate_prompt_tokens < 0:

968 - input_ids = prompt_ids[-self.max_model_len :]

969 - else:

970 - input_ids = prompt_ids[-truncate_prompt_tokens:]

971 -

972 - if tokenizer is None:

973 - input_text = ""

974 - else:

975 - async_tokenizer = self._get_async_tokenizer(tokenizer)

976 - input_text = await async_tokenizer.decode(input_ids)

977 -

978 - return self._validate_input(request, input_ids, input_text)

979 -

890 def _validate_input(

891 self,

892 request: object,

@@ -1061,50 +971,6 @@ class OpenAIServing:

971

972 return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)

973

1064 - async def _tokenize_prompt_input_async(

1065 - self,

1066 - request: AnyRequest,

1067 - tokenizer: TokenizerLike,

1068 - prompt_input: str | list[int],

1069 - add_special_tokens: bool = True,

1070 - ) -> TokensPrompt:

1071 - """

1072 - A simpler implementation that tokenizes a single prompt input.

1073 - """

1074 - async for result in self._tokenize_prompt_inputs_async(

1075 - request,

1076 - tokenizer,

1077 - [prompt_input],

1078 - add_special_tokens=add_special_tokens,

1079 - ):

1080 - return result

1081 - raise ValueError("No results yielded from tokenization")

1082 -

1083 - async def _tokenize_prompt_inputs_async(

1084 - self,

1085 - request: AnyRequest,

1086 - tokenizer: TokenizerLike,

1087 - prompt_inputs: Iterable[str | list[int]],

1088 - add_special_tokens: bool = True,

1089 - ) -> AsyncGenerator[TokensPrompt, None]:

1090 - """

1091 - A simpler implementation that tokenizes multiple prompt inputs.

1092 - """

1093 - for prompt in prompt_inputs:

1094 - if isinstance(prompt, str):

1095 - yield await self._normalize_prompt_text_to_input(

1096 - request,

1097 - prompt=prompt,

1098 - tokenizer=tokenizer,

1099 - add_special_tokens=add_special_tokens,

1100 - )

1101 - else:

1102 - yield await self._normalize_prompt_tokens_to_input(

1103 - request,

1104 - prompt_ids=prompt,

1105 - tokenizer=tokenizer,

1106 - )

1107 -

974 def _validate_chat_template(

975 self,

976 request_chat_template: str | None,

@@ -1137,131 +1003,94 @@ class OpenAIServing:

1003 # Apply server defaults first, then request kwargs override.

1004 return default_chat_template_kwargs | request_chat_template_kwargs

1005

1006 + async def _preprocess_completion(

1007 + self,

1008 + request: RendererRequest,

1009 + prompt_input: str | list[str] | list[int] | list[list[int]] | None,

1010 + prompt_embeds: bytes | list[bytes] | None,

1011 + ) -> list[TokensPrompt | EmbedsPrompt]:

1012 + renderer = self.renderer

1013 + tok_params = request.build_tok_params(self.model_config)

1014 +

1015 + in_prompts = await renderer.render_completions_async(

1016 + prompt_input, prompt_embeds

1017 + )

1018 + engine_prompts = await renderer.tokenize_prompts_async(in_prompts, tok_params)

1019 +

1020 + extra_items = {

1021 + k: v

1022 + for k in ("mm_processor_kwargs", "cache_salt")

1023 + if (v := getattr(request, k, None)) is not None

1024 + }

1025 + for prompt in engine_prompts:

1026 + prompt.update(extra_items) # type: ignore

1027 +

1028 + return engine_prompts

1029 +

1030 async def _preprocess_chat(

1031 self,

1142 - request: ChatLikeRequest | ResponsesRequest,

1143 - renderer: RendererLike,

1032 + request: RendererChatRequest,

1033 messages: list[ChatCompletionMessageParam],

1145 - chat_template: str | None,

1146 - chat_template_content_format: ChatTemplateContentFormatOption,

1147 - add_generation_prompt: bool = True,

1148 - continue_final_message: bool = False,

1034 + default_template: str | None,

1035 + default_template_content_format: ChatTemplateContentFormatOption,

1036 + default_template_kwargs: dict[str, Any] | None,

1037 tool_dicts: list[dict[str, Any]] | None = None,

1150 - documents: list[dict[str, str]] | None = None,

1151 - chat_template_kwargs: dict[str, Any] | None = None,

1152 - default_chat_template_kwargs: dict[str, Any] | None = None,

1038 tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,

1154 - add_special_tokens: bool = False,

1155 - ) -> tuple[list[ConversationMessage], list[TokensPrompt]]:

1156 - chat_template_kwargs = {

1157 - "chat_template": chat_template,

1158 - "add_generation_prompt": add_generation_prompt,

1159 - "continue_final_message": continue_final_message,

1160 - "tools": tool_dicts,

1161 - "documents": documents,

1162 - **(chat_template_kwargs or {}),

1163 - }

1164 - chat_template_kwargs = self._prepare_extra_chat_template_kwargs(

1165 - chat_template_kwargs,

1166 - default_chat_template_kwargs,

1167 - )

1168 -

1169 - # Use the async tokenizer in `OpenAIServing` if possible.

1170 - # Later we can move it into the renderer so that we can return both

1171 - # text and token IDs in the same prompt from `render_messages_async`

1172 - # which is used for logging and `enable_response_messages`.

1039 + ) -> tuple[list[ConversationMessage], list[TokensPrompt | EmbedsPrompt]]:

1040 from vllm.tokenizers.mistral import MistralTokenizer

1041

1175 - conversation, engine_prompt = await renderer.render_messages_async(

1176 - messages,

1177 - chat_template_content_format=chat_template_content_format,

1178 - tokenize=(

1179 - chat_template_kwargs.pop("tokenize", False)

1180 - or isinstance(renderer.tokenizer, MistralTokenizer)

1042 + renderer = self.renderer

1043 +

1044 + default_template_kwargs = merge_kwargs(

1045 + default_template_kwargs,

1046 + dict(

1047 + tools=tool_dicts,

1048 + tokenize=isinstance(renderer.tokenizer, MistralTokenizer),

1049 ),

1182 - **chat_template_kwargs,

1050 )

1051

1185 - if "prompt_token_ids" not in engine_prompt:

1186 - extra_data = engine_prompt

1187 - engine_prompt = await self._tokenize_prompt_input_async(

1188 - request,

1189 - renderer.get_tokenizer(),

1190 - engine_prompt["prompt"],

1191 - add_special_tokens=add_special_tokens,

1192 - )

1193 - # Fill in other keys like MM data

1194 - engine_prompt.update(extra_data) # type: ignore

1195 - else:

1196 - self._validate_input(

1197 - request=request,

1198 - input_ids=engine_prompt["prompt_token_ids"], # type: ignore

1199 - input_text="",

1200 - )

1052 + tok_params = request.build_tok_params(self.model_config)

1053 + chat_params = request.build_chat_params(

1054 + default_template, default_template_content_format

1055 + ).with_defaults(default_template_kwargs)

1056

1202 - engine_prompt = cast(TokensPrompt, engine_prompt)

1057 + conversation, prompt = await renderer.render_messages_async(

1058 + messages, chat_params

1059 + )

1060 + engine_prompt = await renderer.tokenize_prompt_async(prompt, tok_params)

1061

1204 - if request.mm_processor_kwargs is not None:

1205 - engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs

1206 - if (cache_salt := getattr(request, "cache_salt", None)) is not None:

1207 - engine_prompt["cache_salt"] = cache_salt

1062 + extra_items = {

1063 + k: v

1064 + for k in ("mm_processor_kwargs", "cache_salt")

1065 + if (v := getattr(request, k, None)) is not None

1066 + }

1067 + engine_prompt.update(extra_items) # type: ignore

1068

1069 # tool parsing is done only if a tool_parser has been set and if

1070 # tool_choice is not "none" (if tool_choice is "none" but a tool_parser

1071 # is set, we want to prevent parsing a tool_call hallucinated by the LLM

1212 - should_parse_tools = tool_parser is not None and (

1213 - hasattr(request, "tool_choice") and request.tool_choice != "none"

1214 - )

1215 -

1216 - if should_parse_tools:

1217 - if not isinstance(request, ChatCompletionRequest | ResponsesRequest):

1218 - msg = (

1219 - "Tool usage is only supported for Chat Completions API "

1220 - "or Responses API requests."

1221 - )

1222 - raise NotImplementedError(msg)

1072 + if tool_parser is not None:

1073 + tool_choice = getattr(request, "tool_choice", "none")

1074 + if tool_choice != "none":

1075 + if not isinstance(request, ChatCompletionRequest | ResponsesRequest):

1076 + msg = (

1077 + "Tool usage is only supported for Chat Completions API "

1078 + "or Responses API requests."

1079 + )

1080 + raise NotImplementedError(msg)

1081

1224 - tokenizer = renderer.get_tokenizer()

1225 - request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore

1082 + # TODO: Update adjust_request to accept ResponsesRequest

1083 + tokenizer = renderer.get_tokenizer()

1084 + request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore[arg-type]

1085

1086 return conversation, [engine_prompt]

1087

1229 - async def _process_inputs(

1230 - self,

1231 - request_id: str,

1232 - engine_prompt: PromptType,

1233 - params: SamplingParams | PoolingParams,

1234 - *,

1235 - lora_request: LoRARequest | None,

1236 - trace_headers: Mapping[str, str] | None,

1237 - priority: int,

1238 - data_parallel_rank: int | None = None,

1239 - ) -> tuple[EngineCoreRequest, dict[str, Any]]:

1240 - """Use the Processor to process inputs for AsyncLLM."""

1241 - tokenization_kwargs: dict[str, Any] = {}

1242 - _validate_truncation_size(

1243 - self.max_model_len, params.truncate_prompt_tokens, tokenization_kwargs

1244 - )

1245 -

1246 - engine_request = self.input_processor.process_inputs(

1247 - request_id,

1248 - engine_prompt,

1249 - params,

1250 - lora_request=lora_request,

1251 - tokenization_kwargs=tokenization_kwargs,

1252 - trace_headers=trace_headers,

1253 - priority=priority,

1254 - data_parallel_rank=data_parallel_rank,

1255 - )

1256 - return engine_request, tokenization_kwargs

1257 -

1088 async def _render_next_turn(

1089 self,

1090 request: ResponsesRequest,

1261 - renderer: RendererLike,

1091 messages: list[ResponseInputOutputItem],

1092 tool_dicts: list[dict[str, Any]] | None,

1264 - tool_parser,

1093 + tool_parser: Callable[[TokenizerLike], ToolParser] | None,

1094 chat_template: str | None,

1095 chat_template_content_format: ChatTemplateContentFormatOption,

1096 ):

@@ -1271,24 +1100,25 @@ class OpenAIServing:

1100

1101 _, engine_prompts = await self._preprocess_chat(

1102 request,

1274 - renderer,

1103 new_messages,

1104 + default_template=chat_template,

1105 + default_template_content_format=chat_template_content_format,

1106 + default_template_kwargs=None,

1107 tool_dicts=tool_dicts,

1108 tool_parser=tool_parser,

1278 - chat_template=chat_template,

1279 - chat_template_content_format=chat_template_content_format,

1109 )

1110 return engine_prompts

1111

1112 async def _generate_with_builtin_tools(

1113 self,

1114 request_id: str,

1286 - engine_prompt: TokensPrompt,

1115 + engine_prompt: TokensPrompt | EmbedsPrompt,

1116 sampling_params: SamplingParams,

1117 + tok_params: TokenizeParams,

1118 context: ConversationContext,

1119 lora_request: LoRARequest | None = None,

1120 priority: int = 0,

1291 - **kwargs,

1121 + trace_headers: Mapping[str, str] | None = None,

1122 ):

1123 prompt_text, _, _ = get_prompt_components(engine_prompt)

1124

@@ -1297,18 +1127,21 @@ class OpenAIServing:

1127 while True:

1128 # Ensure that each sub-request has a unique request id.

1129 sub_request_id = f"{request_id}_{sub_request}"

1130 +

1131 self._log_inputs(

1132 sub_request_id,

1133 engine_prompt,

1134 params=sampling_params,

1135 lora_request=lora_request,

1136 )

1306 - trace_headers = kwargs.get("trace_headers")

1307 - engine_request, tokenization_kwargs = await self._process_inputs(

1137 +

1138 + tokenization_kwargs = tok_params.get_encode_kwargs()

1139 + engine_request = self.input_processor.process_inputs(

1140 sub_request_id,

1141 engine_prompt,

1142 sampling_params,

1143 lora_request=lora_request,

1144 + tokenization_kwargs=tokenization_kwargs,

1145 trace_headers=trace_headers,

1146 priority=priority,

1147 )

@@ -1318,10 +1151,10 @@ class OpenAIServing:

1151 sampling_params,

1152 sub_request_id,

1153 lora_request=lora_request,

1154 + trace_headers=trace_headers,

1155 priority=priority,

1156 prompt_text=prompt_text,

1157 tokenization_kwargs=tokenization_kwargs,

1324 - **kwargs,

1158 )

1159

1160 async for res in generator:

@@ -1350,7 +1183,6 @@ class OpenAIServing:

1183 elif isinstance(context, ParsableContext):

1184 engine_prompts = await self._render_next_turn(

1185 context.request,

1353 - context.renderer,

1186 context.parser.response_messages,

1187 context.tool_dicts,

1188 context.tool_parser_cls,

vllm/entrypoints/openai/responses/context.py

+1 -5

@@ -43,7 +43,6 @@ from vllm.entrypoints.openai.responses.protocol import (

43 from vllm.entrypoints.openai.responses.utils import construct_tool_dicts

44 from vllm.outputs import RequestOutput

45 from vllm.reasoning.abs_reasoning_parsers import ReasoningParser

46 -from vllm.renderers import RendererLike

46 from vllm.tokenizers import TokenizerLike

47 from vllm.tool_parsers.abstract_tool_parser import ToolParser

48 from vllm.utils import random_uuid

@@ -261,7 +260,7 @@ class ParsableContext(ConversationContext):

260 self,

261 *,

262 response_messages: list[ResponseInputOutputItem],

264 - renderer: RendererLike,

263 + tokenizer: TokenizerLike,

264 reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser] | None,

265 request: ResponsesRequest,

266 available_tools: list[str] | None,

@@ -280,7 +279,6 @@ class ParsableContext(ConversationContext):

279 if reasoning_parser_cls is None:

280 raise ValueError("reasoning_parser_cls must be provided.")

281

283 - tokenizer = renderer.get_tokenizer()

282 self.parser = get_responses_parser_for_simple_context(

283 tokenizer=tokenizer,

284 reasoning_parser_cls=reasoning_parser_cls,

@@ -290,8 +288,6 @@ class ParsableContext(ConversationContext):

288 )

289 self.tool_parser_cls = tool_parser_cls

290 self.request = request

293 - self.renderer = renderer

294 - self.tokenizer = tokenizer

291

292 self.available_tools = available_tools or []

293 self._tool_sessions: dict[str, ClientSession | Tool] = {}

vllm/entrypoints/openai/responses/protocol.py

+42 -3

@@ -59,12 +59,15 @@ from pydantic import (

59 model_validator,

60 )

62 -from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

63 -from vllm.entrypoints.openai.engine.protocol import (

64 - OpenAIBaseModel,

62 +from vllm.config import ModelConfig

63 +from vllm.entrypoints.chat_utils import (

64 + ChatCompletionMessageParam,

65 + ChatTemplateContentFormatOption,

66 )

67 +from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel

68 from vllm.exceptions import VLLMValidationError

69 from vllm.logger import init_logger

70 +from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs

71 from vllm.sampling_params import (

72 RequestOutputKind,

73 SamplingParams,

@@ -230,6 +233,42 @@ class ResponsesRequest(OpenAIBaseModel):

233 previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None

234 # --8<-- [end:responses-extra-params]

235

236 + def build_chat_params(

237 + self,

238 + default_template: str | None,

239 + default_template_content_format: ChatTemplateContentFormatOption,

240 + ) -> ChatParams:

241 + from .utils import should_continue_final_message

242 +

243 + # Check if we should continue the final message (partial completion)

244 + # This enables Anthropic-style partial message completion where the

245 + # user provides an incomplete assistant message to continue from.

246 + continue_final = should_continue_final_message(self.input)

247 +

248 + reasoning = self.reasoning

249 +

250 + return ChatParams(

251 + chat_template=default_template,

252 + chat_template_content_format=default_template_content_format,

253 + chat_template_kwargs=merge_kwargs( # To remove unset values

254 + {},

255 + dict(

256 + add_generation_prompt=not continue_final,

257 + continue_final_message=continue_final,

258 + reasoning_effort=None if reasoning is None else reasoning.effort,

259 + ),

260 + ),

261 + )

262 +

263 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

264 + return TokenizeParams(

265 + max_total_tokens=model_config.max_model_len,

266 + max_output_tokens=self.max_output_tokens or 0,

267 + truncate_prompt_tokens=-1 if self.truncation != "disabled" else None,

268 + max_total_tokens_param="max_model_len",

269 + max_output_tokens_param="max_output_tokens",

270 + )

271 +

272 _DEFAULT_SAMPLING_PARAMS = {

273 "temperature": 1.0,

274 "top_p": 1.0,

vllm/entrypoints/openai/responses/serving.py

+18 -30

@@ -114,16 +114,15 @@ from vllm.entrypoints.openai.responses.utils import (

114 construct_input_messages,

115 construct_tool_dicts,

116 extract_tool_types,

117 - should_continue_final_message,

117 )

118 from vllm.entrypoints.utils import get_max_tokens

119 from vllm.exceptions import VLLMValidationError

121 -from vllm.inputs.data import TokensPrompt

120 +from vllm.inputs.data import EmbedsPrompt, TokensPrompt

121 +from vllm.inputs.parse import get_prompt_len

122 from vllm.logger import init_logger

123 from vllm.logprobs import Logprob as SampleLogprob

124 from vllm.logprobs import SampleLogprobs

125 from vllm.outputs import CompletionOutput

126 -from vllm.renderers import RendererLike

126 from vllm.sampling_params import SamplingParams, StructuredOutputsParams

127 from vllm.tokenizers import TokenizerLike

128 from vllm.utils import random_uuid

@@ -291,13 +290,14 @@ class OpenAIServingResponses(OpenAIServing):

290 self.tool_server = tool_server

291

292 def _validate_generator_input(

294 - self, engine_prompt: TokensPrompt

293 + self,

294 + engine_prompt: TokensPrompt | EmbedsPrompt,

295 ) -> ErrorResponse | None:

296 """Add validations to the input to the generator here."""

297 - if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):

297 + prompt_len = get_prompt_len(engine_prompt)

298 + if self.max_model_len <= prompt_len:

299 error_message = (

299 - "The engine prompt length"

300 - f" {len(engine_prompt['prompt_token_ids'])} "

300 + f"The engine prompt length {prompt_len} "

301 f"exceeds the max_model_len {self.max_model_len}. "

302 "Please reduce prompt."

303 )

@@ -307,6 +307,7 @@ class OpenAIServingResponses(OpenAIServing):

307 status_code=HTTPStatus.BAD_REQUEST,

308 param="input",

309 )

310 +

311 return None

312

313 def _validate_create_responses_input(

@@ -387,8 +388,6 @@ class OpenAIServingResponses(OpenAIServing):

388 try:

389 lora_request = self._maybe_get_adapters(request)

390 model_name = self.models.model_name(lora_request)

390 - renderer = self.engine_client.renderer

391 - tokenizer = renderer.get_tokenizer()

391

392 if self.use_harmony:

393 messages, engine_prompts = self._make_request_with_harmony(

@@ -396,7 +395,7 @@ class OpenAIServingResponses(OpenAIServing):

395 )

396 else:

397 messages, engine_prompts = await self._make_request(

399 - request, prev_response, renderer

398 + request, prev_response

399 )

400

401 except (

@@ -431,6 +430,9 @@ class OpenAIServingResponses(OpenAIServing):

430 assert len(builtin_tool_list) == 0

431 available_tools = []

432 try:

433 + renderer = self.engine_client.renderer

434 + tokenizer = renderer.get_tokenizer()

435 +

436 for engine_prompt in engine_prompts:

437 maybe_error = self._validate_generator_input(engine_prompt)

438 if maybe_error is not None:

@@ -446,6 +448,7 @@ class OpenAIServingResponses(OpenAIServing):

448 sampling_params = request.to_sampling_params(

449 default_max_tokens, self.default_sampling_params

450 )

451 + tok_params = request.build_tok_params(self.model_config)

452

453 trace_headers = (

454 None

@@ -465,7 +468,7 @@ class OpenAIServingResponses(OpenAIServing):

468 # tokens during generation instead of at the end

469 context = ParsableContext(

470 response_messages=messages,

468 - renderer=renderer,

471 + tokenizer=tokenizer,

472 reasoning_parser_cls=self.reasoning_parser,

473 request=request,

474 tool_parser_cls=self.tool_parser,

@@ -495,6 +498,7 @@ class OpenAIServingResponses(OpenAIServing):

498 request_id=request.request_id,

499 engine_prompt=engine_prompt,

500 sampling_params=sampling_params,

501 + tok_params=tok_params,

502 context=context,

503 lora_request=lora_request,

504 priority=request.priority,

@@ -596,7 +600,6 @@ class OpenAIServingResponses(OpenAIServing):

600 self,

601 request: ResponsesRequest,

602 prev_response: ResponsesResponse | None,

599 - renderer: RendererLike,

603 ):

604 tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)

605 # Construct the input messages.

@@ -606,30 +609,15 @@ class OpenAIServingResponses(OpenAIServing):

609 prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,

610 prev_response_output=prev_response.output if prev_response else None,

611 )

609 - # Check if we should continue the final message (partial completion)

610 - # This enables Anthropic-style partial message completion where the

611 - # user provides an incomplete assistant message to continue from.

612 - continue_final = should_continue_final_message(request.input)

613 - chat_template_kwargs = dict(

614 - reasoning_effort=None

615 - if request.reasoning is None

616 - else request.reasoning.effort

617 - )

612

613 _, engine_prompts = await self._preprocess_chat(

614 request,

621 - renderer,

615 messages,

616 + default_template=self.chat_template,

617 + default_template_content_format=self.chat_template_content_format,

618 + default_template_kwargs=None,

619 tool_dicts=tool_dicts,

620 tool_parser=self.tool_parser,

625 - chat_template=self.chat_template,

626 - chat_template_content_format=self.chat_template_content_format,

627 - # When continuing a partial message, we set continue_final_message=True

628 - # and add_generation_prompt=False so the model continues the message

629 - # rather than starting a new one.

630 - add_generation_prompt=not continue_final,

631 - continue_final_message=continue_final,

632 - chat_template_kwargs=chat_template_kwargs,

621 )

622 return messages, engine_prompts

623

vllm/entrypoints/pooling/base/protocol.py

+22 -1

@@ -8,8 +8,12 @@ from pydantic import Field, model_validator

9 from vllm import PoolingParams

10 from vllm.config.pooler import get_use_activation

11 -from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

11 +from vllm.entrypoints.chat_utils import (

12 + ChatCompletionMessageParam,

13 + ChatTemplateContentFormatOption,

14 +)

15 from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel

16 +from vllm.renderers import ChatParams, merge_kwargs

17 from vllm.utils import random_uuid

18 from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness

@@ -119,6 +123,23 @@ class ChatRequestMixin(OpenAIBaseModel):

123 )

124 return data

125

126 + def build_chat_params(

127 + self,

128 + default_template: str | None,

129 + default_template_content_format: ChatTemplateContentFormatOption,

130 + ) -> ChatParams:

131 + return ChatParams(

132 + chat_template=self.chat_template or default_template,

133 + chat_template_content_format=default_template_content_format,

134 + chat_template_kwargs=merge_kwargs(

135 + self.chat_template_kwargs,

136 + dict(

137 + add_generation_prompt=self.add_generation_prompt,

138 + continue_final_message=self.continue_final_message,

139 + ),

140 + ),

141 + )

142 +

143

144 class EncodingRequestMixin(OpenAIBaseModel):

145 # --8<-- [start:encoding-params]

vllm/entrypoints/pooling/classify/protocol.py

+26 -4

@@ -4,10 +4,9 @@

4 import time

5 from typing import Any, TypeAlias

7 -from pydantic import (

8 - Field,

9 -)

7 +from pydantic import Field

9 +from vllm.config import ModelConfig

10 from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo

11 from vllm.entrypoints.pooling.base.protocol import (

12 ChatRequestMixin,

@@ -15,13 +14,24 @@ from vllm.entrypoints.pooling.base.protocol import (

14 CompletionRequestMixin,

15 PoolingBasicRequestMixin,

16 )

17 +from vllm.renderers import TokenizeParams

18 from vllm.utils import random_uuid

21 class ClassificationCompletionRequest(

22 PoolingBasicRequestMixin, CompletionRequestMixin, ClassifyRequestMixin

23 ):

24 - pass

24 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

25 + encoder_config = model_config.encoder_config or {}

26 +

27 + return TokenizeParams(

28 + max_total_tokens=model_config.max_model_len,

29 + max_output_tokens=0,

30 + truncate_prompt_tokens=self.truncate_prompt_tokens,

31 + do_lower_case=encoder_config.get("do_lower_case", False),

32 + add_special_tokens=self.add_special_tokens,

33 + max_total_tokens_param="max_model_len",

34 + )

37 class ClassificationChatRequest(

@@ -33,6 +43,18 @@ class ClassificationChatRequest(

43 description=("Additional kwargs to pass to the HF processor."),

44 )

46 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

47 + encoder_config = model_config.encoder_config or {}

48 +

49 + return TokenizeParams(

50 + max_total_tokens=model_config.max_model_len,

51 + max_output_tokens=0,

52 + truncate_prompt_tokens=self.truncate_prompt_tokens,

53 + do_lower_case=encoder_config.get("do_lower_case", False),

54 + add_special_tokens=self.add_special_tokens,

55 + max_total_tokens_param="max_model_len",

56 + )

57 +

59 ClassificationRequest: TypeAlias = (

60 ClassificationCompletionRequest | ClassificationChatRequest

vllm/entrypoints/pooling/classify/serving.py

+12 -37

@@ -1,8 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 -from http import HTTPStatus

5 -from typing import Final, cast

4 +from typing import Final, TypeAlias

6 import jinja2

7 import numpy as np

@@ -21,15 +20,14 @@ from vllm.entrypoints.pooling.classify.protocol import (

20 ClassificationRequest,

21 ClassificationResponse,

22 )

24 -from vllm.entrypoints.renderer import RenderConfig

23 from vllm.logger import init_logger

26 -from vllm.outputs import ClassificationOutput, PoolingRequestOutput

24 +from vllm.outputs import ClassificationOutput

25 from vllm.pooling_params import PoolingParams

27 logger = init_logger(__name__)

32 -ClassificationServeContext = ServeContext[ClassificationRequest]

30 +ClassificationServeContext: TypeAlias = ServeContext[ClassificationRequest]

33 class ServingClassification(OpenAIServing):

@@ -77,34 +75,18 @@ class ServingClassification(OpenAIServing):

75 if error_check_ret:

76 return error_check_ret

80 - _, engine_prompts = await self._preprocess_chat(

78 + _, ctx.engine_prompts = await self._preprocess_chat(

79 ctx.request,

82 - self.renderer,

80 ctx.request.messages,

84 - chat_template=ctx.request.chat_template or self.chat_template,

85 - chat_template_content_format=self.chat_template_content_format,

86 - add_generation_prompt=ctx.request.add_generation_prompt,

87 - continue_final_message=ctx.request.continue_final_message,

88 - add_special_tokens=ctx.request.add_special_tokens,

81 + default_template=self.chat_template,

82 + default_template_content_format=self.chat_template_content_format,

83 + default_template_kwargs=None,

84 )

90 - ctx.engine_prompts = engine_prompts

91 -

85 elif isinstance(ctx.request, ClassificationCompletionRequest):

93 - input_data = ctx.request.input

94 - if input_data in (None, ""):

95 - return self.create_error_response(

96 - "Input or messages must be provided",

97 - status_code=HTTPStatus.BAD_REQUEST,

98 - )

99 - if isinstance(input_data, list) and not input_data:

100 - ctx.engine_prompts = []

101 - return None

102 -

103 - renderer = self._get_completion_renderer()

104 - prompt_input = cast(str | list[str], input_data)

105 - ctx.engine_prompts = await renderer.render_prompt(

106 - prompt_or_prompts=prompt_input,

107 - config=self._build_render_config(ctx.request),

86 + ctx.engine_prompts = await self._preprocess_completion(

87 + ctx.request,

88 + prompt_input=ctx.request.input,

89 + prompt_embeds=None,

90 )

91 else:

92 return self.create_error_response("Invalid classification request type")

@@ -128,7 +110,7 @@ class ServingClassification(OpenAIServing):

110 items: list[ClassificationData] = []

111 num_prompt_tokens = 0

112

131 - final_res_batch_checked = cast(list[PoolingRequestOutput], ctx.final_res_batch)

113 + final_res_batch_checked = ctx.final_res_batch

114

115 for idx, final_res in enumerate(final_res_batch_checked):

116 classify_res = ClassificationOutput.from_base(final_res.outputs)

@@ -161,13 +143,6 @@ class ServingClassification(OpenAIServing):

143 usage=usage,

144 )

145

164 - def _build_render_config(self, request: ClassificationRequest) -> RenderConfig:

165 - return RenderConfig(

166 - max_length=self.max_model_len,

167 - truncate_prompt_tokens=request.truncate_prompt_tokens,

168 - add_special_tokens=request.add_special_tokens,

169 - )

170 -

146 async def create_classify(

147 self,

148 request: ClassificationRequest,

vllm/entrypoints/pooling/embed/protocol.py

+55 -6

@@ -3,10 +3,9 @@

3 import time

4 from typing import Any, TypeAlias

6 -from pydantic import (

7 - Field,

8 -)

6 +from pydantic import Field

8 +from vllm.config import ModelConfig

9 from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo

10 from vllm.entrypoints.pooling.base.protocol import (

11 ChatRequestMixin,

@@ -14,15 +13,47 @@ from vllm.entrypoints.pooling.base.protocol import (

13 EmbedRequestMixin,

14 PoolingBasicRequestMixin,

15 )

16 +from vllm.renderers import TokenizeParams

17 from vllm.utils import random_uuid

20 +def _get_max_total_output_tokens(

21 + model_config: ModelConfig,

22 +) -> tuple[int | None, int]:

23 + max_total_tokens = model_config.max_model_len

24 + pooler_config = model_config.pooler_config

25 +

26 + if pooler_config is None:

27 + return max_total_tokens, 0

28 +

29 + if pooler_config.enable_chunked_processing:

30 + return None, 0

31 +

32 + max_embed_len = pooler_config.max_embed_len or max_total_tokens

33 + max_output_tokens = max_total_tokens - max_embed_len

34 + return max_total_tokens, max_output_tokens

35 +

36 +

37 class EmbeddingCompletionRequest(

38 PoolingBasicRequestMixin, CompletionRequestMixin, EmbedRequestMixin

39 ):

23 - # Ordered by official OpenAI API documentation

24 - # https://platform.openai.com/docs/api-reference/embeddings

25 - pass

40 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

41 + encoder_config = model_config.encoder_config or {}

42 +

43 + (

44 + max_total_tokens,

45 + max_output_tokens,

46 + ) = _get_max_total_output_tokens(model_config)

47 +

48 + return TokenizeParams(

49 + max_total_tokens=max_total_tokens,

50 + max_output_tokens=max_output_tokens,

51 + truncate_prompt_tokens=self.truncate_prompt_tokens,

52 + do_lower_case=encoder_config.get("do_lower_case", False),

53 + add_special_tokens=self.add_special_tokens,

54 + max_total_tokens_param="max_model_len",

55 + max_output_tokens_param="max_model_len - max_embed_len",

56 + )

59 class EmbeddingChatRequest(

@@ -33,6 +64,24 @@ class EmbeddingChatRequest(

64 description=("Additional kwargs to pass to the HF processor."),

65 )

67 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

68 + encoder_config = model_config.encoder_config or {}

69 +

70 + (

71 + max_total_tokens,

72 + max_output_tokens,

73 + ) = _get_max_total_output_tokens(model_config)

74 +

75 + return TokenizeParams(

76 + max_total_tokens=max_total_tokens,

77 + max_output_tokens=max_output_tokens,

78 + truncate_prompt_tokens=self.truncate_prompt_tokens,

79 + do_lower_case=encoder_config.get("do_lower_case", False),

80 + add_special_tokens=self.add_special_tokens,

81 + max_total_tokens_param="max_model_len",

82 + max_output_tokens_param="max_model_len - max_embed_len",

83 + )

84 +

86 EmbeddingRequest: TypeAlias = EmbeddingCompletionRequest | EmbeddingChatRequest

vllm/entrypoints/pooling/embed/serving.py

+25 -35

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import json

4 from collections.abc import AsyncGenerator, Mapping

5 -from typing import Any, Final, cast

5 +from typing import Any, Final, TypeAlias

7 import torch

8 from fastapi import Request

@@ -22,8 +22,7 @@ from vllm.entrypoints.pooling.embed.protocol import (

22 EmbeddingResponse,

23 EmbeddingResponseData,

24 )

25 -from vllm.entrypoints.renderer import RenderConfig

26 -from vllm.inputs.data import TokensPrompt

25 +from vllm.inputs.data import EmbedsPrompt, TokensPrompt

26 from vllm.logger import init_logger

27 from vllm.outputs import PoolingOutput, PoolingRequestOutput

28 from vllm.pooling_params import PoolingParams

@@ -37,7 +36,7 @@ from vllm.utils.serial_utils import (

36 logger = init_logger(__name__)

40 -EmbeddingServeContext = ServeContext[EmbeddingRequest]

39 +EmbeddingServeContext: TypeAlias = ServeContext[EmbeddingRequest]

42 class OpenAIServingEmbedding(OpenAIServing):

@@ -95,19 +94,16 @@ class OpenAIServingEmbedding(OpenAIServing):

95 _, ctx.engine_prompts = await self._preprocess_chat(

96 ctx.request,

98 - self.renderer,

97 ctx.request.messages,

100 - chat_template=ctx.request.chat_template or self.chat_template,

101 - chat_template_content_format=self.chat_template_content_format,

102 - add_generation_prompt=ctx.request.add_generation_prompt,

103 - continue_final_message=ctx.request.continue_final_message,

104 - add_special_tokens=ctx.request.add_special_tokens,

98 + default_template=self.chat_template,

99 + default_template_content_format=self.chat_template_content_format,

100 + default_template_kwargs=None,

101 )

102 elif isinstance(ctx.request, EmbeddingCompletionRequest):

107 - renderer = self._get_completion_renderer()

108 - ctx.engine_prompts = await renderer.render_prompt(

109 - prompt_or_prompts=ctx.request.input,

110 - config=self._build_render_config(ctx.request),

103 + ctx.engine_prompts = await self._preprocess_completion(

104 + ctx.request,

105 + prompt_input=ctx.request.input,

106 + prompt_embeds=None,

107 )

108 else:

109 return self.create_error_response("Invalid classification request type")

@@ -117,19 +113,6 @@ class OpenAIServingEmbedding(OpenAIServing):

113 logger.exception("Error in preprocessing prompt inputs")

114 return self.create_error_response(str(e))

115

120 - def _build_render_config(self, request: EmbeddingCompletionRequest) -> RenderConfig:

121 - # Set max_length based on chunked processing capability

122 - if self._should_use_chunked_processing(request):

123 - max_length = None

124 - else:

125 - max_length = self.max_embed_len or self.max_model_len

126 -

127 - return RenderConfig(

128 - max_length=max_length,

129 - truncate_prompt_tokens=request.truncate_prompt_tokens,

130 - add_special_tokens=request.add_special_tokens,

131 - )

132 -

116 def _build_response(

117 self,

118 ctx: EmbeddingServeContext,

@@ -246,14 +229,18 @@ class OpenAIServingEmbedding(OpenAIServing):

229 lora_request=ctx.lora_request,

230 )

231

232 + tok_params = ctx.request.build_tok_params(self.model_config)

233 + tokenization_kwargs = tok_params.get_encode_kwargs()

234 +

235 # Create generator for this chunk and wrap it to return indices

236 original_generator = self.engine_client.encode(

237 chunk_engine_prompt,

238 pooling_params,

239 chunk_request_id,

240 lora_request=ctx.lora_request,

241 + tokenization_kwargs=tokenization_kwargs,

242 trace_headers=trace_headers,

256 - priority=getattr(ctx.request, "priority", 0),

243 + priority=ctx.request.priority,

244 )

245

246 generators.append(original_generator)

@@ -338,7 +325,7 @@ class OpenAIServingEmbedding(OpenAIServing):

325 async def _create_single_prompt_generator(

326 self,

327 ctx: EmbeddingServeContext,

341 - engine_prompt: TokensPrompt,

328 + engine_prompt: TokensPrompt | EmbedsPrompt,

329 pooling_params: PoolingParams,

330 trace_headers: Mapping[str, str] | None,

331 prompt_index: int,

@@ -353,23 +340,25 @@ class OpenAIServingEmbedding(OpenAIServing):

340 lora_request=ctx.lora_request,

341 )

342

343 + tok_params = ctx.request.build_tok_params(self.model_config)

344 + tokenization_kwargs = tok_params.get_encode_kwargs()

345 +

346 # Return the original generator without wrapping

347 return self.engine_client.encode(

348 engine_prompt,

349 pooling_params,

350 request_id_item,

351 lora_request=ctx.lora_request,

352 + tokenization_kwargs=tokenization_kwargs,

353 trace_headers=trace_headers,

363 - priority=getattr(ctx.request, "priority", 0),

354 + priority=ctx.request.priority,

355 )

356

357 async def _prepare_generators(

358 self,

368 - ctx: ServeContext,

359 + ctx: EmbeddingServeContext,

360 ) -> ErrorResponse | None:

361 """Override to support chunked processing."""

371 - ctx = cast(EmbeddingServeContext, ctx)

372 -

362 # Check if we should use chunked processing

363 use_chunked = self._should_use_chunked_processing(ctx.request)

364

@@ -405,7 +394,8 @@ class OpenAIServingEmbedding(OpenAIServing):

394 for i, engine_prompt in enumerate(ctx.engine_prompts):

395 # Check if this specific prompt needs chunked processing

396 if "prompt_token_ids" in engine_prompt:

408 - prompt_token_ids = engine_prompt["prompt_token_ids"]

397 + prompt_token_ids = engine_prompt["prompt_token_ids"] # type: ignore[typeddict-item]

398 +

399 if len(prompt_token_ids) > max_pos_embeddings:

400 # Use chunked processing for this prompt

401 chunk_generators = await self._process_chunked_request(

@@ -573,7 +563,7 @@ class OpenAIServingEmbedding(OpenAIServing):

563 "token IDs"

564 )

565

576 - original_token_ids = original_prompt["prompt_token_ids"]

566 + original_token_ids = original_prompt["prompt_token_ids"] # type: ignore[typeddict-item]

567

568 pooling_request_output = PoolingRequestOutput(

569 request_id=aggregator["request_id"],

vllm/entrypoints/pooling/pooling/protocol.py

+27 -3

@@ -3,11 +3,10 @@

3 import time

4 from typing import Any, Generic, TypeAlias, TypeVar

6 -from pydantic import (

7 - Field,

8 -)

6 +from pydantic import Field

8 from vllm import PoolingParams

9 +from vllm.config import ModelConfig

10 from vllm.config.pooler import get_use_activation

11 from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo

12 from vllm.entrypoints.pooling.base.protocol import (

@@ -18,6 +17,7 @@ from vllm.entrypoints.pooling.base.protocol import (

17 EncodingRequestMixin,

18 PoolingBasicRequestMixin,

19 )

20 +from vllm.renderers import TokenizeParams

21 from vllm.tasks import PoolingTask

22 from vllm.utils import random_uuid

@@ -30,6 +30,18 @@ class PoolingCompletionRequest(

30 ):

31 task: PoolingTask | None = None

33 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

34 + encoder_config = model_config.encoder_config or {}

35 +

36 + return TokenizeParams(

37 + max_total_tokens=model_config.max_model_len,

38 + max_output_tokens=0,

39 + truncate_prompt_tokens=self.truncate_prompt_tokens,

40 + do_lower_case=encoder_config.get("do_lower_case", False),

41 + add_special_tokens=self.add_special_tokens,

42 + max_total_tokens_param="max_model_len",

43 + )

44 +

45 def to_pooling_params(self):

46 return PoolingParams(

47 truncate_prompt_tokens=self.truncate_prompt_tokens,

@@ -48,6 +60,18 @@ class PoolingChatRequest(

60 description=("Additional kwargs to pass to the HF processor."),

61 )

63 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

64 + encoder_config = model_config.encoder_config or {}

65 +

66 + return TokenizeParams(

67 + max_total_tokens=model_config.max_model_len,

68 + max_output_tokens=0,

69 + truncate_prompt_tokens=self.truncate_prompt_tokens,

70 + do_lower_case=encoder_config.get("do_lower_case", False),

71 + add_special_tokens=self.add_special_tokens,

72 + max_total_tokens_param="max_model_len",

73 + )

74 +

75 def to_pooling_params(self):

76 return PoolingParams(

77 truncate_prompt_tokens=self.truncate_prompt_tokens,

vllm/entrypoints/pooling/pooling/serving.py

+16 -29

@@ -5,7 +5,7 @@ import asyncio

5 import json

6 import time

7 from collections.abc import AsyncGenerator, Sequence

8 -from typing import Final, cast

8 +from typing import Any, Final, cast

10 import jinja2

11 from fastapi import Request

@@ -14,10 +14,7 @@ from typing_extensions import assert_never

14 from vllm.engine.protocol import EngineClient

15 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption

16 from vllm.entrypoints.logger import RequestLogger

17 -from vllm.entrypoints.openai.engine.protocol import (

18 - ErrorResponse,

19 - UsageInfo,

20 -)

17 +from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo

18 from vllm.entrypoints.openai.engine.serving import OpenAIServing

19 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

20 from vllm.entrypoints.pooling.pooling.protocol import (

@@ -30,8 +27,6 @@ from vllm.entrypoints.pooling.pooling.protocol import (

27 PoolingResponse,

28 PoolingResponseData,

29 )

33 -from vllm.entrypoints.renderer import RenderConfig

34 -from vllm.entrypoints.utils import _validate_truncation_size

30 from vllm.logger import init_logger

31 from vllm.outputs import PoolingRequestOutput

32 from vllm.tasks import PoolingTask, SupportedTask

@@ -99,11 +94,6 @@ class OpenAIServingPooling(OpenAIServing):

94 "dimensions is currently not supported"

95 )

102 - truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)

103 - truncate_prompt_tokens = _validate_truncation_size(

104 - self.max_model_len, truncate_prompt_tokens

105 - )

106 -

97 if is_io_processor_request:

98 if self.io_processor is None:

99 raise ValueError(

@@ -134,19 +124,16 @@ class OpenAIServingPooling(OpenAIServing):

124

125 _, engine_prompts = await self._preprocess_chat(

126 request,

137 - self.renderer,

127 request.messages,

139 - chat_template=request.chat_template or self.chat_template,

140 - chat_template_content_format=self.chat_template_content_format,

141 - add_generation_prompt=request.add_generation_prompt,

142 - continue_final_message=request.continue_final_message,

143 - add_special_tokens=request.add_special_tokens,

128 + default_template=self.chat_template,

129 + default_template_content_format=self.chat_template_content_format,

130 + default_template_kwargs=None,

131 )

132 elif isinstance(request, PoolingCompletionRequest):

146 - renderer = self._get_completion_renderer()

147 - engine_prompts = await renderer.render_prompt(

148 - prompt_or_prompts=request.input,

149 - config=self._build_render_config(request),

133 + engine_prompts = await self._preprocess_completion(

134 + request,

135 + prompt_input=request.input,

136 + prompt_embeds=None,

137 )

138 else:

139 raise ValueError(f"Unsupported request of type {type(request)}")

@@ -207,11 +194,18 @@ class OpenAIServingPooling(OpenAIServing):

194 else await self._get_trace_headers(raw_request.headers)

195 )

196

197 + if is_io_processor_request:

198 + tokenization_kwargs: dict[str, Any] = {}

199 + else:

200 + tok_params = request.build_tok_params(self.model_config) # type: ignore

201 + tokenization_kwargs = tok_params.get_encode_kwargs()

202 +

203 generator = self.engine_client.encode(

204 engine_prompt,

205 pooling_params,

206 request_id_item,

207 lora_request=lora_request,

208 + tokenization_kwargs=tokenization_kwargs,

209 trace_headers=trace_headers,

210 priority=request.priority,

211 )

@@ -338,10 +332,3 @@ class OpenAIServingPooling(OpenAIServing):

332 return encode_bytes(bytes_only=encoding_format == "bytes_only")

333 else:

334 assert_never(encoding_format)

341 -

342 - def _build_render_config(self, request: PoolingCompletionRequest) -> RenderConfig:

343 - return RenderConfig(

344 - max_length=self.max_model_len,

345 - truncate_prompt_tokens=request.truncate_prompt_tokens,

346 - add_special_tokens=request.add_special_tokens,

347 - )

vllm/entrypoints/pooling/score/protocol.py

+25 -4

@@ -3,12 +3,10 @@

3 import time

4 from typing import Any, TypeAlias

6 -from pydantic import (

7 - BaseModel,

8 - Field,

9 -)

6 +from pydantic import BaseModel, Field

8 from vllm import PoolingParams

9 +from vllm.config import ModelConfig

10 from vllm.config.pooler import get_use_activation

11 from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo

12 from vllm.entrypoints.pooling.base.protocol import (

@@ -19,6 +17,7 @@ from vllm.entrypoints.pooling.score.utils import (

17 ScoreContentPartParam,

18 ScoreMultiModalParam,

19 )

20 +from vllm.renderers import TokenizeParams

21 from vllm.utils import random_uuid

@@ -30,6 +29,17 @@ class ScoreRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):

29 )

30 # --8<-- [end:score-extra-params]

32 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

33 + encoder_config = model_config.encoder_config or {}

34 +

35 + return TokenizeParams(

36 + max_total_tokens=model_config.max_model_len,

37 + max_output_tokens=0,

38 + truncate_prompt_tokens=self.truncate_prompt_tokens,

39 + do_lower_case=encoder_config.get("do_lower_case", False),

40 + max_total_tokens_param="max_model_len",

41 + )

42 +

43 def to_pooling_params(self):

44 return PoolingParams(

45 truncate_prompt_tokens=self.truncate_prompt_tokens,

@@ -85,6 +95,17 @@ class RerankRequest(PoolingBasicRequestMixin, ClassifyRequestMixin):

95 )

96 # --8<-- [end:rerank-extra-params]

98 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

99 + encoder_config = model_config.encoder_config or {}

100 +

101 + return TokenizeParams(

102 + max_total_tokens=model_config.max_model_len,

103 + max_output_tokens=0,

104 + truncate_prompt_tokens=self.truncate_prompt_tokens,

105 + do_lower_case=encoder_config.get("do_lower_case", False),

106 + max_total_tokens_param="max_model_len",

107 + )

108 +

109

110 class RerankDocument(BaseModel):

111 text: str | None = None

vllm/entrypoints/pooling/score/serving.py

+18 -29

@@ -34,7 +34,6 @@ from vllm.entrypoints.pooling.score.utils import (

34 compress_token_type_ids,

35 get_score_prompt,

36 )

37 -from vllm.entrypoints.utils import _validate_truncation_size

37 from vllm.inputs.data import TokensPrompt

38 from vllm.logger import init_logger

39 from vllm.lora.request import LoRARequest

@@ -68,31 +67,31 @@ class ServingScores(OpenAIServing):

68 async def _embedding_score(

69 self,

71 - tokenizer: TokenizerLike,

70 data_1: list[str],

71 data_2: list[str],

72 request: RerankRequest | ScoreRequest,

73 request_id: str,

76 - tokenization_kwargs: dict[str, Any] | None = None,

74 lora_request: LoRARequest | None | None = None,

75 trace_headers: Mapping[str, str] | None = None,

76 ) -> list[PoolingRequestOutput] | ErrorResponse:

80 - input_texts = data_1 + data_2

77 + model_config = self.model_config

78 + tokenizer = self.renderer.get_tokenizer()

82 - engine_prompts: list[TokensPrompt] = []

83 - tokenize_async = make_async(

84 - tokenizer.__call__, executor=self._tokenizer_executor

80 + encode_async = make_async(

81 + tokenizer.encode,

82 + executor=self._tokenizer_executor,

83 )

87 - tokenization_kwargs = tokenization_kwargs or {}

85 + input_texts = data_1 + data_2

86 +

87 + tokenization_kwargs = request.build_tok_params(model_config).get_encode_kwargs()

88 tokenized_prompts = await asyncio.gather(

89 - *(tokenize_async(t, **tokenization_kwargs) for t in input_texts)

89 + *(encode_async(t, **tokenization_kwargs) for t in input_texts)

90 )

92 + engine_prompts: list[TokensPrompt] = []

93 for tok_result, input_text in zip(tokenized_prompts, input_texts):

93 - text_token_prompt = self._validate_input(

94 - request, tok_result["input_ids"], input_text

95 - )

94 + text_token_prompt = self._validate_input(request, tok_result, input_text)

96 engine_prompts.append(

97 TokensPrompt(prompt_token_ids=text_token_prompt["prompt_token_ids"])

@@ -184,15 +183,16 @@ class ServingScores(OpenAIServing):

183

184 async def _cross_encoding_score(

185 self,

187 - tokenizer: TokenizerLike,

186 data_1: list[str] | list[ScoreContentPartParam],

187 data_2: list[str] | list[ScoreContentPartParam],

188 request: RerankRequest | ScoreRequest,

189 request_id: str,

192 - tokenization_kwargs: dict[str, Any] | None = None,

190 lora_request: LoRARequest | None | None = None,

191 trace_headers: Mapping[str, str] | None = None,

192 ) -> list[PoolingRequestOutput] | ErrorResponse:

193 + model_config = self.model_config

194 + tokenizer = self.renderer.get_tokenizer()

195 +

196 request_prompts: list[str] = []

197 engine_prompts: list[TokensPrompt] = []

198

@@ -202,12 +202,13 @@ class ServingScores(OpenAIServing):

202 if isinstance(tokenizer, MistralTokenizer):

203 raise ValueError("MistralTokenizer not supported for cross-encoding")

204

205 - tokenization_kwargs = tokenization_kwargs or {}

205 + tok_kwargs = request.build_tok_params(model_config).get_encode_kwargs()

206

207 input_pairs = [(t1, t2) for t1, t2 in zip(data_1, data_2)]

208

209 preprocess_async = make_async(

210 - self._preprocess_score, executor=self._tokenizer_executor

210 + self._preprocess_score,

211 + executor=self._tokenizer_executor,

212 )

213

214 preprocessed_prompts = await asyncio.gather(

@@ -215,7 +216,7 @@ class ServingScores(OpenAIServing):

216 preprocess_async(

217 request=request,

218 tokenizer=tokenizer,

218 - tokenization_kwargs=tokenization_kwargs,

219 + tokenization_kwargs=tok_kwargs,

220 data_1=t1,

221 data_2=t2,

222 )

@@ -286,14 +287,6 @@ class ServingScores(OpenAIServing):

287 raw_request: Request | None = None,

288 ) -> list[PoolingRequestOutput] | ErrorResponse:

289 lora_request = self._maybe_get_adapters(request)

289 - tokenizer = self.renderer.get_tokenizer()

290 -

291 - truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)

292 -

293 - tokenization_kwargs: dict[str, Any] = {}

294 - _validate_truncation_size(

295 - self.max_model_len, truncate_prompt_tokens, tokenization_kwargs

296 - )

290

291 trace_headers = (

292 None

@@ -322,24 +315,20 @@ class ServingScores(OpenAIServing):

315

316 if self.model_config.is_cross_encoder:

317 return await self._cross_encoding_score(

325 - tokenizer=tokenizer,

318 data_1=data_1, # type: ignore[arg-type]

319 data_2=data_2, # type: ignore[arg-type]

320 request=request,

321 request_id=request_id,

330 - tokenization_kwargs=tokenization_kwargs,

322 lora_request=lora_request,

323 trace_headers=trace_headers,

324 )

325

326 else:

327 return await self._embedding_score(

337 - tokenizer=tokenizer,

328 data_1=data_1, # type: ignore[arg-type]

329 data_2=data_2, # type: ignore[arg-type]

330 request=request,

331 request_id=request_id,

342 - tokenization_kwargs=tokenization_kwargs,

332 lora_request=lora_request,

333 trace_headers=trace_headers,

334 )

vllm/entrypoints/renderer.py deleted

-411

@@ -1,411 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import asyncio

5 -import io

6 -from abc import ABC, abstractmethod

7 -from dataclasses import dataclass

8 -from typing import Annotated

9 -

10 -import pybase64

11 -import torch

12 -from pydantic import Field

13 -

14 -from vllm.config import ModelConfig

15 -from vllm.exceptions import VLLMValidationError

16 -from vllm.inputs.data import EmbedsPrompt, TextPrompt, TokensPrompt

17 -from vllm.inputs.parse import get_prompt_components, parse_raw_prompts

18 -from vllm.tokenizers import TokenizerLike

19 -from vllm.utils.async_utils import AsyncMicrobatchTokenizer

20 -

21 -

22 -@dataclass(frozen=True)

23 -class RenderConfig:

24 - """Configuration to control how prompts are prepared."""

25 -

26 - max_length: int | None = None

27 - """Maximum allowable total input token length. If provided,

28 - token inputs longer than this raise `ValueError`."""

29 -

30 - truncate_prompt_tokens: int | None = None

31 - """Number of tokens to keep. `None` means no truncation.

32 - `0` yields an empty list (and skips embeds).

33 - `-1` maps to `model_config.max_model_len`."""

34 -

35 - add_special_tokens: bool = True

36 - """Whether to add model-specific special tokens during tokenization."""

37 -

38 - cache_salt: str | None = None

39 - """String to disambiguate prefix cache entries."""

40 -

41 - needs_detokenization: bool | None = False

42 - """If True, detokenize IDs back to text for inclusion in outputs."""

43 -

44 - def verify_truncate_prompt_tokens(self, model_config: ModelConfig) -> int | None:

45 - """Validate and normalize `truncate_prompt_tokens` parameter."""

46 - truncate_prompt_tokens = self.truncate_prompt_tokens

47 - if truncate_prompt_tokens is None or truncate_prompt_tokens == 0:

48 - return truncate_prompt_tokens

49 -

50 - if truncate_prompt_tokens < 0:

51 - truncate_prompt_tokens = model_config.max_model_len

52 -

53 - max_length = self.max_length

54 - if max_length is not None and truncate_prompt_tokens > max_length: # type: ignore[operator]

55 - raise ValueError(

56 - f"{truncate_prompt_tokens=} cannot be greater than "

57 - f"{max_length=}. Please select a smaller truncation size."

58 - )

59 -

60 - return truncate_prompt_tokens

61 -

62 -

63 -class BaseRenderer(ABC):

64 - """

65 - Base class for unified input processing and rendering.

66 -

67 - The Renderer serves as a unified input processor that consolidates

68 - tokenization, chat template formatting, and multimodal input handling

69 - into a single component.

70 - It converts high-level API requests (OpenAI-style JSON) into token IDs and

71 - multimodal features ready for engine consumption.

72 -

73 - Key responsibilities:

74 - - Convert text prompts to token sequences with proper special tokens

75 - - Apply chat templates and format conversations

76 - - Handle multimodal inputs (images, audio, etc.) when applicable

77 - - Manage prompt truncation and length validation

78 - - Provide clean separation between API layer and engine core

79 - """

80 -

81 - def __init__(

82 - self,

83 - model_config: ModelConfig,

84 - tokenizer: TokenizerLike | None = None,

85 - ):

86 - super().__init__()

87 - self.model_config = model_config

88 - self.tokenizer = tokenizer

89 -

90 - @abstractmethod

91 - async def render_prompt(

92 - self,

93 - *,

94 - prompt_or_prompts: str | list[str] | list[int] | list[list[int]],

95 - config: RenderConfig,

96 - ) -> list[TokensPrompt]:

97 - """

98 - Convert text or token inputs into engine-ready TokensPrompt objects.

99 -

100 - This method accepts text or token inputs and produces a

101 - list of [`TokensPrompt`][vllm.inputs.data.TokensPrompt] objects

102 - for the engine.

103 -

104 - Args:

105 - prompt_or_prompts: One of:

106 - - `str`: Single text prompt.

107 - - `list[str]`: Batch of text prompts.

108 - - `list[int]`: Single pre-tokenized sequence.

109 - - `list[list[int]]`: Batch of pre-tokenized sequences.

110 - config: Render configuration controlling how prompts are prepared

111 - (e.g., tokenization and length handling).

112 -

113 - Returns:

114 - list[TokensPrompt]: Engine-ready token prompts.

115 -

116 - Raises:

117 - ValueError: If input formats are invalid or length limits exceeded.

118 - """

119 - raise NotImplementedError

120 -

121 - @abstractmethod

122 - async def render_prompt_and_embeds(

123 - self,

124 - *,

125 - prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None,

126 - prompt_embeds: bytes | list[bytes] | None = None,

127 - config: RenderConfig,

128 - ) -> list[TokensPrompt | EmbedsPrompt]:

129 - """

130 - Convert text/token and/or base64-encoded embeddings inputs into

131 - engine-ready prompt objects using a unified RenderConfig.

132 -

133 - At least one of `prompt_or_prompts` or `prompt_embeds` must be

134 - provided and non-empty. If both are omitted or empty (e.g., empty

135 - string and empty list), a `ValueError` is raised.

136 -

137 - Args:

138 - prompt_or_prompts: Text or token inputs to include.

139 - prompt_embeds: Base64-encoded bytes (or list thereof) containing a

140 - torch-saved tensor to be used as prompt embeddings.

141 - config: Render configuration controlling how prompts are prepared

142 - (e.g., tokenization and length handling).

143 -

144 - Returns:

145 - list[Union[TokensPrompt, EmbedsPrompt]]:

146 - Engine-ready prompt objects.

147 -

148 - Raises:

149 - ValueError: If both `prompt_or_prompts` and `prompt_embeds`

150 - are omitted or empty (decoder prompt cannot be empty), or if

151 - length limits are exceeded.

152 - """

153 - raise NotImplementedError

154 -

155 - def load_prompt_embeds(

156 - self,

157 - prompt_embeds: bytes | list[bytes],

158 - truncate_prompt_tokens: Annotated[int, Field(ge=0)] | None = None,

159 - cache_salt: str | None = None,

160 - ) -> list[EmbedsPrompt]:

161 - """Load and validate base64-encoded embeddings into prompt objects."""

162 - if not self.model_config.enable_prompt_embeds:

163 - raise VLLMValidationError(

164 - "You must set `--enable-prompt-embeds` to input `prompt_embeds`.",

165 - parameter="prompt_embeds",

166 - )

167 -

168 - def _load_and_validate_embed(embed: bytes) -> EmbedsPrompt:

169 - # Enable sparse tensor integrity checks to prevent out-of-bounds

170 - # writes from maliciously crafted tensors

171 - with torch.sparse.check_sparse_tensor_invariants():

172 - tensor = torch.load(

173 - io.BytesIO(pybase64.b64decode(embed, validate=True)),

174 - weights_only=True,

175 - map_location=torch.device("cpu"),

176 - )

177 - assert isinstance(tensor, torch.Tensor) and tensor.dtype in (

178 - torch.float32,

179 - torch.bfloat16,

180 - torch.float16,

181 - )

182 - tensor = tensor.to_dense()

183 - if tensor.dim() > 2:

184 - tensor = tensor.squeeze(0)

185 - assert tensor.dim() == 2

186 - if truncate_prompt_tokens is not None:

187 - tensor = tensor[-truncate_prompt_tokens:]

188 - embeds_prompt = EmbedsPrompt(prompt_embeds=tensor)

189 - if cache_salt is not None:

190 - embeds_prompt["cache_salt"] = cache_salt

191 - return embeds_prompt

192 -

193 - if isinstance(prompt_embeds, list):

194 - return [_load_and_validate_embed(embed) for embed in prompt_embeds]

195 -

196 - return [_load_and_validate_embed(prompt_embeds)]

197 -

198 -

199 -class CompletionRenderer(BaseRenderer):

200 - def __init__(

201 - self,

202 - model_config: ModelConfig,

203 - tokenizer: TokenizerLike | None = None,

204 - async_tokenizer_pool: dict[TokenizerLike, AsyncMicrobatchTokenizer]

205 - | None = None,

206 - ):

207 - super().__init__(model_config, tokenizer)

208 - self.async_tokenizer_pool = async_tokenizer_pool

209 - self.async_tokenizer: AsyncMicrobatchTokenizer | None = None

210 -

211 - async def render_prompt(

212 - self,

213 - *,

214 - prompt_or_prompts: str | list[str] | list[int] | list[list[int]],

215 - config: RenderConfig,

216 - ) -> list[TokensPrompt]:

217 - """Implementation of prompt rendering for completion-style requests.

218 -

219 - Uses async tokenizer pooling for improved performance. See base class

220 - for detailed parameter documentation.

221 - """

222 - truncate_prompt_tokens = config.verify_truncate_prompt_tokens(self.model_config)

223 - if truncate_prompt_tokens == 0:

224 - return []

225 -

226 - tasks = (

227 - self._create_prompt(

228 - prompt_input,

229 - config=config,

230 - truncate_prompt_tokens=truncate_prompt_tokens,

231 - )

232 - for prompt_input in parse_raw_prompts(prompt_or_prompts)

233 - )

234 -

235 - return await asyncio.gather(*tasks)

236 -

237 - async def render_prompt_and_embeds(

238 - self,

239 - *,

240 - prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None,

241 - prompt_embeds: bytes | list[bytes] | None = None,

242 - config: RenderConfig,

243 - ) -> list[TokensPrompt | EmbedsPrompt]:

244 - """

245 - Render text/token prompts and/or precomputed embedding prompts. At

246 - least one of `prompt_or_prompts` or `prompt_embeds` must be provided.

247 - """

248 - truncate_prompt_tokens = config.verify_truncate_prompt_tokens(self.model_config)

249 - if truncate_prompt_tokens == 0:

250 - return []

251 -

252 - rendered: list[TokensPrompt | EmbedsPrompt] = []

253 -

254 - if prompt_embeds is not None:

255 - rendered.extend(

256 - self.load_prompt_embeds(

257 - prompt_embeds, truncate_prompt_tokens, config.cache_salt

258 - )

259 - )

260 - if prompt_or_prompts is None or prompt_or_prompts == "":

261 - return rendered

262 -

263 - token_prompts = await self.render_prompt(

264 - prompt_or_prompts=prompt_or_prompts,

265 - config=config,

266 - )

267 - rendered.extend(token_prompts)

268 -

269 - return rendered

270 -

271 - def _maybe_apply_truncation(

272 - self, token_ids: list[int], truncate_prompt_tokens: int | None

273 - ) -> list[int]:

274 - """Apply truncation to token sequence."""

275 - if truncate_prompt_tokens is None:

276 - return token_ids

277 - if truncate_prompt_tokens >= len(token_ids):

278 - return token_ids

279 -

280 - return token_ids[-truncate_prompt_tokens:]

281 -

282 - async def _create_prompt(

283 - self,

284 - prompt_input: TextPrompt | TokensPrompt,

285 - config: RenderConfig,

286 - truncate_prompt_tokens: int | None,

287 - ) -> TokensPrompt:

288 - prompt, prompt_token_ids, _ = get_prompt_components(prompt_input)

289 -

290 - if prompt_token_ids is not None:

291 - # NOTE: detokenization is needed when echo is enabled,

292 - # where the input token IDs are decoded back to text.

293 - return await self._create_prompt_from_token_ids(

294 - prompt_token_ids,

295 - config.max_length,

296 - truncate_prompt_tokens,

297 - config.cache_salt,

298 - config.needs_detokenization,

299 - )

300 -

301 - if prompt is not None:

302 - return await self._create_prompt_from_text(

303 - prompt,

304 - config.max_length,

305 - truncate_prompt_tokens,

306 - config.add_special_tokens,

307 - config.cache_salt,

308 - )

309 -

310 - # TODO: Also handle embeds prompt using this method

311 - raise NotImplementedError

312 -

313 - async def _create_prompt_from_text(

314 - self,

315 - text: str,

316 - max_length: int | None,

317 - truncate_prompt_tokens: int | None,

318 - add_special_tokens: bool,

319 - cache_salt: str | None,

320 - ) -> TokensPrompt:

321 - """Tokenize text input asynchronously."""

322 - async_tokenizer = self._get_async_tokenizer()

323 -

324 - # Handle encoder-specific preprocessing

325 - if (

326 - self.model_config.encoder_config is not None

327 - and self.model_config.encoder_config.get("do_lower_case", False)

328 - ):

329 - text = text.lower()

330 -

331 - # Tokenize texts

332 - if truncate_prompt_tokens is None:

333 - encoded = await async_tokenizer(text, add_special_tokens=add_special_tokens)

334 - else:

335 - encoded = await async_tokenizer(

336 - text,

337 - add_special_tokens=add_special_tokens,

338 - truncation=True,

339 - max_length=truncate_prompt_tokens,

340 - )

341 -

342 - return self._create_tokens_prompt(

343 - encoded.input_ids, max_length, cache_salt, text

344 - )

345 -

346 - async def _create_prompt_from_token_ids(

347 - self,

348 - token_ids: list[int],

349 - max_length: int | None,

350 - truncate_prompt_tokens: int | None,

351 - cache_salt: str | None,

352 - needs_detokenization: bool | None = False,

353 - ) -> TokensPrompt:

354 - """Optionally detokenize token IDs and build a tokens prompt."""

355 - token_ids = self._maybe_apply_truncation(token_ids, truncate_prompt_tokens)

356 -

357 - prompt = None

358 - if needs_detokenization:

359 - async_tokenizer = self._get_async_tokenizer()

360 - prompt = await async_tokenizer.decode(token_ids)

361 -

362 - return self._create_tokens_prompt(

363 - token_ids=token_ids,

364 - max_length=max_length,

365 - cache_salt=cache_salt,

366 - prompt=prompt,

367 - )

368 -

369 - def _get_async_tokenizer(self) -> AsyncMicrobatchTokenizer:

370 - """Get or create async tokenizer using shared pool."""

371 - async_tokenizer = self.async_tokenizer

372 - if async_tokenizer is not None:

373 - return async_tokenizer

374 -

375 - tokenizer = self.tokenizer

376 - if tokenizer is None:

377 - raise ValueError("No tokenizer available for text input processing")

378 -

379 - if self.async_tokenizer_pool is None:

380 - async_tokenizer = AsyncMicrobatchTokenizer(tokenizer)

381 - else:

382 - async_tokenizer = self.async_tokenizer_pool.get(tokenizer)

383 - if async_tokenizer is None:

384 - async_tokenizer = AsyncMicrobatchTokenizer(tokenizer)

385 - self.async_tokenizer_pool[tokenizer] = async_tokenizer

386 - self.async_tokenizer = async_tokenizer

387 - return async_tokenizer

388 -

389 - def _create_tokens_prompt(

390 - self,

391 - token_ids: list[int],

392 - max_length: int | None = None,

393 - cache_salt: str | None = None,

394 - prompt: str | None = None,

395 - ) -> TokensPrompt:

396 - """Create validated TokensPrompt."""

397 - if max_length is not None and len(token_ids) > max_length:

398 - raise VLLMValidationError(

399 - f"This model's maximum context length is {max_length} tokens. "

400 - f"However, your request has {len(token_ids)} input tokens. "

401 - "Please reduce the length of the input messages.",

402 - parameter="input_tokens",

403 - value=len(token_ids),

404 - )

405 -

406 - tokens_prompt = TokensPrompt(prompt_token_ids=token_ids)

407 - if cache_salt is not None:

408 - tokens_prompt["cache_salt"] = cache_salt

409 - if prompt is not None:

410 - tokens_prompt["prompt"] = prompt

411 - return tokens_prompt

vllm/entrypoints/serve/disagg/protocol.py

@@ -4,12 +4,14 @@ from typing import Any

5 from pydantic import BaseModel, Field

7 +from vllm.config import ModelConfig

8 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs

9 from vllm.entrypoints.openai.engine.protocol import (

10 SamplingParams,

11 StreamOptions,

12 )

13 from vllm.logprobs import Logprob

14 +from vllm.renderers import TokenizeParams

15 from vllm.utils import random_uuid

@@ -62,6 +64,12 @@ class GenerateRequest(BaseModel):

64 description="KVTransfer parameters used for disaggregated serving.",

65 )

67 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

68 + return TokenizeParams(

69 + max_total_tokens=None,

70 + max_output_tokens=0,

71 + )

72 +

74 class GenerateResponseChoice(BaseModel):

75 index: int

vllm/entrypoints/serve/disagg/serving.py

+11 -6

@@ -101,12 +101,13 @@ class ServingTokens(OpenAIServing):

101

102 # TODO(NickLucche): Change to EngineCoreRequest once Renderer work is

103 # completed

104 - engine_prompt = TokensPrompt(prompt_token_ids=request.token_ids)

105 - if request.features is not None:

106 - engine_prompt["multi_modal_data"] = None

107 -

108 - if hasattr(request, "cache_salt") and request.cache_salt is not None:

109 - engine_prompt["cache_salt"] = request.cache_salt

104 + engine_prompts = await self._preprocess_completion(

105 + request,

106 + prompt_input=request.token_ids,

107 + prompt_embeds=None,

108 + )

109 + assert len(engine_prompts) == 1

110 + engine_prompt = engine_prompts[0]

111

112 # Schedule the request and get the result generator.

113 result_generator: AsyncGenerator[RequestOutput, None] | None = None

@@ -128,11 +129,15 @@ class ServingTokens(OpenAIServing):

129 else await self._get_trace_headers(raw_request.headers)

130 )

131

132 + tok_params = request.build_tok_params(self.model_config)

133 + tokenization_kwargs = tok_params.get_encode_kwargs()

134 +

135 result_generator = self.engine_client.generate(

136 engine_prompt,

137 sampling_params,

138 request_id,

139 lora_request=lora_request,

140 + tokenization_kwargs=tokenization_kwargs,

141 trace_headers=trace_headers,

142 priority=request.priority,

143 )

vllm/entrypoints/serve/tokenize/protocol.py

+41

@@ -6,8 +6,10 @@ from typing import Any, TypeAlias

7 from pydantic import ConfigDict, Field, model_validator

9 +from vllm.config import ModelConfig

10 from vllm.entrypoints.chat_utils import (

11 ChatCompletionMessageParam,

12 + ChatTemplateContentFormatOption,

13 )

14 from vllm.entrypoints.openai.chat_completion.protocol import (

15 ChatCompletionToolsParam,

@@ -15,6 +17,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (

17 from vllm.entrypoints.openai.engine.protocol import (

18 OpenAIBaseModel,

19 )

20 +from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs

23 class TokenizeCompletionRequest(OpenAIBaseModel):

@@ -35,6 +38,13 @@ class TokenizeCompletionRequest(OpenAIBaseModel):

38 ),

39 )

41 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

42 + return TokenizeParams(

43 + max_total_tokens=None,

44 + max_output_tokens=0,

45 + add_special_tokens=self.add_special_tokens,

46 + )

47 +

49 class TokenizeChatRequest(OpenAIBaseModel):

50 model: str | None = None

@@ -109,6 +119,30 @@ class TokenizeChatRequest(OpenAIBaseModel):

119 )

120 return data

121

122 + def build_chat_params(

123 + self,

124 + default_template: str | None,

125 + default_template_content_format: ChatTemplateContentFormatOption,

126 + ) -> ChatParams:

127 + return ChatParams(

128 + chat_template=self.chat_template or default_template,

129 + chat_template_content_format=default_template_content_format,

130 + chat_template_kwargs=merge_kwargs(

131 + self.chat_template_kwargs,

132 + dict(

133 + add_generation_prompt=self.add_generation_prompt,

134 + continue_final_message=self.continue_final_message,

135 + ),

136 + ),

137 + )

138 +

139 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

140 + return TokenizeParams(

141 + max_total_tokens=None,

142 + max_output_tokens=0,

143 + add_special_tokens=self.add_special_tokens,

144 + )

145 +

146

147 TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest

148

@@ -124,6 +158,13 @@ class DetokenizeRequest(OpenAIBaseModel):

158 model: str | None = None

159 tokens: list[int]

160

161 + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

162 + return TokenizeParams(

163 + max_total_tokens=None,

164 + max_output_tokens=0,

165 + needs_detokenization=True,

166 + )

167 +

168

169 class DetokenizeResponse(OpenAIBaseModel):

170 prompt: str

vllm/entrypoints/serve/tokenize/serving.py

+19 -28

@@ -9,12 +9,9 @@ from fastapi import Request

9 from vllm.engine.protocol import EngineClient

10 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption

11 from vllm.entrypoints.logger import RequestLogger

12 -from vllm.entrypoints.openai.engine.protocol import (

13 - ErrorResponse,

14 -)

12 +from vllm.entrypoints.openai.engine.protocol import ErrorResponse

13 from vllm.entrypoints.openai.engine.serving import OpenAIServing

14 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

17 -from vllm.entrypoints.renderer import RenderConfig

15 from vllm.entrypoints.serve.tokenize.protocol import (

16 DetokenizeRequest,

17 DetokenizeResponse,

@@ -83,21 +80,17 @@ class OpenAIServingTokenization(OpenAIServing):

81 _, engine_prompts = await self._preprocess_chat(

82 request,

86 - self.renderer,

83 request.messages,

84 + default_template=self.chat_template,

85 + default_template_content_format=self.chat_template_content_format,

86 + default_template_kwargs=None,

87 tool_dicts=tool_dicts,

89 - chat_template=request.chat_template or self.chat_template,

90 - chat_template_content_format=self.chat_template_content_format,

91 - add_generation_prompt=request.add_generation_prompt,

92 - continue_final_message=request.continue_final_message,

93 - chat_template_kwargs=request.chat_template_kwargs,

94 - add_special_tokens=request.add_special_tokens,

88 )

89 else:

97 - renderer = self._get_completion_renderer()

98 - engine_prompts = await renderer.render_prompt(

99 - prompt_or_prompts=request.prompt,

100 - config=self._build_render_config(request),

90 + engine_prompts = await self._preprocess_completion(

91 + request,

92 + prompt_input=request.prompt,

93 + prompt_embeds=None,

94 )

95 except (ValueError, TypeError, jinja2.TemplateError) as e:

96 logger.exception("Error in preprocessing prompt inputs")

@@ -106,11 +99,14 @@ class OpenAIServingTokenization(OpenAIServing):

99 input_ids: list[int] = []

100 for engine_prompt in engine_prompts:

101 self._log_inputs(

109 - request_id, engine_prompt, params=None, lora_request=lora_request

102 + request_id,

103 + engine_prompt,

104 + params=None,

105 + lora_request=lora_request,

106 )

107

112 - if isinstance(engine_prompt, dict) and "prompt_token_ids" in engine_prompt:

113 - input_ids.extend(engine_prompt["prompt_token_ids"])

108 + if "prompt_token_ids" in engine_prompt:

109 + input_ids.extend(engine_prompt["prompt_token_ids"]) # type: ignore[typeddict-item]

110

111 token_strs = None

112 if request.return_token_strs:

@@ -136,7 +132,6 @@ class OpenAIServingTokenization(OpenAIServing):

132 request_id = f"tokenize-{self._base_request_id(raw_request)}"

133

134 lora_request = self._maybe_get_adapters(request)

139 - tokenizer = self.renderer.get_tokenizer()

135

136 self._log_inputs(

137 request_id,

@@ -145,14 +140,13 @@ class OpenAIServingTokenization(OpenAIServing):

140 lora_request=lora_request,

141 )

142

148 - prompt_input = await self._tokenize_prompt_input_async(

149 - request,

150 - tokenizer,

151 - request.tokens,

143 + engine_prompt = await self.renderer.tokenize_prompt_async(

144 + TokensPrompt(prompt_token_ids=request.tokens),

145 + request.build_tok_params(self.model_config),

146 )

153 - input_text = prompt_input["prompt"]

147 + prompt_text = engine_prompt["prompt"] # type: ignore[typeddict-item]

148

155 - return DetokenizeResponse(prompt=input_text)

149 + return DetokenizeResponse(prompt=prompt_text)

150

151 async def get_tokenizer_info(

152 self,

@@ -165,9 +159,6 @@ class OpenAIServingTokenization(OpenAIServing):

159 except Exception as e:

160 return self.create_error_response(f"Failed to get tokenizer info: {str(e)}")

161

168 - def _build_render_config(self, request: TokenizeRequest) -> RenderConfig:

169 - return RenderConfig(add_special_tokens=request.add_special_tokens)

170 -

162

163 @dataclass

164 class TokenizerInfo:

vllm/entrypoints/utils.py

+4 -36

@@ -8,7 +8,7 @@ import os

8 from argparse import Namespace

9 from logging import Logger

10 from string import Template

11 -from typing import TYPE_CHECKING, Any

11 +from typing import TYPE_CHECKING

13 import regex as re

14 from fastapi import Request

@@ -18,9 +18,9 @@ from starlette.background import BackgroundTask, BackgroundTasks

18 from vllm import envs

19 from vllm.engine.arg_utils import EngineArgs

20 from vllm.inputs import EmbedsPrompt, TokensPrompt

21 +from vllm.inputs.parse import get_prompt_len

22 from vllm.logger import current_formatter_type, init_logger

23 from vllm.platforms import current_platform

23 -from vllm.utils import length_from_prompt_token_ids_or_embeds

24 from vllm.utils.argparse_utils import FlexibleArgumentParser

26 if TYPE_CHECKING:

@@ -34,9 +34,7 @@ if TYPE_CHECKING:

34 StreamOptions,

35 )

36 from vllm.entrypoints.openai.models.protocol import LoRAModulePath

37 - from vllm.entrypoints.openai.responses.protocol import (

38 - ResponsesRequest,

39 - )

37 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest

38 else:

39 ChatCompletionRequest = object

40 CompletionRequest = object

@@ -188,33 +186,6 @@ def cli_env_setup():

186 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"

187

188

191 -def _validate_truncation_size(

192 - max_model_len: int,

193 - truncate_prompt_tokens: int | None,

194 - tokenization_kwargs: dict[str, Any] | None = None,

195 -) -> int | None:

196 - if truncate_prompt_tokens is not None:

197 - if truncate_prompt_tokens <= -1:

198 - truncate_prompt_tokens = max_model_len

199 -

200 - if truncate_prompt_tokens > max_model_len:

201 - raise ValueError(

202 - f"truncate_prompt_tokens value ({truncate_prompt_tokens}) "

203 - f"is greater than max_model_len ({max_model_len})."

204 - f" Please, select a smaller truncation size."

205 - )

206 -

207 - if tokenization_kwargs is not None:

208 - tokenization_kwargs["truncation"] = True

209 - tokenization_kwargs["max_length"] = truncate_prompt_tokens

210 -

211 - else:

212 - if tokenization_kwargs is not None:

213 - tokenization_kwargs["truncation"] = False

214 -

215 - return truncate_prompt_tokens

216 -

217 -

189 def get_max_tokens(

190 max_model_len: int,

191 request: "CompletionRequest | ChatCompletionRequest | ResponsesRequest",

@@ -233,10 +204,7 @@ def get_max_tokens(

204 # CompletionRequest (also a fallback for ChatCompletionRequest)

205 max_tokens = getattr(request, "max_tokens", None)

206

236 - input_length = length_from_prompt_token_ids_or_embeds(

237 - prompt.get("prompt_token_ids"), # type: ignore[arg-type]

238 - prompt.get("prompt_embeds"), # type: ignore[arg-type]

239 - )

207 + input_length = get_prompt_len(prompt)

208 default_max_tokens = max_model_len - input_length

209 max_output_tokens = current_platform.get_max_output_tokens(input_length)

210

vllm/inputs/data.py

+12 -42

@@ -21,12 +21,7 @@ else:

21 MultiModalUUIDDict = object

24 -class TextPrompt(TypedDict):

25 - """Schema for a text prompt."""

26 -

27 - prompt: str

28 - """The input text to be tokenized before passing to the model."""

29 -

24 +class _CommonKeys(TypedDict):

25 multi_modal_data: NotRequired[MultiModalDataDict | None]

26 """

27 Optional multi-modal data to pass to the model,

@@ -56,7 +51,14 @@ class TextPrompt(TypedDict):

51 """

59 -class TokensPrompt(TypedDict):

54 +class TextPrompt(_CommonKeys):

55 + """Schema for a text prompt."""

56 +

57 + prompt: str

58 + """The input text to be tokenized before passing to the model."""

59 +

60 +

61 +class TokensPrompt(_CommonKeys):

62 """Schema for a tokenized prompt."""

64 prompt_token_ids: list[int]

@@ -68,47 +70,15 @@ class TokensPrompt(TypedDict):

70 token_type_ids: NotRequired[list[int]]

71 """A list of token type IDs to pass to the cross encoder model."""

71 - multi_modal_data: NotRequired[MultiModalDataDict | None]

72 - """

73 - Optional multi-modal data to pass to the model,

74 - if the model supports it.

75 - """

76 -

77 - mm_processor_kwargs: NotRequired[dict[str, Any] | None]

78 - """

79 - Optional multi-modal processor kwargs to be forwarded to the

80 - multimodal input mapper & processor. Note that if multiple modalities

81 - have registered mappers etc for the model being considered, we attempt

82 - to pass the mm_processor_kwargs to each of them.

83 - """

84 -

85 - multi_modal_uuids: NotRequired[MultiModalUUIDDict]

86 - """

87 - Optional user-specified UUIDs for multimodal items, mapped by modality.

88 - Lists must match the number of items per modality and may contain `None`.

89 - For `None` entries, the hasher will compute IDs automatically; non-None

90 - entries override the default hashes for caching.

91 - """

93 - cache_salt: NotRequired[str]

94 - """

95 - Optional cache salt to be used for prefix caching.

96 - """

97 -

98 -

99 -class EmbedsPrompt(TypedDict):

74 +class EmbedsPrompt(_CommonKeys):

75 """Schema for a prompt provided via token embeddings."""

77 prompt_embeds: torch.Tensor

78 """The embeddings of the prompt."""

105 - cache_salt: NotRequired[str]

106 - """

107 - Optional cache salt to be used for prefix caching.

108 - """

109 -

111 -class DataPrompt(TypedDict):

81 +class DataPrompt(_CommonKeys):

82 """Represents generic inputs handled by IO processor plugins."""

84 data: Any

@@ -197,7 +167,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):

167 mm_processor_kwargs: NotRequired[dict[str, Any]]

168

169

200 -PromptType: TypeAlias = SingletonPrompt | ExplicitEncoderDecoderPrompt

170 +PromptType: TypeAlias = SingletonPrompt | ExplicitEncoderDecoderPrompt[Any, Any]

171 """

172 Set of possible schemas for an LLM input, including

173 both decoder-only and encoder/decoder input types:

vllm/inputs/parse.py

+9 -47

@@ -1,11 +1,10 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from collections.abc import Sequence

4 -from typing import TYPE_CHECKING, Literal, NamedTuple, TypeAlias, TypedDict, cast

3 +from typing import TYPE_CHECKING, Literal, NamedTuple, TypeAlias, TypedDict

5 from typing_extensions import TypeIs

8 -from vllm.utils.collection_utils import is_list_of

7 +from vllm.utils import length_from_prompt_token_ids_or_embeds

9 from .data import (

10 EmbedsPrompt,

@@ -22,50 +21,6 @@ if TYPE_CHECKING:

21 import torch

25 -def parse_raw_prompts(

26 - prompt: str | list[str] | list[int] | list[list[int]],

27 -) -> Sequence[TextPrompt] | Sequence[TokensPrompt]:

28 - if isinstance(prompt, str):

29 - # case 1: a string

30 - return [TextPrompt(prompt=prompt)]

31 -

32 - if isinstance(prompt, list):

33 - if len(prompt) == 0:

34 - raise ValueError("please provide at least one prompt")

35 -

36 - # case 2: array of strings

37 - if is_list_of(prompt, str):

38 - prompt = cast(list[str], prompt)

39 - return [TextPrompt(prompt=elem) for elem in prompt]

40 -

41 - # case 3: array of tokens

42 - if is_list_of(prompt, int):

43 - prompt = cast(list[int], prompt)

44 - return [TokensPrompt(prompt_token_ids=prompt)]

45 -

46 - # case 4: array of token arrays

47 - if is_list_of(prompt, list):

48 - if len(prompt) == 1 and isinstance(prompt[0], list) and len(prompt[0]) == 0:

49 - raise ValueError("please provide at least one prompt")

50 - for elem in prompt:

51 - if not isinstance(elem, list):

52 - raise TypeError(

53 - "prompt must be a list of lists, but found a non-list element."

54 - )

55 - if not is_list_of(elem, int):

56 - raise TypeError(

57 - "Nested lists of tokens must contain only integers."

58 - )

59 -

60 - prompt = cast(list[list[int]], prompt)

61 - return [TokensPrompt(prompt_token_ids=elem) for elem in prompt]

62 -

63 - raise TypeError(

64 - "prompt must be a string, array of strings, "

65 - "array of tokens, or array of token arrays"

66 - )

67 -

68 -

24 class ParsedStrPrompt(TypedDict):

25 type: Literal["str"]

26 content: str

@@ -145,3 +100,10 @@ def get_prompt_components(prompt: PromptType) -> PromptComponents:

100 token_ids=prompt.get("prompt_token_ids"), # type: ignore[arg-type]

101 embeds=prompt.get("prompt_embeds"),

102 )

103 +

104 +

105 +def get_prompt_len(prompt: TokensPrompt | EmbedsPrompt):

106 + return length_from_prompt_token_ids_or_embeds(

107 + prompt.get("prompt_token_ids"), # type: ignore[arg-type]

108 + prompt.get("prompt_embeds"), # type: ignore[arg-type]

109 + )

vllm/model_executor/models/ultravox.py

@@ -209,6 +209,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo

209 item_processor_data = dict(**mm_data, audios=audios)

210

211 # some tokenizer kwargs are incompatible with UltravoxProcessor

212 + tok_kwargs.pop("add_special_tokens", None)

213 tok_kwargs.pop("padding", None)

214 tok_kwargs.pop("truncation", None)

215

vllm/renderers/__init__.py

+9 -1

@@ -1,7 +1,15 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 +from .params import ChatParams, TokenizeParams, merge_kwargs

5 from .protocol import RendererLike

6 from .registry import RendererRegistry, renderer_from_config

7 -__all__ = ["RendererLike", "RendererRegistry", "renderer_from_config"]

8 +__all__ = [

9 + "RendererLike",

10 + "RendererRegistry",

11 + "renderer_from_config",

12 + "ChatParams",

13 + "TokenizeParams",

14 + "merge_kwargs",

15 +]

vllm/renderers/deepseek_v32.py

+12 -19

@@ -9,11 +9,12 @@ from vllm.entrypoints.chat_utils import (

9 parse_chat_messages,

10 parse_chat_messages_async,

11 )

12 -from vllm.inputs import TextPrompt, TokensPrompt

12 +from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

13 from vllm.logger import init_logger

14 from vllm.tokenizers import cached_get_tokenizer

15 from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer

17 +from .params import ChatParams

18 from .protocol import RendererLike

20 logger = init_logger(__name__)

@@ -61,8 +62,8 @@ class DeepseekV32Renderer(RendererLike):

62 def render_messages(

63 self,

64 messages: list[ChatCompletionMessageParam],

64 - **kwargs,

65 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

65 + params: ChatParams,

66 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

67 tokenizer = self.get_tokenizer()

68 conversation, mm_data, mm_uuids = parse_chat_messages(

69 messages,

@@ -74,26 +75,22 @@ class DeepseekV32Renderer(RendererLike):

75 prompt_raw = tokenizer.apply_chat_template(

76 conversation=conversation,

77 messages=messages,

77 - **kwargs,

78 + **params.get_apply_chat_template_kwargs(),

79 )

80 - prompt = (

81 - TextPrompt(prompt=prompt_raw)

82 - if isinstance(prompt_raw, str)

83 - else TokensPrompt(prompt_token_ids=prompt_raw)

84 - )

81 + prompt = self.render_completion(prompt_raw)

82 if mm_data is not None:

83 prompt["multi_modal_data"] = mm_data

84 if mm_uuids is not None:

85 prompt["multi_modal_uuids"] = mm_uuids

90 - return conversation, prompt # type: ignore[return-value]

87 + return conversation, prompt

89 async def render_messages_async(

90 self,

91 messages: list[ChatCompletionMessageParam],

95 - **kwargs,

96 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

92 + params: ChatParams,

93 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

94 tokenizer = self.get_tokenizer()

95 conversation, mm_data, mm_uuids = await parse_chat_messages_async(

96 messages,

@@ -105,17 +102,13 @@ class DeepseekV32Renderer(RendererLike):

102 prompt_raw = tokenizer.apply_chat_template(

103 conversation=conversation,

104 messages=messages,

108 - **kwargs,

105 + **params.get_apply_chat_template_kwargs(),

106 )

107

111 - prompt = (

112 - TextPrompt(prompt=prompt_raw)

113 - if isinstance(prompt_raw, str)

114 - else TokensPrompt(prompt_token_ids=prompt_raw)

115 - )

108 + prompt = self.render_completion(prompt_raw)

109 if mm_data is not None:

110 prompt["multi_modal_data"] = mm_data

111 if mm_uuids is not None:

112 prompt["multi_modal_uuids"] = mm_uuids

113

121 - return conversation, prompt # type: ignore[return-value]

114 + return conversation, prompt

vllm/renderers/embed_utils.py new

+44

@@ -0,0 +1,44 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from io import BytesIO

4 +from typing import TYPE_CHECKING

5 +

6 +import pybase64

7 +import torch

8 +

9 +from vllm.exceptions import VLLMValidationError

10 +

11 +if TYPE_CHECKING:

12 + from vllm.config import ModelConfig

13 +

14 +

15 +def safe_load_prompt_embeds(

16 + model_config: "ModelConfig",

17 + embed: bytes,

18 +) -> torch.Tensor:

19 + if not model_config.enable_prompt_embeds:

20 + raise VLLMValidationError(

21 + "You must set `--enable-prompt-embeds` to input `prompt_embeds`.",

22 + parameter="prompt_embeds",

23 + )

24 +

25 + # Enable sparse tensor integrity checks to prevent out-of-bounds

26 + # writes from maliciously crafted tensors

27 + with torch.sparse.check_sparse_tensor_invariants():

28 + tensor = torch.load(

29 + BytesIO(pybase64.b64decode(embed, validate=True)),

30 + weights_only=True,

31 + map_location=torch.device("cpu"),

32 + )

33 + assert isinstance(tensor, torch.Tensor) and tensor.dtype in (

34 + torch.float32,

35 + torch.bfloat16,

36 + torch.float16,

37 + )

38 + tensor = tensor.to_dense()

39 +

40 + if tensor.dim() > 2:

41 + tensor = tensor.squeeze(0)

42 + assert tensor.dim() == 2

43 +

44 + return tensor

vllm/renderers/grok2.py

+12 -19

@@ -9,11 +9,12 @@ from vllm.entrypoints.chat_utils import (

9 parse_chat_messages,

10 parse_chat_messages_async,

11 )

12 -from vllm.inputs import TextPrompt, TokensPrompt

12 +from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

13 from vllm.logger import init_logger

14 from vllm.tokenizers import cached_get_tokenizer

15 from vllm.tokenizers.grok2 import Grok2Tokenizer

17 +from .params import ChatParams

18 from .protocol import RendererLike

20 logger = init_logger(__name__)

@@ -61,8 +62,8 @@ class Grok2Renderer(RendererLike):

62 def render_messages(

63 self,

64 messages: list[ChatCompletionMessageParam],

64 - **kwargs,

65 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

65 + params: ChatParams,

66 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

67 tokenizer = self.get_tokenizer()

68 conversation, mm_data, mm_uuids = parse_chat_messages(

69 messages,

@@ -74,26 +75,22 @@ class Grok2Renderer(RendererLike):

75 prompt_raw = tokenizer.apply_chat_template(

76 conversation=conversation,

77 messages=messages,

77 - **kwargs,

78 + **params.get_apply_chat_template_kwargs(),

79 )

80 - prompt = (

81 - TextPrompt(prompt=prompt_raw)

82 - if isinstance(prompt_raw, str)

83 - else TokensPrompt(prompt_token_ids=prompt_raw)

84 - )

81 + prompt = self.render_completion(prompt_raw)

82 if mm_data is not None:

83 prompt["multi_modal_data"] = mm_data

84 if mm_uuids is not None:

85 prompt["multi_modal_uuids"] = mm_uuids

90 - return conversation, prompt # type: ignore[return-value]

87 + return conversation, prompt

89 async def render_messages_async(

90 self,

91 messages: list[ChatCompletionMessageParam],

95 - **kwargs,

96 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

92 + params: ChatParams,

93 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

94 tokenizer = self.get_tokenizer()

95 conversation, mm_data, mm_uuids = await parse_chat_messages_async(

96 messages,

@@ -105,17 +102,13 @@ class Grok2Renderer(RendererLike):

102 prompt_raw = tokenizer.apply_chat_template(

103 conversation=conversation,

104 messages=messages,

108 - **kwargs,

105 + **params.get_apply_chat_template_kwargs(),

106 )

107

111 - prompt = (

112 - TextPrompt(prompt=prompt_raw)

113 - if isinstance(prompt_raw, str)

114 - else TokensPrompt(prompt_token_ids=prompt_raw)

115 - )

108 + prompt = self.render_completion(prompt_raw)

109 if mm_data is not None:

110 prompt["multi_modal_data"] = mm_data

111 if mm_uuids is not None:

112 prompt["multi_modal_uuids"] = mm_uuids

113

121 - return conversation, prompt # type: ignore[return-value]

114 + return conversation, prompt

vllm/renderers/hf.py

+20 -31

@@ -25,7 +25,7 @@ from vllm.entrypoints.chat_utils import (

25 parse_chat_messages,

26 parse_chat_messages_async,

27 )

28 -from vllm.inputs import TextPrompt, TokensPrompt

28 +from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

29 from vllm.logger import init_logger

30 from vllm.tokenizers import cached_get_tokenizer

31 from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer

@@ -33,6 +33,7 @@ from vllm.transformers_utils.chat_templates import get_chat_template_fallback_pa

33 from vllm.transformers_utils.processor import cached_get_processor

34 from vllm.utils.func_utils import supports_kw

36 +from .params import ChatParams

37 from .protocol import RendererLike

39 if TYPE_CHECKING:

@@ -632,9 +633,8 @@ class HfRenderer(RendererLike):

633 def render_messages(

634 self,

635 messages: list[ChatCompletionMessageParam],

635 - chat_template_content_format: ChatTemplateContentFormatOption = "auto",

636 - **kwargs,

637 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

636 + params: ChatParams,

637 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

638 model_config = self.config

639 tokenizer = self.get_tokenizer()

640

@@ -642,9 +642,9 @@ class HfRenderer(RendererLike):

642 messages,

643 model_config,

644 content_format=resolve_chat_template_content_format(

645 - chat_template=kwargs.get("chat_template"),

646 - tools=kwargs.get("tools"),

647 - given_format=chat_template_content_format,

645 + chat_template=params.chat_template,

646 + tools=params.chat_template_kwargs.get("tools"),

647 + given_format=params.chat_template_content_format,

648 tokenizer=tokenizer,

649 model_config=model_config,

650 ),

@@ -654,7 +654,7 @@ class HfRenderer(RendererLike):

654 model_config,

655 tokenizer,

656 conversation,

657 - **kwargs,

657 + **params.get_apply_chat_template_kwargs(),

658 )

659

660 # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5

@@ -666,7 +666,7 @@ class HfRenderer(RendererLike):

666 ):

667 mm_uuids = rebuild_mm_uuids_from_mm_data(mm_uuids, mm_data)

668

669 - # get video placehoder, replace it with runtime video-chunk prompts

669 + # get video placeholder, replace it with runtime video-chunk prompts

670 video_placeholder = getattr(

671 model_config.hf_config, "video_placeholder", None

672 )

@@ -676,24 +676,19 @@ class HfRenderer(RendererLike):

676 video_placeholder,

677 )

678

679 - prompt = (

680 - TextPrompt(prompt=prompt_raw)

681 - if isinstance(prompt_raw, str)

682 - else TokensPrompt(prompt_token_ids=prompt_raw)

683 - )

679 + prompt = self.render_completion(prompt_raw)

680 if mm_data is not None:

681 prompt["multi_modal_data"] = mm_data

682 if mm_uuids is not None:

683 prompt["multi_modal_uuids"] = mm_uuids

684

689 - return conversation, prompt # type: ignore[return-value]

685 + return conversation, prompt

686

687 async def render_messages_async(

688 self,

689 messages: list[ChatCompletionMessageParam],

694 - chat_template_content_format: ChatTemplateContentFormatOption = "auto",

695 - **kwargs,

696 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

690 + params: ChatParams,

691 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

692 model_config = self.config

693 tokenizer = self.get_tokenizer()

694

@@ -701,9 +696,9 @@ class HfRenderer(RendererLike):

696 messages,

697 model_config,

698 content_format=resolve_chat_template_content_format(

704 - chat_template=kwargs.get("chat_template"),

705 - tools=kwargs.get("tools"),

706 - given_format=chat_template_content_format,

699 + chat_template=params.chat_template,

700 + tools=params.chat_template_kwargs.get("tools"),

701 + given_format=params.chat_template_content_format,

702 tokenizer=tokenizer,

703 model_config=model_config,

704 ),

@@ -713,7 +708,7 @@ class HfRenderer(RendererLike):

708 model_config,

709 tokenizer,

710 conversation,

716 - **kwargs,

711 + **params.get_apply_chat_template_kwargs(),

712 )

713

714 # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5

@@ -723,9 +718,7 @@ class HfRenderer(RendererLike):

718 and mm_uuids is not None

719 and mm_data is not None

720 ):

726 - mm_uuids = rebuild_mm_uuids_from_mm_data(mm_uuids, mm_data)

727 -

728 - # get video placehoder, replace it with runtime video-chunk prompts

721 + # get video placeholder, replace it with runtime video-chunk prompts

722 video_placeholder = getattr(

723 model_config.hf_config, "video_placeholder", None

724 )

@@ -735,14 +728,10 @@ class HfRenderer(RendererLike):

728 video_placeholder,

729 )

730

738 - prompt = (

739 - TextPrompt(prompt=prompt_raw)

740 - if isinstance(prompt_raw, str)

741 - else TokensPrompt(prompt_token_ids=prompt_raw)

742 - )

731 + prompt = self.render_completion(prompt_raw)

732 if mm_data is not None:

733 prompt["multi_modal_data"] = mm_data

734 if mm_uuids is not None:

735 prompt["multi_modal_uuids"] = mm_uuids

736

748 - return conversation, prompt # type: ignore[return-value]

737 + return conversation, prompt

vllm/renderers/mistral.py

+18 -19

@@ -10,12 +10,13 @@ from vllm.entrypoints.chat_utils import (

10 parse_chat_messages,

11 parse_chat_messages_async,

12 )

13 -from vllm.inputs import TextPrompt, TokensPrompt

13 +from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

14 from vllm.logger import init_logger

15 from vllm.tokenizers import cached_get_tokenizer

16 from vllm.tokenizers.mistral import MistralTokenizer

17 from vllm.utils.async_utils import make_async

19 +from .params import ChatParams

20 from .protocol import RendererLike

22 logger = init_logger(__name__)

@@ -95,8 +96,8 @@ class MistralRenderer(RendererLike):

96 def render_messages(

97 self,

98 messages: list[ChatCompletionMessageParam],

98 - **kwargs,

99 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

99 + params: ChatParams,

100 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

101 tokenizer = self.get_tokenizer()

102 conversation, mm_data, mm_uuids = parse_chat_messages(

103 messages,

@@ -104,25 +105,25 @@ class MistralRenderer(RendererLike):

105 content_format="string",

106 )

107

107 - prompt_raw = safe_apply_chat_template(tokenizer, messages, **kwargs)

108 -

109 - prompt = (

110 - TextPrompt(prompt=prompt_raw)

111 - if isinstance(prompt_raw, str)

112 - else TokensPrompt(prompt_token_ids=prompt_raw)

108 + prompt_raw = safe_apply_chat_template(

109 + tokenizer,

110 + messages,

111 + **params.get_apply_chat_template_kwargs(),

112 )

113 +

114 + prompt = self.render_completion(prompt_raw)

115 if mm_data is not None:

116 prompt["multi_modal_data"] = mm_data

117 if mm_uuids is not None:

118 prompt["multi_modal_uuids"] = mm_uuids

119

119 - return conversation, prompt # type: ignore[return-value]

120 + return conversation, prompt

121

122 async def render_messages_async(

123 self,

124 messages: list[ChatCompletionMessageParam],

124 - **kwargs,

125 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

125 + params: ChatParams,

126 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

127 tokenizer = self.get_tokenizer()

128 conversation, mm_data, mm_uuids = await parse_chat_messages_async(

129 messages,

@@ -131,17 +132,15 @@ class MistralRenderer(RendererLike):

132 )

133

134 prompt_raw = await self._apply_chat_template_async(

134 - tokenizer, messages, **kwargs

135 + tokenizer,

136 + messages,

137 + **params.get_apply_chat_template_kwargs(),

138 )

139

137 - prompt = (

138 - TextPrompt(prompt=prompt_raw)

139 - if isinstance(prompt_raw, str)

140 - else TokensPrompt(prompt_token_ids=prompt_raw)

141 - )

140 + prompt = self.render_completion(prompt_raw)

141 if mm_data is not None:

142 prompt["multi_modal_data"] = mm_data

143 if mm_uuids is not None:

144 prompt["multi_modal_uuids"] = mm_uuids

145

147 - return conversation, prompt # type: ignore[return-value]

146 + return conversation, prompt

vllm/renderers/params.py new

+351

@@ -0,0 +1,351 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from dataclasses import dataclass, field

4 +from typing import TYPE_CHECKING, Any, TypeVar

5 +

6 +from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption

7 +from vllm.exceptions import VLLMValidationError

8 +from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

9 +from vllm.logger import init_logger

10 +from vllm.tokenizers import TokenizerLike

11 +from vllm.utils.import_utils import LazyLoader

12 +

13 +if TYPE_CHECKING:

14 + import torch

15 +else:

16 + torch = LazyLoader("torch", globals(), "torch")

17 +

18 +logger = init_logger(__name__)

19 +

20 +

21 +_S = TypeVar("_S", list[int], "torch.Tensor")

22 +

23 +

24 +def merge_kwargs(

25 + defaults: dict[str, Any] | None,

26 + overrides: dict[str, Any] | None,

27 + /,

28 + *,

29 + unset_values: tuple[object, ...] = (None, "auto"),

30 +) -> dict[str, Any]:

31 + if defaults is None:

32 + defaults = {}

33 + if overrides is None:

34 + overrides = {}

35 +

36 + return defaults | {k: v for k, v in overrides.items() if v not in unset_values}

37 +

38 +

39 +@dataclass(frozen=True)

40 +class ChatParams:

41 + """Configuration to control how to parse chat messages."""

42 +

43 + chat_template: str | None = None

44 + """The chat template to apply."""

45 +

46 + chat_template_content_format: ChatTemplateContentFormatOption = "auto"

47 + """The format of the chat template."""

48 +

49 + chat_template_kwargs: dict[str, Any] = field(default_factory=dict)

50 + """The kwargs to pass to the chat template."""

51 +

52 + def with_defaults(self, default_chat_template_kwargs: dict[str, Any] | None):

53 + if not default_chat_template_kwargs:

54 + return self

55 +

56 + return ChatParams(

57 + chat_template=self.chat_template,

58 + chat_template_content_format=self.chat_template_content_format,

59 + chat_template_kwargs=merge_kwargs(

60 + default_chat_template_kwargs,

61 + self.chat_template_kwargs,

62 + ),

63 + )

64 +

65 + def get_apply_chat_template_kwargs(self) -> dict[str, Any]:

66 + """The arguments to pass to `tokenizer.apply_chat_template`."""

67 + return merge_kwargs(

68 + self.chat_template_kwargs,

69 + dict(chat_template=self.chat_template),

70 + )

71 +

72 +

73 +@dataclass(frozen=True)

74 +class TokenizeParams:

75 + """Configuration to control how prompts are tokenized."""

76 +

77 + max_total_tokens: int | None

78 + """

79 + Maximum allowed number of input + output tokens.

80 +

81 + Usually, this refers to the model's context length.

82 + """

83 +

84 + max_output_tokens: int = 0

85 + """Maximum requested number of output tokens."""

86 +

87 + pad_prompt_tokens: int | None = None

88 + """

89 + Number of tokens to pad to:

90 + - `None` means no padding.

91 + - `-1` maps to `max_input_tokens`.

92 + """

93 +

94 + truncate_prompt_tokens: int | None = None

95 + """

96 + Number of tokens to keep:

97 + - `None` means no truncation.

98 + - `-1` maps to `max_input_tokens`.

99 + """

100 +

101 + do_lower_case: bool = False

102 + """Whether to normalize text to lower case before tokenization."""

103 +

104 + add_special_tokens: bool = True

105 + """Whether to add special tokens."""

106 +

107 + needs_detokenization: bool = False

108 + """

109 + Whether the tokenized prompt needs to contain the original text.

110 +

111 + Not to be confused with `SamplingParams.detokenize` which deals

112 + with the output generated by the model.

113 + """

114 +

115 + max_total_tokens_param: str = "max_total_tokens"

116 + """Override this to edit the message for validation errors."""

117 +

118 + max_output_tokens_param: str = "max_output_tokens"

119 + """Override this to edit the message for validation errors."""

120 +

121 + truncate_prompt_tokens_param: str = "truncate_prompt_tokens"

122 + """Override this to edit the message for validation errors."""

123 +

124 + @property

125 + def max_input_tokens(self) -> int | None:

126 + """Maximum allowed number of input tokens."""

127 + if self.max_total_tokens is None:

128 + return None

129 +

130 + return self.max_total_tokens - self.max_output_tokens

131 +

132 + def __post_init__(self) -> None:

133 + max_total_tokens = self.max_total_tokens

134 + max_output_tokens = self.max_output_tokens

135 + max_input_tokens = self.max_input_tokens

136 + truncate_prompt_tokens = self.truncate_prompt_tokens

137 +

138 + if (

139 + max_output_tokens is not None

140 + and max_total_tokens is not None

141 + and max_output_tokens > max_total_tokens

142 + ):

143 + raise VLLMValidationError(

144 + f"{self.max_output_tokens_param}={max_output_tokens}"

145 + f"cannot be greater than "

146 + f"{self.max_total_tokens_param}={max_total_tokens=}. "

147 + f"Please request fewer output tokens.",

148 + parameter=self.max_output_tokens_param,

149 + value=max_output_tokens,

150 + )

151 +

152 + if (

153 + max_input_tokens is not None

154 + and truncate_prompt_tokens is not None

155 + and truncate_prompt_tokens > max_input_tokens

156 + ):

157 + raise VLLMValidationError(

158 + f"{self.truncate_prompt_tokens_param}={truncate_prompt_tokens} "

159 + f"cannot be greater than {self.max_total_tokens_param} - "

160 + f"{self.max_output_tokens_param} = {max_input_tokens}. "

161 + f"Please request a smaller truncation size.",

162 + parameter=self.truncate_prompt_tokens_param,

163 + value=truncate_prompt_tokens,

164 + )

165 +

166 + def with_kwargs(self, tokenization_kwargs: dict[str, Any] | None):

167 + if tokenization_kwargs is None:

168 + tokenization_kwargs = {}

169 +

170 + max_length = tokenization_kwargs.pop("max_length", self.max_input_tokens)

171 + pad_prompt_tokens = tokenization_kwargs.pop(

172 + "pad_prompt_tokens", self.pad_prompt_tokens

173 + )

174 + truncate_prompt_tokens = tokenization_kwargs.pop(

175 + "truncate_prompt_tokens", self.truncate_prompt_tokens

176 + )

177 + do_lower_case = tokenization_kwargs.pop("do_lower_case", self.do_lower_case)

178 + add_special_tokens = tokenization_kwargs.pop(

179 + "add_special_tokens", self.add_special_tokens

180 + )

181 + needs_detokenization = tokenization_kwargs.pop(

182 + "needs_detokenization", self.needs_detokenization

183 + )

184 +

185 + # https://huggingface.co/docs/transformers/en/pad_truncation

186 + if padding := tokenization_kwargs.pop("padding", None):

187 + if padding == "max_length":

188 + pad_prompt_tokens = max_length

189 + elif padding in (False, "do_not_pad"):

190 + pad_prompt_tokens = None

191 + else:

192 + # To emit the below warning

193 + tokenization_kwargs["padding"] = padding

194 +

195 + if truncation := tokenization_kwargs.pop("truncation", None):

196 + if truncation in (True, "longest_first"):

197 + truncate_prompt_tokens = max_length

198 + elif truncation in (False, "do_not_truncate"):

199 + truncate_prompt_tokens = None

200 + else:

201 + # To emit the below warning

202 + tokenization_kwargs["truncation"] = truncation

203 +

204 + if tokenization_kwargs:

205 + logger.warning(

206 + "The following tokenization arguments are not supported "

207 + "by vLLM Renderer and will be ignored: %s",

208 + tokenization_kwargs,

209 + )

210 +

211 + max_total_tokens = self.max_total_tokens

212 +

213 + return TokenizeParams(

214 + max_total_tokens=max_total_tokens,

215 + max_output_tokens=(

216 + 0

217 + if max_total_tokens is None or max_length is None

218 + else max_total_tokens - max_length

219 + ),

220 + pad_prompt_tokens=pad_prompt_tokens,

221 + truncate_prompt_tokens=truncate_prompt_tokens,

222 + do_lower_case=do_lower_case,

223 + add_special_tokens=add_special_tokens,

224 + needs_detokenization=needs_detokenization,

225 + )

226 +

227 + def get_encode_kwargs(self) -> dict[str, Any]:

228 + """The arguments to pass to `tokenizer.encode`."""

229 + max_length = self.truncate_prompt_tokens

230 + if max_length is not None and max_length < 0:

231 + max_length = self.max_input_tokens

232 +

233 + return dict(

234 + truncation=self.truncate_prompt_tokens is not None,

235 + max_length=max_length,

236 + add_special_tokens=self.add_special_tokens,

237 + )

238 +

239 + def _apply_lowercase(self, tokenizer: TokenizerLike | None, text: str) -> str:

240 + if self.do_lower_case:

241 + text = text.lower()

242 +

243 + return text

244 +

245 + def _validate_text(self, tokenizer: TokenizerLike | None, text: str) -> str:

246 + """Apply all validators to prompt text."""

247 + # TODO: Implement https://github.com/vllm-project/vllm/pull/31366

248 + for validator in (self._apply_lowercase,):

249 + text = validator(tokenizer, text)

250 +

251 + return text

252 +

253 + def apply_pre_tokenization(

254 + self,

255 + tokenizer: TokenizerLike | None,

256 + prompt: TextPrompt,

257 + ) -> TextPrompt:

258 + """

259 + Ensure that the prompt meets the requirements set out by this config.

260 + If that is not possible, raise a `VLLMValidationError`.

261 +

262 + This method is run before tokenization occurs.

263 + """

264 + prompt["prompt"] = self._validate_text(tokenizer, prompt["prompt"])

265 +

266 + return prompt

267 +

268 + def _apply_padding(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:

269 + """Apply padding to a token sequence."""

270 + pad_length = self.pad_prompt_tokens

271 + if pad_length is not None and pad_length < 0:

272 + pad_length = self.max_input_tokens

273 +

274 + if pad_length is None or pad_length <= len(tokens):

275 + return tokens

276 +

277 + if tokenizer is None:

278 + raise ValueError("Cannot pad tokens when `skip_tokenizer_init=True`")

279 + if not isinstance(tokens, list):

280 + raise ValueError("Cannot pad tokens for embedding inputs")

281 +

282 + return tokens + [tokenizer.pad_token_id] * (pad_length - len(tokens))

283 +

284 + def _apply_truncation(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:

285 + """Apply truncation to a token sequence."""

286 + max_length = self.truncate_prompt_tokens

287 + if max_length is not None and max_length < 0:

288 + max_length = self.max_input_tokens

289 +

290 + if max_length is None or max_length >= len(tokens):

291 + return tokens

292 + if max_length == 0:

293 + return tokens[:0]

294 +

295 + if getattr(tokenizer, "truncation_side", "left") == "left":

296 + return tokens[-max_length:]

297 +

298 + return tokens[:max_length]

299 +

300 + def _apply_length_check(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:

301 + """Apply length checks to a token sequence."""

302 + max_input_tokens = self.max_input_tokens

303 +

304 + if max_input_tokens is not None and len(tokens) > max_input_tokens:

305 + raise VLLMValidationError(

306 + f"You passed {len(tokens)} input tokens and "

307 + f"requested {self.max_output_tokens} output tokens. "

308 + f"However, the model's context length is only "

309 + f"{self.max_total_tokens}, resulting in a maximum "

310 + f"input length of {max_input_tokens}. "

311 + f"Please reduce the length of the input messages.",

312 + parameter="input_tokens",

313 + value=len(tokens),

314 + )

315 +

316 + return tokens

317 +

318 + def _validate_tokens(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:

319 + """Apply all validators to a token sequence."""

320 + for validator in (

321 + self._apply_padding,

322 + self._apply_truncation,

323 + self._apply_length_check,

324 + ):

325 + tokens = validator(tokenizer, tokens)

326 +

327 + return tokens

328 +

329 + def apply_post_tokenization(

330 + self,

331 + tokenizer: TokenizerLike | None,

332 + prompt: TokensPrompt | EmbedsPrompt,

333 + ) -> TokensPrompt | EmbedsPrompt:

334 + """

335 + Ensure that the prompt meets the requirements set out by this config.

336 + If that is not possible, raise a `VLLMValidationError`.

337 +

338 + This method is run after tokenization occurs.

339 + """

340 + if "prompt_token_ids" in prompt:

341 + prompt["prompt_token_ids"] = self._validate_tokens( # type: ignore[typeddict-unknown-key]

342 + tokenizer,

343 + prompt["prompt_token_ids"], # type: ignore[typeddict-item]

344 + )

345 + if "prompt_embeds" in prompt:

346 + prompt["prompt_embeds"] = self._validate_tokens( # type: ignore[typeddict-unknown-key]

347 + tokenizer,

348 + prompt["prompt_embeds"], # type: ignore[typeddict-item]

349 + )

350 +

351 + return prompt

vllm/renderers/protocol.py

+146 -6

@@ -1,9 +1,15 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import asyncio

4 from typing import TYPE_CHECKING, Any, Protocol

5 -from vllm.inputs import TextPrompt, TokensPrompt

6 +from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

7 from vllm.tokenizers import TokenizerLike

8 +from vllm.utils.async_utils import AsyncMicrobatchTokenizer

9 +from vllm.utils.collection_utils import is_list_of

10 +

11 +from .embed_utils import safe_load_prompt_embeds

12 +from .params import ChatParams, TokenizeParams

14 if TYPE_CHECKING:

15 from vllm.config import ModelConfig

@@ -14,6 +20,9 @@ if TYPE_CHECKING:

22 class RendererLike(Protocol):

23 + config: "ModelConfig"

24 + _async_tokenizer: AsyncMicrobatchTokenizer

25 +

26 @classmethod

27 def from_config(

28 cls,

@@ -33,16 +42,147 @@ class RendererLike(Protocol):

43 return tokenizer

45 + def get_async_tokenizer(self) -> AsyncMicrobatchTokenizer:

46 + # Lazy initialization since offline LLM doesn't use async

47 + if not hasattr(self, "_async_tokenizer"):

48 + self._async_tokenizer = AsyncMicrobatchTokenizer(self.get_tokenizer())

49 +

50 + return self._async_tokenizer

51 +

52 + # Step 1: Convert raw inputs to prompts

53 + def render_completion(

54 + self,

55 + prompt_raw: str | list[int] | bytes,

56 + ) -> TextPrompt | TokensPrompt | EmbedsPrompt:

57 + error_msg = "Each prompt must be a string or an array of tokens"

58 +

59 + if isinstance(prompt_raw, str):

60 + return TextPrompt(prompt=prompt_raw)

61 +

62 + if isinstance(prompt_raw, list):

63 + if not is_list_of(prompt_raw, int):

64 + raise TypeError(error_msg)

65 +

66 + return TokensPrompt(prompt_token_ids=prompt_raw)

67 +

68 + if isinstance(prompt_raw, bytes):

69 + embeds = safe_load_prompt_embeds(self.config, prompt_raw)

70 + return EmbedsPrompt(prompt_embeds=embeds)

71 +

72 + raise TypeError(error_msg)

73 +

74 + def render_completions(

75 + self,

76 + prompt_input: str | list[str] | list[int] | list[list[int]] | None = None,

77 + prompt_embeds: bytes | list[bytes] | None = None,

78 + ) -> list[TextPrompt | TokensPrompt | EmbedsPrompt]:

79 + prompts_raw = list[str | list[int] | bytes]()

80 +

81 + if prompt_embeds is not None: # embeds take higher priority

82 + if isinstance(prompt_embeds, bytes):

83 + prompts_raw.append(prompt_embeds)

84 + else:

85 + prompts_raw.extend(prompt_embeds)

86 +

87 + if prompt_input is not None:

88 + if isinstance(prompt_input, str) or (

89 + len(prompt_input) > 0 and is_list_of(prompt_input, int)

90 + ):

91 + prompts_raw.append(prompt_input) # type: ignore[arg-type]

92 + else:

93 + prompts_raw.extend(prompt_input) # type: ignore[arg-type]

94 +

95 + if len(prompts_raw) == 0:

96 + raise ValueError("You must pass at least one prompt")

97 +

98 + return [self.render_completion(prompt) for prompt in prompts_raw]

99 +

100 + async def render_completions_async(

101 + self,

102 + prompt_input: str | list[str] | list[int] | list[list[int]] | None = None,

103 + prompt_embeds: bytes | list[bytes] | None = None,

104 + ) -> list[TextPrompt | TokensPrompt | EmbedsPrompt]:

105 + return self.render_completions(prompt_input, prompt_embeds)

106 +

107 def render_messages(

108 self,

109 messages: list["ChatCompletionMessageParam"],

39 - **kwargs,

40 - ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt]:

110 + params: ChatParams,

111 + ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt | EmbedsPrompt]:

112 raise NotImplementedError

113

114 async def render_messages_async(

115 self,

116 messages: list["ChatCompletionMessageParam"],

46 - **kwargs,

47 - ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt]:

48 - return self.render_messages(messages, **kwargs)

117 + params: ChatParams,

118 + ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt | EmbedsPrompt]:

119 + return self.render_messages(messages, params)

120 +

121 + # Step 2: Tokenize prompts if necessary

122 + def tokenize_prompt(

123 + self,

124 + prompt: TextPrompt | TokensPrompt | EmbedsPrompt,

125 + params: TokenizeParams,

126 + ) -> TokensPrompt | EmbedsPrompt:

127 + if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

128 + prompt = params.apply_pre_tokenization(self.tokenizer, prompt)

129 +

130 + tokenizer = self.get_tokenizer()

131 + prompt_token_ids = tokenizer.encode(

132 + prompt["prompt"],

133 + **params.get_encode_kwargs(),

134 + )

135 +

136 + prompt = TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)

137 +

138 + if params.needs_detokenization and "prompt" not in prompt:

139 + if "prompt_token_ids" not in prompt:

140 + raise RuntimeError("Cannot run detokenization on embeddings")

141 +

142 + tokenizer = self.get_tokenizer()

143 + prompt_text = tokenizer.decode(prompt["prompt_token_ids"]) # type: ignore[typeddict-item]

144 + prompt["prompt"] = prompt_text # type: ignore[typeddict-unknown-key]

145 +

146 + return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

147 +

148 + def tokenize_prompts(

149 + self,

150 + prompts: list[TextPrompt | TokensPrompt | EmbedsPrompt],

151 + params: TokenizeParams,

152 + ) -> list[TokensPrompt | EmbedsPrompt]:

153 + return [self.tokenize_prompt(prompt, params) for prompt in prompts]

154 +

155 + async def tokenize_prompt_async(

156 + self,

157 + prompt: TextPrompt | TokensPrompt | EmbedsPrompt,

158 + params: TokenizeParams,

159 + ) -> TokensPrompt | EmbedsPrompt:

160 + if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

161 + prompt = params.apply_pre_tokenization(self.tokenizer, prompt)

162 +

163 + tokenizer = self.get_async_tokenizer()

164 + prompt_token_ids = await tokenizer.encode(

165 + prompt["prompt"],

166 + **params.get_encode_kwargs(),

167 + )

168 +

169 + prompt = TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)

170 +

171 + if params.needs_detokenization and "prompt" not in prompt:

172 + if "prompt_token_ids" not in prompt:

173 + raise RuntimeError("Cannot run detokenization on embeddings")

174 +

175 + tokenizer = self.get_async_tokenizer()

176 + prompt_text = await tokenizer.decode(prompt["prompt_token_ids"]) # type: ignore[typeddict-item]

177 + prompt["prompt"] = prompt_text # type: ignore[typeddict-unknown-key]

178 +

179 + return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

180 +

181 + async def tokenize_prompts_async(

182 + self,

183 + prompts: list[TextPrompt | TokensPrompt | EmbedsPrompt],

184 + params: TokenizeParams,

185 + ) -> list[TokensPrompt | EmbedsPrompt]:

186 + return await asyncio.gather(

187 + *(self.tokenize_prompt_async(prompt, params) for prompt in prompts)

188 + )

vllm/renderers/terratorch.py

+8 -7

@@ -9,10 +9,11 @@ from vllm.entrypoints.chat_utils import (

9 parse_chat_messages,

10 parse_chat_messages_async,

11 )

12 -from vllm.inputs import TextPrompt, TokensPrompt

12 +from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

13 from vllm.logger import init_logger

14 from vllm.tokenizers import TokenizerLike

16 +from .params import ChatParams

17 from .protocol import RendererLike

19 logger = init_logger(__name__)

@@ -45,8 +46,8 @@ class TerratorchRenderer(RendererLike):

46 def render_messages(

47 self,

48 messages: list[ChatCompletionMessageParam],

48 - **kwargs,

49 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

49 + params: ChatParams,

50 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

51 model_config = self.config

53 conversation, mm_data, mm_uuids = parse_chat_messages(

@@ -55,7 +56,7 @@ class TerratorchRenderer(RendererLike):

56 content_format="string",

57 )

58 - prompt = TokensPrompt(prompt_token_ids=[1])

59 + prompt = self.render_completion([1]) # Dummy token IDs

60 if mm_data is not None:

61 prompt["multi_modal_data"] = mm_data

62 if mm_uuids is not None:

@@ -66,8 +67,8 @@ class TerratorchRenderer(RendererLike):

67 async def render_messages_async(

68 self,

69 messages: list[ChatCompletionMessageParam],

69 - **kwargs,

70 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:

70 + params: ChatParams,

71 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

72 model_config = self.config

74 conversation, mm_data, mm_uuids = await parse_chat_messages_async(

@@ -76,7 +77,7 @@ class TerratorchRenderer(RendererLike):

77 content_format="string",

78 )

79 - prompt = TokensPrompt(prompt_token_ids=[1]) # Dummy token IDs

80 + prompt = self.render_completion([1]) # Dummy token IDs

81 if mm_data is not None:

82 prompt["multi_modal_data"] = mm_data

83 if mm_uuids is not None:

vllm/transformers_utils/config.py

+1 -1

@@ -834,7 +834,7 @@ def parse_pooling_type(pooling_name: str):

834 @cache

835 def get_sentence_transformer_tokenizer_config(

836 model: str | Path, revision: str | None = "main"

837 -):

837 +) -> dict[str, Any] | None:

838 """

839 Returns the tokenization configuration dictionary for a

840 given Sentence Transformer BERT model.

vllm/utils/async_utils.py

+5 -2

@@ -50,14 +50,17 @@ class AsyncMicrobatchTokenizer:

50 self._executor = ThreadPoolExecutor(max_workers=1)

52 # === Public async API ===

53 - async def __call__(self, prompt, **kwargs):

53 + async def __call__(self, prompt, **kwargs) -> BatchEncoding:

54 result_future: Future = self._loop.create_future()

55 key = self._queue_key("encode", kwargs)

56 queue = self._get_queue(self._loop, key)

57 await queue.put((prompt, kwargs, result_future))

58 return await result_future

60 - async def decode(self, token_ids, **kwargs):

60 + async def encode(self, prompt, **kwargs) -> list[int]:

61 + return (await self(prompt, **kwargs)).input_ids

62 +

63 + async def decode(self, token_ids, **kwargs) -> str:

64 result_future: Future = self._loop.create_future()

65 key = self._queue_key("decode", kwargs)

66 queue = self._get_queue(self._loop, key)

vllm/v1/engine/async_llm.py

+21 -28

@@ -16,7 +16,6 @@ from vllm import TokensPrompt

16 from vllm.config import VllmConfig

17 from vllm.engine.arg_utils import AsyncEngineArgs

18 from vllm.engine.protocol import EngineClient

19 -from vllm.entrypoints.utils import _validate_truncation_size

19 from vllm.inputs import PromptType

20 from vllm.inputs.data import StreamingInput

21 from vllm.logger import init_logger

@@ -25,7 +24,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

24 from vllm.outputs import STREAM_FINISHED, PoolingRequestOutput, RequestOutput

25 from vllm.plugins.io_processors import get_io_processor

26 from vllm.pooling_params import PoolingParams

28 -from vllm.renderers import RendererLike

27 +from vllm.renderers import RendererLike, merge_kwargs

28 from vllm.sampling_params import RequestOutputKind, SamplingParams

29 from vllm.tasks import SupportedTask

30 from vllm.tokenizers import TokenizerLike

@@ -304,13 +303,20 @@ class AsyncLLM(EngineClient):

303 "prompt logprobs"

304 )

305

307 - if tokenization_kwargs is None:

308 - tokenization_kwargs = {}

309 - _validate_truncation_size(

310 - self.model_config.max_model_len,

311 - params.truncate_prompt_tokens,

312 - tokenization_kwargs,

313 - )

306 + if params.truncate_prompt_tokens is not None:

307 + params_type = type(params).__name__

308 + warnings.warn(

309 + f"The `truncate_prompt_tokens` parameter in `{params_type}` "

310 + "is deprecated and will be removed in v0.16. "

311 + "Please pass it via `tokenization_kwargs` instead.",

312 + DeprecationWarning,

313 + stacklevel=2,

314 + )

315 +

316 + tokenization_kwargs = merge_kwargs(

317 + tokenization_kwargs,

318 + dict(truncate_prompt_tokens=params.truncate_prompt_tokens),

319 + )

320

321 if isinstance(prompt, AsyncGenerator):

322 # Streaming input case.

@@ -344,12 +350,12 @@ class AsyncLLM(EngineClient):

350 request_id,

351 prompt,

352 params,

347 - arrival_time,

348 - lora_request,

349 - tokenization_kwargs,

350 - trace_headers,

351 - priority,

352 - data_parallel_rank,

353 + arrival_time=arrival_time,

354 + lora_request=lora_request,

355 + tokenization_kwargs=tokenization_kwargs,

356 + trace_headers=trace_headers,

357 + priority=priority,

358 + data_parallel_rank=data_parallel_rank,

359 )

360 prompt_text = get_prompt_text(prompt)

361

@@ -757,7 +763,6 @@ class AsyncLLM(EngineClient):

763 lora_request: LoRARequest | None = None,

764 trace_headers: Mapping[str, str] | None = None,

765 priority: int = 0,

760 - truncate_prompt_tokens: int | None = None,

766 tokenization_kwargs: dict[str, Any] | None = None,

767 ) -> AsyncGenerator[PoolingRequestOutput, None]:

768 """

@@ -772,22 +777,10 @@ class AsyncLLM(EngineClient):

777

778 The caller of generate() iterates the returned AsyncGenerator,

779 returning the RequestOutput back to the caller.

775 -

776 - NOTE: truncate_prompt_tokens is deprecated in v0.14.

777 - TODO: Remove truncate_prompt_tokens in v0.15.

780 """

781

782 q: RequestOutputCollector | None = None

783 try:

782 - if truncate_prompt_tokens is not None:

783 - warnings.warn(

784 - "The `truncate_prompt_tokens` parameter in `AsyncLLM.encode()` "

785 - "is deprecated and will be removed in v0.15. "

786 - "Please use `pooling_params.truncate_prompt_tokens` instead.",

787 - DeprecationWarning,

788 - stacklevel=2,

789 - )

790 -

784 q = await self.add_request(

785 request_id,

786 prompt,