@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cd8b405bd0

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 17:45 UTC

[Refactor] Consolidate sequence normalization and enc-dec parsing (#33928)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 6, 2026 at 23:43 UTC cd8b405bd0c0f386ef7eedcb7d792a8bb72a8b65

38 files changed +1256 -848

tests/entrypoints/openai/test_chat_error.py

@@ -54,6 +54,7 @@ class MockModelConfig:

54 generation_config: str = "auto"

55 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

56 skip_tokenizer_init = False

57 + is_encoder_decoder: bool = False

59 def get_diff_sampling_param(self):

60 return self.diff_sampling_param or {}

tests/entrypoints/openai/test_completion_error.py

@@ -53,6 +53,7 @@ class MockModelConfig:

53 generation_config: str = "auto"

54 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

55 skip_tokenizer_init = False

56 + is_encoder_decoder: bool = False

58 def get_diff_sampling_param(self):

59 return self.diff_sampling_param or {}

tests/entrypoints/openai/test_lora_resolvers.py

@@ -52,6 +52,7 @@ class MockModelConfig:

52 encoder_config = None

53 generation_config: str = "auto"

54 skip_tokenizer_init: bool = False

55 + is_encoder_decoder: bool = False

57 def get_diff_sampling_param(self):

58 return self.diff_sampling_param or {}

tests/entrypoints/openai/test_serving_chat.py

@@ -529,6 +529,7 @@ class MockModelConfig:

529 generation_config: str = "auto"

530 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

531 skip_tokenizer_init: bool = False

532 + is_encoder_decoder: bool = False

533

534 def get_diff_sampling_param(self):

535 return self.diff_sampling_param or {}

tests/renderers/inputs/__init__.py

tests/renderers/inputs/test_preprocess.py new

+41

@@ -0,0 +1,41 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from vllm.renderers.inputs.preprocess import prompt_to_seq

4 +

5 +

6 +def test_empty_input():

7 + assert prompt_to_seq([]) == []

8 + assert prompt_to_seq([[]]) == [[]]

9 + assert prompt_to_seq([[], []]) == [[], []]

10 +

11 +

12 +def test_text_input():

13 + assert prompt_to_seq("foo") == ["foo"]

14 + assert prompt_to_seq(["foo"]) == ["foo"]

15 + assert prompt_to_seq(["foo", "bar"]) == ["foo", "bar"]

16 +

17 +

18 +def test_token_input():

19 + assert prompt_to_seq([1, 2]) == [[1, 2]]

20 + assert prompt_to_seq([[1, 2]]) == [[1, 2]]

21 + assert prompt_to_seq([[1, 2], [3, 4]]) == [[1, 2], [3, 4]]

22 +

23 +

24 +def test_text_token_input():

25 + assert prompt_to_seq([[1, 2], "foo"]) == [[1, 2], "foo"]

26 + assert prompt_to_seq(["foo", [1, 2]]) == ["foo", [1, 2]]

27 +

28 +

29 +def test_bytes_input():

30 + assert prompt_to_seq(b"foo") == [b"foo"]

31 + assert prompt_to_seq([b"foo"]) == [b"foo"]

32 + assert prompt_to_seq([b"foo", b"bar"]) == [b"foo", b"bar"]

33 +

34 +

35 +def test_dict_input():

36 + assert prompt_to_seq({"prompt": "foo"}) == [{"prompt": "foo"}]

37 + assert prompt_to_seq([{"prompt": "foo"}]) == [{"prompt": "foo"}]

38 + assert prompt_to_seq([{"prompt": "foo"}, {"prompt_token_ids": [1, 2]}]) == [

39 + {"prompt": "foo"},

40 + {"prompt_token_ids": [1, 2]},

41 + ]

tests/renderers/test_completions.py

+90 -79

@@ -2,6 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 import io

5 +from collections.abc import Sequence

6 from dataclasses import dataclass

7 from typing import Any

@@ -9,8 +10,11 @@ import pybase64

10 import pytest

11 import torch

13 +from vllm.config import ModelConfig

14 +from vllm.inputs import SingletonPrompt

15 from vllm.renderers import TokenizeParams

16 from vllm.renderers.hf import HfRenderer

17 +from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq

18 from vllm.tokenizers.registry import tokenizer_args_from_config

20 MODEL_NAME = "openai-community/gpt2"

@@ -33,6 +37,7 @@ class MockModelConfig:

37 encoder_config: dict[str, Any] | None = None

38 enable_prompt_embeds: bool = True

39 skip_tokenizer_init: bool = False

40 + is_encoder_decoder: bool = False

43 @dataclass

@@ -80,65 +85,34 @@ def _build_renderer(

85 return renderer

83 -class TestValidatePrompt:

84 - STRING_INPUTS = [

85 - "",

86 - "foo",

87 - "foo bar",

88 - "foo baz bar",

89 - "foo bar qux baz",

90 - ]

91 -

92 - TOKEN_INPUTS = [

93 - [-1],

94 - [1],

95 - [1, 2],

96 - [1, 3, 4],

97 - [1, 2, 4, 3],

88 +def _preprocess_prompt(

89 + mdoel_config: ModelConfig,

90 + prompt_or_prompts: SingletonPrompt | bytes | Sequence[SingletonPrompt | bytes],

91 +):

92 + return [

93 + (

94 + prompt

95 + if isinstance(prompt, bytes)

96 + else parse_model_prompt(mdoel_config, prompt)

97 + )

98 + for prompt in prompt_to_seq(prompt_or_prompts)

99 ]

100

100 - INPUTS_SLICES = [

101 - slice(None, None, -1),

102 - slice(None, None, 2),

103 - slice(None, None, -2),

104 - ]

101

106 - # Test that a nested mixed-type list of lists raises a TypeError.

102 +class TestValidatePrompt:

103 def test_empty_input(self):

104 renderer = _build_renderer(MockModelConfig())

105

106 with pytest.raises(ValueError, match="at least one prompt"):

111 - renderer.render_completions([])

107 + renderer.render_prompts(_preprocess_prompt(renderer.config, []))

108

109 def test_invalid_type(self):

110 renderer = _build_renderer(MockModelConfig())

111

116 - with pytest.raises(TypeError, match="string or an array of tokens"):

117 - renderer.render_completions([[1, 2], ["foo", "bar"]])

118 -

119 - @pytest.mark.parametrize("string_input", STRING_INPUTS)

120 - def test_string_consistent(self, string_input: str):

121 - renderer = _build_renderer(MockModelConfig())

122 -

123 - assert renderer.render_completions(string_input) == renderer.render_completions(

124 - [string_input]

125 - )

126 -

127 - @pytest.mark.parametrize("token_input", TOKEN_INPUTS)

128 - def test_token_consistent(self, token_input: list[int]):

129 - renderer = _build_renderer(MockModelConfig())

130 -

131 - assert renderer.render_completions(token_input) == renderer.render_completions(

132 - [token_input]

133 - )

134 -

135 - @pytest.mark.parametrize("inputs_slice", INPUTS_SLICES)

136 - def test_string_slice(self, inputs_slice: slice):

137 - renderer = _build_renderer(MockModelConfig())

138 -

139 - assert renderer.render_completions(self.STRING_INPUTS)[

140 - inputs_slice

141 - ] == renderer.render_completions(self.STRING_INPUTS[inputs_slice])

112 + with pytest.raises(TypeError, match="should be a list of integers"):

113 + renderer.render_prompts(

114 + _preprocess_prompt(renderer.config, [[1, 2], ["foo", "bar"]]) # type: ignore[arg-type]

115 + )

116

117

118 class TestRenderPrompt:

@@ -146,7 +120,7 @@ class TestRenderPrompt:

120 renderer = _build_renderer(MockModelConfig())

121

122 tokens = [101, 7592, 2088]

149 - prompts = renderer.render_completions(tokens)

123 + prompts = renderer.render_prompts(_preprocess_prompt(renderer.config, tokens))

124 results = renderer.tokenize_prompts(

125 prompts,

126 TokenizeParams(max_total_tokens=100),

@@ -159,7 +133,9 @@ class TestRenderPrompt:

133 renderer = _build_renderer(MockModelConfig())

134

135 token_lists = [[101, 7592, 2088], [102, 1234, 5678, 9012], [103, 4567]]

162 - prompts = renderer.render_completions(token_lists)

136 + prompts = renderer.render_prompts(

137 + _preprocess_prompt(renderer.config, token_lists)

138 + )

139 results = renderer.tokenize_prompts(

140 prompts,

141 TokenizeParams(max_total_tokens=100),

@@ -174,7 +150,9 @@ class TestRenderPrompt:

150 renderer = _build_renderer(MockModelConfig())

151

152 text_input = "x" * 10

177 - prompts = renderer.render_completions(text_input)

153 + prompts = renderer.render_prompts(

154 + _preprocess_prompt(renderer.config, text_input)

155 + )

156 results = renderer.tokenize_prompts(

157 prompts,

158 TokenizeParams(max_total_tokens=100),

@@ -187,7 +165,9 @@ class TestRenderPrompt:

165 renderer = _build_renderer(MockModelConfig())

166

167 text_list_input = ["x" * 10, "x" * 12, "x" * 14]

190 - prompts = renderer.render_completions(text_list_input)

168 + prompts = renderer.render_prompts(

169 + _preprocess_prompt(renderer.config, text_list_input)

170 + )

171 results = renderer.tokenize_prompts(

172 prompts,

173 TokenizeParams(max_total_tokens=100),

@@ -200,7 +180,9 @@ class TestRenderPrompt:

180 def test_zero_truncation(self):

181 renderer = _build_renderer(MockModelConfig())

182

203 - prompts = renderer.render_completions("x" * 200)

183 + prompts = renderer.render_prompts(

184 + _preprocess_prompt(renderer.config, "x" * 200)

185 + )

186 results = renderer.tokenize_prompts(

187 prompts,

188 TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=0),

@@ -212,7 +194,9 @@ class TestRenderPrompt:

194 def test_pos_truncation(self):

195 renderer = _build_renderer(MockModelConfig())

196

215 - prompts = renderer.render_completions("x" * 200)

197 + prompts = renderer.render_prompts(

198 + _preprocess_prompt(renderer.config, "x" * 200)

199 + )

200 results = renderer.tokenize_prompts(

201 prompts,

202 TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=50),

@@ -224,7 +208,9 @@ class TestRenderPrompt:

208 def test_neg_truncation(self):

209 renderer = _build_renderer(MockModelConfig())

210

227 - prompts = renderer.render_completions("x" * 200)

211 + prompts = renderer.render_prompts(

212 + _preprocess_prompt(renderer.config, "x" * 200)

213 + )

214 results = renderer.tokenize_prompts(

215 prompts,

216 TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=-1),

@@ -237,7 +223,9 @@ class TestRenderPrompt:

223 renderer = _build_renderer(MockModelConfig(), truncation_side="left")

224

225 long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens

240 - prompts = renderer.render_completions(long_tokens)

226 + prompts = renderer.render_prompts(

227 + _preprocess_prompt(renderer.config, long_tokens)

228 + )

229 results = renderer.tokenize_prompts(

230 prompts,

231 TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=5),

@@ -251,7 +239,9 @@ class TestRenderPrompt:

239 renderer = _build_renderer(MockModelConfig(), truncation_side="right")

240

241 long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens

254 - prompts = renderer.render_completions(long_tokens)

242 + prompts = renderer.render_prompts(

243 + _preprocess_prompt(renderer.config, long_tokens)

244 + )

245 results = renderer.tokenize_prompts(

246 prompts,

247 TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=5),

@@ -266,7 +256,9 @@ class TestRenderPrompt:

256

257 # Exceeds max_total_tokens and max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN

258 long_tokens = "x" * 150

269 - prompts = renderer.render_completions(long_tokens)

259 + prompts = renderer.render_prompts(

260 + _preprocess_prompt(renderer.config, long_tokens)

261 + )

262

263 with pytest.raises(

264 ValueError,

@@ -285,7 +277,9 @@ class TestRenderPrompt:

277

278 # Exceeds max_total_tokens but not max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN

279 long_tokens = "x" * 150

288 - prompts = renderer.render_completions(long_tokens)

280 + prompts = renderer.render_prompts(

281 + _preprocess_prompt(renderer.config, long_tokens)

282 + )

283

284 with pytest.raises(

285 ValueError,

@@ -304,7 +298,9 @@ class TestRenderPrompt:

298 renderer = _build_renderer(MockModelConfig())

299

300 long_tokens = list(range(150)) # Exceeds max_total_tokens=100

307 - prompts = renderer.render_completions(long_tokens)

301 + prompts = renderer.render_prompts(

302 + _preprocess_prompt(renderer.config, long_tokens)

303 + )

304

305 with pytest.raises(

306 ValueError,

@@ -318,7 +314,9 @@ class TestRenderPrompt:

314 def test_no_tokenizer_for_text(self):

315 renderer = _build_renderer(MockModelConfig(skip_tokenizer_init=True))

316

321 - prompts = renderer.render_completions("Hello world")

317 + prompts = renderer.render_prompts(

318 + _preprocess_prompt(renderer.config, "Hello world")

319 + )

320

321 with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"):

322 renderer.tokenize_prompts(

@@ -330,7 +328,7 @@ class TestRenderPrompt:

328 renderer = _build_renderer(MockModelConfig())

329

330 tokens = [1, 2, 3, 4]

333 - prompts = renderer.render_completions(tokens)

331 + prompts = renderer.render_prompts(_preprocess_prompt(renderer.config, tokens))

332 results = renderer.tokenize_prompts(

333 prompts,

334 TokenizeParams(

@@ -359,7 +357,9 @@ class TestRenderEmbedPrompt:

357 tensor_input = torch.randn(10, 768, dtype=torch.float32)

358 embed_bytes = self._create_test_embed_bytes(tensor_input)

359

362 - prompts = renderer.render_completions(prompt_embeds=embed_bytes)

360 + prompts = renderer.render_prompts(

361 + _preprocess_prompt(renderer.config, embed_bytes)

362 + )

363 results = renderer.tokenize_prompts(

364 prompts,

365 TokenizeParams(max_total_tokens=100),

@@ -377,8 +377,11 @@ class TestRenderEmbedPrompt:

377 torch.randn(12, 512, dtype=torch.float32),

378 ]

379

380 - prompts = renderer.render_completions(

381 - prompt_embeds=[self._create_test_embed_bytes(t) for t in tensor_inputs],

380 + prompts = renderer.render_prompts(

381 + _preprocess_prompt(

382 + renderer.config,

383 + [self._create_test_embed_bytes(t) for t in tensor_inputs],

384 + )

385 )

386 results = renderer.tokenize_prompts(

387 prompts,

@@ -395,8 +398,10 @@ class TestRenderEmbedPrompt:

398 # Create tensor with more tokens than truncation limit

399 tensor_input = torch.randn(20, 768, dtype=torch.float32)

400

398 - prompts = renderer.render_completions(

399 - prompt_embeds=self._create_test_embed_bytes(tensor_input),

401 + prompts = renderer.render_prompts(

402 + _preprocess_prompt(

403 + renderer.config, self._create_test_embed_bytes(tensor_input)

404 + )

405 )

406 results = renderer.tokenize_prompts(

407 prompts,

@@ -420,8 +425,10 @@ class TestRenderEmbedPrompt:

425 for dtype in dtypes:

426 tensor_input = torch.randn(5, 256, dtype=dtype)

427

423 - prompts = renderer.render_completions(

424 - prompt_embeds=self._create_test_embed_bytes(tensor_input),

428 + prompts = renderer.render_prompts(

429 + _preprocess_prompt(

430 + renderer.config, self._create_test_embed_bytes(tensor_input)

431 + )

432 )

433 results = renderer.tokenize_prompts(

434 prompts,

@@ -437,8 +444,10 @@ class TestRenderEmbedPrompt:

444 # Test tensor with batch dimension gets squeezed

445 tensor_input = torch.randn(1, 10, 768, dtype=torch.float32)

446

440 - prompts = renderer.render_completions(

441 - prompt_embeds=self._create_test_embed_bytes(tensor_input),

447 + prompts = renderer.render_prompts(

448 + _preprocess_prompt(

449 + renderer.config, self._create_test_embed_bytes(tensor_input)

450 + )

451 )

452 results = renderer.tokenize_prompts(

453 prompts,

@@ -455,9 +464,11 @@ class TestRenderEmbedPrompt:

464 text_input = "Hello world"

465 tensor_input = torch.randn(5, 256, dtype=torch.float32)

466

458 - prompts = renderer.render_completions(

459 - text_input,

460 - prompt_embeds=self._create_test_embed_bytes(tensor_input),

467 + prompts = renderer.render_prompts(

468 + _preprocess_prompt(

469 + renderer.config,

470 + [text_input, self._create_test_embed_bytes(tensor_input)],

471 + )

472 )

473 results = renderer.tokenize_prompts(

474 prompts,

@@ -465,8 +476,8 @@ class TestRenderEmbedPrompt:

476 )

477

478 assert len(results) == 2

468 - # First should be embed prompt

469 - assert torch.equal(results[0]["prompt_embeds"], tensor_input)

470 - # Second should be tokens prompt

471 - assert "prompt_token_ids" in results[1]

472 - assert len(results[1]["prompt_token_ids"]) == len(text_input)

479 + # First should be tokens prompt

480 + assert "prompt_token_ids" in results[0]

481 + assert len(results[0]["prompt_token_ids"]) == len(text_input)

482 + # Second should be embed prompt

483 + assert torch.equal(results[1]["prompt_embeds"], tensor_input)

tests/renderers/test_mistral.py

+27 -2

@@ -3,16 +3,40 @@

4 import asyncio

5 import time

6 +from dataclasses import dataclass

7 +from typing import Any

8 from unittest.mock import Mock

10 import pytest

11 from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy

11 -from vllm.config import ModelConfig

13 from vllm.renderers import ChatParams

14 from vllm.renderers.mistral import MistralRenderer, safe_apply_chat_template

15 from vllm.tokenizers.mistral import MistralTokenizer

17 +MODEL_NAME = "mistralai/Mistral-7B-Instruct-v0.3"

18 +

19 +

20 +@dataclass

21 +class MockHFConfig:

22 + model_type: str = "any"

23 +

24 +

25 +@dataclass

26 +class MockModelConfig:

27 + runner_type = "generate"

28 + model: str = MODEL_NAME

29 + tokenizer: str = MODEL_NAME

30 + trust_remote_code: bool = False

31 + max_model_len: int = 100

32 + tokenizer_revision = None

33 + tokenizer_mode = "mistral"

34 + hf_config = MockHFConfig()

35 + encoder_config: dict[str, Any] | None = None

36 + enable_prompt_embeds: bool = True

37 + skip_tokenizer_init: bool = False

38 + is_encoder_decoder: bool = False

39 +

41 @pytest.mark.asyncio

42 async def test_async_mistral_tokenizer_does_not_block_event_loop():

@@ -23,9 +47,10 @@ async def test_async_mistral_tokenizer_does_not_block_event_loop():

47 time.sleep(2)

48 return expected_tokens

50 + mock_model_config = MockModelConfig(skip_tokenizer_init=True)

51 mock_tokenizer = Mock(spec=MistralTokenizer)

52 mock_tokenizer.apply_chat_template = mocked_apply_chat_template

28 - mock_renderer = MistralRenderer(Mock(spec=ModelConfig), tokenizer_kwargs={})

53 + mock_renderer = MistralRenderer(mock_model_config, tokenizer_kwargs={})

54 mock_renderer._tokenizer = mock_tokenizer

56 task = mock_renderer.render_messages_async([], ChatParams())

tests/test_inputs.py

+2 -41

@@ -4,52 +4,13 @@

4 import pytest

6 from vllm.config import ModelConfig

7 -from vllm.inputs import zip_enc_dec_prompts

7 from vllm.inputs.preprocess import InputPreprocessor

9 pytestmark = pytest.mark.cpu_test

13 -@pytest.mark.parametrize(

14 - "mm_processor_kwargs,expected_mm_kwargs",

15 - [

16 - (None, [{}, {}]),

17 - ({}, [{}, {}]),

18 - ({"foo": 100}, [{"foo": 100}, {"foo": 100}]),

19 - ([{"foo": 100}, {"bar": 200}], [{"foo": 100}, {"bar": 200}]),

20 - ],

21 -)

22 -def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):

23 - """Test mm_processor_kwargs init for zipping enc/dec prompts."""

24 - encoder_prompts = ["An encoder prompt", "Another encoder prompt"]

25 - decoder_prompts = ["A decoder prompt", "Another decoder prompt"]

26 - zipped_prompts = zip_enc_dec_prompts(

27 - encoder_prompts, decoder_prompts, mm_processor_kwargs

28 - )

29 - assert len(zipped_prompts) == len(encoder_prompts) == len(decoder_prompts)

30 - for enc, dec, exp_kwargs, zipped in zip(

31 - encoder_prompts, decoder_prompts, expected_mm_kwargs, zipped_prompts

32 - ):

33 - assert isinstance(zipped, dict)

34 - assert len(zipped.keys()) == 3

35 - assert zipped["encoder_prompt"] == enc

36 - assert zipped["decoder_prompt"] == dec

37 - assert zipped["mm_processor_kwargs"] == exp_kwargs

38 -

39 -

40 -@pytest.mark.parametrize(

41 - "model_id",

42 - [

43 - "facebook/chameleon-7b",

44 - ],

45 -)

46 -@pytest.mark.parametrize(

47 - "prompt",

48 - [

49 - "",

50 - {"prompt_token_ids": []},

51 - ],

52 -)

12 +@pytest.mark.parametrize("model_id", ["facebook/chameleon-7b"])

13 +@pytest.mark.parametrize("prompt", ["", {"prompt_token_ids": []}])

14 @pytest.mark.skip(

15 reason=(

16 "Applying huggingface processor on text inputs results in "

vllm/engine/protocol.py

+7 -2

@@ -16,6 +16,7 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput

16 from vllm.plugins.io_processors import IOProcessor

17 from vllm.pooling_params import PoolingParams

18 from vllm.renderers import BaseRenderer

19 +from vllm.renderers.inputs import DictPrompt, TokPrompt

20 from vllm.sampling_params import SamplingParams

21 from vllm.tasks import SupportedTask

22 from vllm.v1.engine import EngineCoreRequest

@@ -53,7 +54,11 @@ class EngineClient(ABC):

54 @abstractmethod

55 def generate(

56 self,

56 - prompt: EngineCoreRequest | PromptType | AsyncGenerator[StreamingInput, None],

57 + prompt: EngineCoreRequest

58 + | PromptType

59 + | DictPrompt

60 + | TokPrompt

61 + | AsyncGenerator[StreamingInput, None],

62 sampling_params: SamplingParams,

63 request_id: str,

64 *,

@@ -70,7 +75,7 @@ class EngineClient(ABC):

75 @abstractmethod

76 def encode(

77 self,

73 - prompt: PromptType,

78 + prompt: PromptType | DictPrompt | TokPrompt,

79 pooling_params: PoolingParams,

80 request_id: str,

81 lora_request: LoRARequest | None = None,

vllm/entrypoints/llm.py

+187 -149

@@ -4,7 +4,7 @@

4 import itertools

5 import warnings

6 from collections.abc import Callable, Sequence

7 -from typing import TYPE_CHECKING, Any, TypeAlias, cast

7 +from typing import TYPE_CHECKING, Any, cast

9 import cloudpickle

10 import torch.nn as nn

@@ -53,16 +53,13 @@ from vllm.entrypoints.pooling.score.utils import (

53 validate_score_input,

54 )

55 from vllm.entrypoints.utils import log_non_default_args

56 -from vllm.inputs import (

56 +from vllm.inputs.data import (

57 DataPrompt,

58 - EmbedsPrompt,

59 - ExplicitEncoderDecoderPrompt,

58 PromptType,

59 SingletonPrompt,

60 TextPrompt,

61 TokensPrompt,

62 )

65 -from vllm.inputs.parse import get_prompt_components, is_explicit_encoder_decoder_prompt

63 from vllm.logger import init_logger

64 from vllm.lora.request import LoRARequest

65 from vllm.model_executor.layers.quantization import QuantizationMethods

@@ -76,6 +73,13 @@ from vllm.outputs import (

73 from vllm.platforms import current_platform

74 from vllm.pooling_params import PoolingParams

75 from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs

76 +from vllm.renderers.inputs import DictPrompt, SingletonDictPrompt, TokPrompt

77 +from vllm.renderers.inputs.preprocess import (

78 + conversation_to_seq,

79 + extract_prompt_components,

80 + parse_model_prompt,

81 + prompt_to_seq,

82 +)

83 from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams

84 from vllm.tasks import PoolingTask

85 from vllm.tokenizers import TokenizerLike

@@ -93,9 +97,6 @@ logger = init_logger(__name__)

98 _R = TypeVar("_R", default=Any)

96 -EnginePrompt: TypeAlias = TextPrompt | TokensPrompt | EmbedsPrompt

97 -EngineEncDecPrompt: TypeAlias = ExplicitEncoderDecoderPrompt[EnginePrompt, EnginePrompt]

98 -

100

101 class LLM:

102 """An LLM for generating texts from given prompts and sampling parameters.

@@ -445,21 +446,20 @@ class LLM:

446 if sampling_params is None:

447 sampling_params = self.get_default_sampling_params()

448

448 - self._validate_and_add_requests(

449 + outputs = self._run_completion(

450 prompts=prompts,

451 params=sampling_params,

452 use_tqdm=use_tqdm,

452 - lora_request=self._get_modality_specific_lora_reqs(prompts, lora_request),

453 + lora_request=lora_request,

454 tokenization_kwargs=tokenization_kwargs,

455 priority=priority,

456 )

457

457 - outputs = self._run_engine(use_tqdm=use_tqdm)

458 return self.engine_class.validate_outputs(outputs, RequestOutput)

459

460 def _get_modality_specific_lora_reqs(

461 self,

462 - prompts: PromptType | Sequence[PromptType],

462 + prompts: Sequence[DictPrompt | TokPrompt],

463 lora_request: list[LoRARequest] | LoRARequest | None,

464 ):

465 # Grab the lora config off the vllm config on the engine,

@@ -475,9 +475,6 @@ class LLM:

475 ):

476 return lora_request

477

478 - if not isinstance(prompts, Sequence) or isinstance(prompts, str):

479 - prompts = [prompts]

480 -

478 optional_loras = (

479 [lora_request] * len(prompts)

480 if not isinstance(lora_request, Sequence)

@@ -495,14 +492,12 @@ class LLM:

492

493 def _resolve_single_prompt_mm_lora(

494 self,

498 - prompt: PromptType,

495 + prompt: DictPrompt | TokPrompt,

496 lora_request: LoRARequest | None,

497 default_mm_loras: dict[str, str] | None,

498 ):

502 - if (

503 - not default_mm_loras

504 - or not isinstance(prompt, dict)

505 - or not (mm_data := prompt.get("multi_modal_data") or {})

499 + if not default_mm_loras or not (

500 + mm_data := prompt.get("multi_modal_data") or {}

501 ):

502 return lora_request

503

@@ -806,61 +801,11 @@ class LLM:

801 add_special_tokens=not model_config.is_encoder_decoder,

802 ).with_kwargs(tokenization_kwargs)

803

809 - def _normalize_prompts(

810 - self,

811 - prompts: PromptType | Sequence[PromptType],

812 - ) -> list[EnginePrompt | EngineEncDecPrompt]:

813 - if isinstance(prompts, str):

814 - prompts = TextPrompt(prompt=prompts)

815 -

816 - return prompts if isinstance(prompts, Sequence) else [prompts] # type: ignore[return-value]

817 -

818 - def _preprocess_cmpl_singleton(

819 - self,

820 - prompt: SingletonPrompt,

821 - tok_params: TokenizeParams,

822 - *,

823 - tokenize: bool,

824 - ) -> EnginePrompt:

825 - renderer = self.llm_engine.renderer

826 -

827 - if not isinstance(prompt, dict):

828 - prompt = renderer.render_completion(prompt)

829 -

830 - return renderer.tokenize_prompt(prompt, tok_params) if tokenize else prompt

831 -

832 - def _preprocess_cmpl_enc_dec(

833 - self,

834 - prompt: ExplicitEncoderDecoderPrompt,

835 - tok_params: TokenizeParams,

836 - ) -> EngineEncDecPrompt:

837 - enc_prompt = prompt["encoder_prompt"]

838 - dec_prompt = prompt["decoder_prompt"]

839 -

840 - return EngineEncDecPrompt(

841 - encoder_prompt=self._preprocess_cmpl_singleton(

842 - enc_prompt,

843 - tok_params,

844 - # TODO: Move multi-modal processor into tokenization

845 - tokenize=not self.model_config.is_multimodal_model,

846 - ),

847 - decoder_prompt=(

848 - None

849 - if dec_prompt is None

850 - else self._preprocess_cmpl_singleton(

851 - dec_prompt,

852 - tok_params,

853 - # TODO: Move multi-modal processor into tokenization

854 - tokenize=not self.model_config.is_multimodal_model,

855 - )

856 - ),

857 - )

858 -

804 def _preprocess_completion(

805 self,

861 - prompts: PromptType | Sequence[PromptType],

806 + prompts: Sequence[PromptType],

807 tokenization_kwargs: dict[str, Any] | None = None,

863 - ) -> list[EnginePrompt | EngineEncDecPrompt]:

808 + ) -> list[DictPrompt | TokPrompt]:

809 """

810 Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into

811 a format that can be passed to `_add_request`.

@@ -871,32 +816,26 @@ class LLM:

816 A list of `TokensPrompts` objects containing the tokenized prompt

817 after chat template interpolation, and the raw multi-modal inputs.

818 """

819 + renderer = self.llm_engine.renderer

820 + model_config = self.model_config

821 +

822 tok_params = self._get_cmpl_tok_params(tokenization_kwargs)

823

876 - engine_prompts = list[EnginePrompt | EngineEncDecPrompt]()

877 - for prompt in self._normalize_prompts(prompts):

878 - if is_explicit_encoder_decoder_prompt(prompt):

879 - engine_prompts.append(self._preprocess_cmpl_enc_dec(prompt, tok_params))

880 - else:

881 - # Some MM models have non-default `add_special_tokens`

882 - # TODO: Move multi-modal processor into tokenization

883 - engine_prompts.append(

884 - self._preprocess_cmpl_singleton(

885 - prompt,

886 - tok_params,

887 - tokenize=not self.model_config.is_multimodal_model,

888 - )

889 - )

824 + engine_prompts = list[DictPrompt | TokPrompt]()

825 + for prompt in prompts:

826 + parsed_prompt = parse_model_prompt(model_config, prompt)

827 + in_prompt = renderer.render_prompt(parsed_prompt)

828 +

829 + # Some MM models have non-default `add_special_tokens`

830 + # TODO: Move multi-modal processor into tokenization

831 + engine_prompts.append(

832 + in_prompt

833 + if model_config.is_multimodal_model

834 + else renderer.tokenize_prompt(in_prompt, tok_params)

835 + )

836

837 return engine_prompts

838

893 - def _normalize_conversations(

894 - self,

895 - conversations: list[ChatCompletionMessageParam]

896 - | list[list[ChatCompletionMessageParam]],

897 - ) -> list[list[ChatCompletionMessageParam]]:

898 - return conversations if is_list_of(conversations, list) else [conversations] # type: ignore[list-item,return-value]

899 -

839 def _get_chat_tok_params(self, tokenization_kwargs: dict[str, Any] | None):

840 model_config = self.model_config

841 encoder_config = model_config.encoder_config or {}

@@ -909,8 +848,7 @@ class LLM:

848

849 def _preprocess_chat(

850 self,

912 - conversations: list[ChatCompletionMessageParam]

913 - | list[list[ChatCompletionMessageParam]],

851 + conversations: Sequence[list[ChatCompletionMessageParam]],

852 chat_template: str | None = None,

853 chat_template_content_format: ChatTemplateContentFormatOption = "auto",

854 chat_template_kwargs: dict[str, Any] | None = None,

@@ -919,7 +857,7 @@ class LLM:

857 tools: list[dict[str, Any]] | None = None,

858 tokenization_kwargs: dict[str, Any] | None = None,

859 mm_processor_kwargs: dict[str, Any] | None = None,

922 - ) -> list[EnginePrompt]:

860 + ) -> list[DictPrompt | TokPrompt]:

861 """

862 Convert a list of conversations into prompts so that they can then

863 be used as input for other LLM APIs.

@@ -947,11 +885,14 @@ class LLM:

885 )

886 tok_params = self._get_chat_tok_params(tokenization_kwargs)

887

950 - engine_prompts = list[EnginePrompt]()

951 - for conversation in self._normalize_conversations(conversations):

888 + engine_prompts = list[DictPrompt | TokPrompt]()

889 + for conversation in conversations:

890 _, in_prompt = renderer.render_messages(conversation, chat_params)

891 if mm_processor_kwargs is not None:

954 - in_prompt["mm_processor_kwargs"] = mm_processor_kwargs

892 + target_prompt: SingletonDictPrompt = in_prompt.get( # type: ignore

893 + "encoder_prompt", in_prompt

894 + )

895 + target_prompt["mm_processor_kwargs"] = mm_processor_kwargs # type: ignore

896

897 engine_prompts.append(renderer.tokenize_prompt(in_prompt, tok_params))

898

@@ -960,8 +901,8 @@ class LLM:

901 def chat(

902 self,

903 messages: list[ChatCompletionMessageParam]

963 - | list[list[ChatCompletionMessageParam]],

964 - sampling_params: SamplingParams | list[SamplingParams] | None = None,

904 + | Sequence[list[ChatCompletionMessageParam]],

905 + sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,

906 use_tqdm: bool | Callable[..., tqdm] = True,

907 lora_request: LoRARequest | None = None,

908 chat_template: str | None = None,

@@ -984,7 +925,7 @@ class LLM:

925 to the OpenAI API.

926

927 Args:

987 - messages: A list of conversations or a single conversation.

928 + messages: A sequence of conversations or a single conversation.

929

930 - Each conversation is represented as a list of messages.

931 - Each message is a dictionary with 'role' and 'content' keys.

@@ -1023,8 +964,23 @@ class LLM:

964 A list of `RequestOutput` objects containing the generated

965 responses in the same order as the input messages.

966 """

1026 - prompts = self._preprocess_chat(

1027 - messages,

967 + model_config = self.model_config

968 + runner_type = model_config.runner_type

969 + if runner_type != "generate":

970 + raise ValueError(

971 + "LLM.chat() is only supported for generative models. "

972 + "Try passing `--runner generate` to use the model as a "

973 + "generative model."

974 + )

975 +

976 + if sampling_params is None:

977 + sampling_params = self.get_default_sampling_params()

978 +

979 + outputs = self._run_chat(

980 + messages=messages,

981 + params=sampling_params,

982 + use_tqdm=use_tqdm,

983 + lora_request=lora_request,

984 chat_template=chat_template,

985 chat_template_content_format=chat_template_content_format,

986 chat_template_kwargs=chat_template_kwargs,

@@ -1035,13 +991,7 @@ class LLM:

991 mm_processor_kwargs=mm_processor_kwargs,

992 )

993

1038 - return self.generate(

1039 - prompts,

1040 - sampling_params=sampling_params,

1041 - use_tqdm=use_tqdm,

1042 - lora_request=lora_request,

1043 - tokenization_kwargs=tokenization_kwargs,

1044 - )

994 + return self.engine_class.validate_outputs(outputs, RequestOutput)

995

996 def encode(

997 self,

@@ -1163,7 +1113,7 @@ class LLM:

1113 msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"

1114 raise ValueError(msg)

1115

1166 - self._validate_and_add_requests(

1116 + outputs = self._run_completion(

1117 prompts=prompts,

1118 params=pooling_params,

1119 use_tqdm=use_tqdm,

@@ -1171,8 +1121,6 @@ class LLM:

1121 tokenization_kwargs=tokenization_kwargs,

1122 )

1123

1174 - outputs = self._run_engine(use_tqdm=use_tqdm)

1175 -

1124 model_outputs = self.engine_class.validate_outputs(

1125 outputs, PoolingRequestOutput

1126 )

@@ -1523,14 +1471,13 @@ class LLM:

1471

1472 prompts.append(engine_prompt)

1473

1526 - self._validate_and_add_requests(

1474 + outputs = self._run_completion(

1475 prompts=prompts,

1476 params=pooling_params_list,

1477 use_tqdm=use_tqdm,

1478 lora_request=lora_request,

1479 )

1480

1533 - outputs = self._run_engine(use_tqdm=use_tqdm)

1481 items = self.engine_class.validate_outputs(outputs, PoolingRequestOutput)

1482

1483 return [ScoringRequestOutput.from_base(item) for item in items]

@@ -1727,33 +1674,29 @@ class LLM:

1674 """

1675 return self.llm_engine.get_metrics()

1676

1730 - def _validate_and_add_requests(

1677 + def _params_to_seq(

1678 self,

1732 - prompts: PromptType | Sequence[PromptType],

1679 params: SamplingParams

1734 - | Sequence[SamplingParams]

1680 | PoolingParams

1736 - | Sequence[PoolingParams],

1737 - *,

1738 - use_tqdm: bool | Callable[..., tqdm] = True,

1739 - lora_request: Sequence[LoRARequest | None] | LoRARequest | None,

1740 - tokenization_kwargs: dict[str, Any] | None = None,

1741 - priority: list[int] | None = None,

1742 - ) -> None:

1743 - in_prompts = self._normalize_prompts(prompts)

1744 - num_requests = len(in_prompts)

1745 -

1681 + | Sequence[SamplingParams | PoolingParams],

1682 + num_requests: int,

1683 + ) -> Sequence[SamplingParams | PoolingParams]:

1684 if isinstance(params, Sequence):

1685 if len(params) != num_requests:

1686 raise ValueError(

1687 f"The lengths of prompts ({params}) "

1750 - f"and lora_request ({len(params)}) must be the same."

1688 + f"and params ({len(params)}) must be the same."

1689 )

1690

1753 - engine_params = params

1754 - else:

1755 - engine_params = [params] * num_requests

1691 + return params

1692

1693 + return [params] * num_requests

1694 +

1695 + def _lora_request_to_seq(

1696 + self,

1697 + lora_request: LoRARequest | None | Sequence[LoRARequest | None],

1698 + num_requests: int,

1699 + ) -> Sequence[LoRARequest | None]:

1700 if isinstance(lora_request, Sequence):

1701 if len(lora_request) != num_requests:

1702 raise ValueError(

@@ -1761,28 +1704,50 @@ class LLM:

1704 f"and lora_request ({len(lora_request)}) must be the same."

1705 )

1706

1764 - engine_lora_requests: Sequence[LoRARequest | None] = lora_request

1765 - else:

1766 - engine_lora_requests = [lora_request] * num_requests

1707 + return lora_request

1708 +

1709 + return [lora_request] * num_requests

1710

1711 + def _priority_to_seq(

1712 + self,

1713 + priority: list[int] | None,

1714 + num_requests: int,

1715 + ) -> Sequence[int]:

1716 if priority is not None:

1717 if len(priority) != num_requests:

1718 raise ValueError(

1719 f"The lengths of prompts ({num_requests}) "

1720 f"and priority ({len(priority)}) must be the same."

1721 )

1774 - else:

1775 - priority = [0] * num_requests

1722

1777 - if any(param.truncate_prompt_tokens is not None for param in engine_params):

1723 + return priority

1724 +

1725 + return [0] * num_requests

1726 +

1727 + def _run_completion(

1728 + self,

1729 + prompts: PromptType | Sequence[PromptType],

1730 + params: SamplingParams

1731 + | PoolingParams

1732 + | Sequence[SamplingParams | PoolingParams],

1733 + *,

1734 + use_tqdm: bool | Callable[..., tqdm] = True,

1735 + lora_request: list[LoRARequest] | LoRARequest | None = None,

1736 + priority: list[int] | None = None,

1737 + tokenization_kwargs: dict[str, Any] | None = None,

1738 + ):

1739 + seq_prompts = prompt_to_seq(prompts)

1740 + seq_params = self._params_to_seq(params, len(seq_prompts))

1741 +

1742 + if any(param.truncate_prompt_tokens is not None for param in seq_params):

1743 # TODO: Remove this after deprecating `param.truncate_prompt_tokens`

1744 # Then, move the code from the `else` block to the top and let

1745 # `self._preprocess_completion` handle prompt normalization

1746 engine_prompts = [

1747 engine_prompt

1783 - for in_prompt, param in zip(in_prompts, engine_params)

1748 + for prompt, param in zip(seq_prompts, seq_params)

1749 for engine_prompt in self._preprocess_completion(

1785 - [in_prompt],

1750 + [prompt],

1751 tokenization_kwargs=merge_kwargs(

1752 tokenization_kwargs,

1753 dict(truncate_prompt_tokens=param.truncate_prompt_tokens),

@@ -1791,17 +1756,90 @@ class LLM:

1756 ]

1757 else:

1758 engine_prompts = self._preprocess_completion(

1794 - in_prompts,

1759 + seq_prompts,

1760 tokenization_kwargs=tokenization_kwargs,

1761 )

1762

1798 - for sp in engine_params:

1763 + self._validate_and_add_requests(

1764 + prompts=engine_prompts,

1765 + params=seq_params,

1766 + use_tqdm=use_tqdm,

1767 + lora_request=self._get_modality_specific_lora_reqs(

1768 + engine_prompts, lora_request

1769 + ),

1770 + tokenization_kwargs=tokenization_kwargs,

1771 + priority=priority,

1772 + )

1773 +

1774 + return self._run_engine(use_tqdm=use_tqdm)

1775 +

1776 + def _run_chat(

1777 + self,

1778 + messages: list[ChatCompletionMessageParam]

1779 + | Sequence[list[ChatCompletionMessageParam]],

1780 + params: SamplingParams

1781 + | PoolingParams

1782 + | Sequence[SamplingParams | PoolingParams],

1783 + *,

1784 + use_tqdm: bool | Callable[..., tqdm] = True,

1785 + lora_request: LoRARequest | None = None,

1786 + chat_template: str | None = None,

1787 + chat_template_content_format: ChatTemplateContentFormatOption = "auto",

1788 + add_generation_prompt: bool = True,

1789 + continue_final_message: bool = False,

1790 + tools: list[dict[str, Any]] | None = None,

1791 + chat_template_kwargs: dict[str, Any] | None = None,

1792 + tokenization_kwargs: dict[str, Any] | None = None,

1793 + mm_processor_kwargs: dict[str, Any] | None = None,

1794 + ):

1795 + engine_prompts = self._preprocess_chat(

1796 + conversation_to_seq(messages),

1797 + chat_template=chat_template,

1798 + chat_template_content_format=chat_template_content_format,

1799 + chat_template_kwargs=chat_template_kwargs,

1800 + add_generation_prompt=add_generation_prompt,

1801 + continue_final_message=continue_final_message,

1802 + tools=tools,

1803 + tokenization_kwargs=tokenization_kwargs,

1804 + mm_processor_kwargs=mm_processor_kwargs,

1805 + )

1806 +

1807 + self._validate_and_add_requests(

1808 + prompts=engine_prompts,

1809 + params=params,

1810 + use_tqdm=use_tqdm,

1811 + lora_request=self._get_modality_specific_lora_reqs(

1812 + engine_prompts, lora_request

1813 + ),

1814 + tokenization_kwargs=tokenization_kwargs,

1815 + )

1816 +

1817 + return self._run_engine(use_tqdm=use_tqdm)

1818 +

1819 + def _validate_and_add_requests(

1820 + self,

1821 + prompts: Sequence[DictPrompt | TokPrompt],

1822 + params: SamplingParams

1823 + | PoolingParams

1824 + | Sequence[SamplingParams | PoolingParams],

1825 + *,

1826 + use_tqdm: bool | Callable[..., tqdm] = True,

1827 + lora_request: Sequence[LoRARequest | None] | LoRARequest | None,

1828 + tokenization_kwargs: dict[str, Any] | None = None,

1829 + priority: list[int] | None = None,

1830 + ) -> None:

1831 + num_requests = len(prompts)

1832 + seq_params = self._params_to_seq(params, num_requests)

1833 + seq_lora_requests = self._lora_request_to_seq(lora_request, num_requests)

1834 + seq_priority = self._priority_to_seq(priority, num_requests)

1835 +

1836 + for sp in seq_params:

1837 if isinstance(sp, SamplingParams):

1838 # We only care about the final output

1839 sp.output_kind = RequestOutputKind.FINAL_ONLY

1840

1841 # Add requests to the engine.

1804 - it = engine_prompts

1842 + it = prompts

1843 if use_tqdm:

1844 tqdm_func = use_tqdm if callable(use_tqdm) else tqdm

1845 it = tqdm_func(it, desc="Adding requests")

@@ -1812,10 +1850,10 @@ class LLM:

1850 for i, prompt in enumerate(it):

1851 request_id = self._add_request(

1852 prompt,

1815 - engine_params[i],

1816 - lora_request=engine_lora_requests[i],

1853 + seq_params[i],

1854 + lora_request=seq_lora_requests[i],

1855 tokenization_kwargs=tokenization_kwargs,

1818 - priority=priority[i],

1856 + priority=seq_priority[i],

1857 )

1858 added_request_ids.append(request_id)

1859 except Exception as e:

@@ -1825,13 +1863,13 @@ class LLM:

1863

1864 def _add_request(

1865 self,

1828 - prompt: PromptType,

1866 + prompt: PromptType | DictPrompt | TokPrompt,

1867 params: SamplingParams | PoolingParams,

1868 lora_request: LoRARequest | None = None,

1869 tokenization_kwargs: dict[str, Any] | None = None,

1870 priority: int = 0,

1871 ) -> str:

1834 - prompt_text, _, _ = get_prompt_components(prompt)

1872 + prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)

1873 request_id = str(next(self.request_counter))

1874

1875 if params.truncate_prompt_tokens is not None:

vllm/entrypoints/openai/chat_completion/serving.py

+8 -10

@@ -67,12 +67,13 @@ from vllm.entrypoints.openai.parser.harmony_utils import (

67 )

68 from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls

69 from vllm.entrypoints.utils import get_max_tokens, should_include_usage

70 -from vllm.inputs.data import EmbedsPrompt, TokensPrompt

70 +from vllm.inputs.data import TokensPrompt

71 from vllm.logger import init_logger

72 from vllm.logprobs import Logprob

73 from vllm.outputs import CompletionOutput, RequestOutput

74 from vllm.parser import ParserManager

75 from vllm.reasoning import ReasoningParser

76 +from vllm.renderers.inputs import TokPrompt

77 from vllm.sampling_params import BeamSearchParams, SamplingParams

78 from vllm.tokenizers import TokenizerLike

79 from vllm.tokenizers.mistral import (

@@ -218,10 +219,7 @@ class OpenAIServingChat(OpenAIServing):

219 async def render_chat_request(

220 self,

221 request: ChatCompletionRequest,

221 - ) -> (

222 - tuple[list[ConversationMessage], list[TokensPrompt | EmbedsPrompt]]

223 - | ErrorResponse

224 - ):

222 + ) -> tuple[list[ConversationMessage], list[TokPrompt]] | ErrorResponse:

223 """

224 render chat request by validating and preprocessing inputs.

225

@@ -380,7 +378,7 @@ class OpenAIServingChat(OpenAIServing):

378 generators: list[AsyncGenerator[RequestOutput, None]] = []

379 try:

380 for i, engine_prompt in enumerate(engine_prompts):

383 - prompt_text = engine_prompt.get("prompt")

381 + prompt_text = self._extract_prompt_text(engine_prompt)

382

383 # If we are creating sub requests for multiple prompts, ensure that they

384 # have unique request ids.

@@ -389,10 +387,10 @@ class OpenAIServingChat(OpenAIServing):

387 )

388

389 max_tokens = get_max_tokens(

392 - max_model_len=self.max_model_len,

393 - request=request,

394 - prompt=engine_prompt,

395 - default_sampling_params=self.default_sampling_params,

390 + self.max_model_len,

391 + request,

392 + self._extract_prompt_len(engine_prompt),

393 + self.default_sampling_params,

394 )

395

396 sampling_params: SamplingParams | BeamSearchParams

vllm/entrypoints/openai/completion/serving.py

+10 -10

@@ -34,10 +34,10 @@ from vllm.entrypoints.openai.engine.serving import (

34 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

35 from vllm.entrypoints.utils import get_max_tokens, should_include_usage

36 from vllm.exceptions import VLLMValidationError

37 -from vllm.inputs.data import EmbedsPrompt, TokensPrompt

37 from vllm.logger import init_logger

38 from vllm.logprobs import Logprob

39 from vllm.outputs import RequestOutput

40 +from vllm.renderers.inputs import TokPrompt

41 from vllm.sampling_params import BeamSearchParams, SamplingParams

42 from vllm.tokenizers import TokenizerLike

43 from vllm.utils.async_utils import merge_async_iterators

@@ -78,7 +78,7 @@ class OpenAIServingCompletion(OpenAIServing):

78 async def render_completion_request(

79 self,

80 request: CompletionRequest,

81 - ) -> list[TokensPrompt | EmbedsPrompt] | ErrorResponse:

81 + ) -> list[TokPrompt] | ErrorResponse:

82 """

83 render completion request by validating and preprocessing inputs.

@@ -160,13 +160,13 @@ class OpenAIServingCompletion(OpenAIServing):

160 generators: list[AsyncGenerator[RequestOutput, None]] = []

161 try:

162 for i, engine_prompt in enumerate(engine_prompts):

163 - prompt_text = engine_prompt.get("prompt")

163 + prompt_text = self._extract_prompt_text(engine_prompt)

164

165 max_tokens = get_max_tokens(

166 - max_model_len=self.max_model_len,

167 - request=request,

168 - prompt=engine_prompt,

169 - default_sampling_params=self.default_sampling_params,

166 + self.max_model_len,

167 + request,

168 + self._extract_prompt_len(engine_prompt),

169 + self.default_sampling_params,

170 )

171

172 sampling_params: SamplingParams | BeamSearchParams

@@ -277,7 +277,7 @@ class OpenAIServingCompletion(OpenAIServing):

277 # with the inputs token IDs

278 if final_res.prompt is None:

279 engine_prompt = engine_prompts[i]

280 - final_res.prompt = engine_prompt.get("prompt")

280 + final_res.prompt = self._extract_prompt_text(engine_prompt)

281

282 final_res_batch_checked = cast(list[RequestOutput], final_res_batch)

283

@@ -313,7 +313,7 @@ class OpenAIServingCompletion(OpenAIServing):

313 async def completion_stream_generator(

314 self,

315 request: CompletionRequest,

316 - engine_prompts: list[TokensPrompt | EmbedsPrompt],

316 + engine_prompts: list[TokPrompt],

317 result_generator: AsyncIterator[tuple[int, RequestOutput]],

318 request_id: str,

319 created_time: int,

@@ -347,7 +347,7 @@ class OpenAIServingCompletion(OpenAIServing):

347 prompt_text = res.prompt

348 if prompt_text is None:

349 engine_prompt = engine_prompts[prompt_idx]

350 - prompt_text = engine_prompt.get("prompt")

350 + prompt_text = self._extract_prompt_text(engine_prompt)

351

352 # Prompt details are excluded from later streamed outputs

353 if prompt_token_ids is not None:

vllm/entrypoints/openai/engine/serving.py

+68 -41

@@ -96,11 +96,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (

96 )

97 from vllm.entrypoints.utils import get_max_tokens, sanitize_message

98 from vllm.exceptions import VLLMValidationError

99 -from vllm.inputs.data import EmbedsPrompt, PromptType, TokensPrompt

100 -from vllm.inputs.parse import (

101 - get_prompt_components,

102 - is_explicit_encoder_decoder_prompt,

103 -)

99 +from vllm.inputs.data import PromptType, SingletonPrompt, TokensPrompt

100 from vllm.logger import init_logger

101 from vllm.logprobs import Logprob, PromptLogprobs

102 from vllm.lora.request import LoRARequest

@@ -108,6 +104,14 @@ from vllm.multimodal import MultiModalDataDict

104 from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput

105 from vllm.pooling_params import PoolingParams

106 from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs

107 +from vllm.renderers.inputs import TokPrompt

108 +from vllm.renderers.inputs.preprocess import (

109 + SingletonDictPrompt,

110 + extract_prompt_components,

111 + extract_prompt_len,

112 + parse_model_prompt,

113 + prompt_to_seq,

114 +)

115 from vllm.sampling_params import BeamSearchParams, SamplingParams

116 from vllm.tokenizers import TokenizerLike

117 from vllm.tool_parsers import ToolParser

@@ -203,7 +207,7 @@ class ServeContext(Generic[RequestT]):

207 request_id: str

208 created_time: int = field(default_factory=lambda: int(time.time()))

209 lora_request: LoRARequest | None = None

206 - engine_prompts: list[TokensPrompt | EmbedsPrompt] | None = None

210 + engine_prompts: list[TokPrompt] | None = None

211

212 result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (

213 None

@@ -247,7 +251,7 @@ class OpenAIServing:

251

252 async def beam_search(

253 self,

250 - prompt: PromptType,

254 + prompt: TokPrompt,

255 request_id: str,

256 params: BeamSearchParams,

257 lora_request: LoRARequest | None = None,

@@ -271,20 +275,12 @@ class OpenAIServing:

275

276 eos_token_id: int = tokenizer.eos_token_id # type: ignore

277

274 - if is_explicit_encoder_decoder_prompt(prompt):

275 - raise NotImplementedError

278 + if isinstance(prompt, dict) and "encoder_prompt" in prompt:

279 + raise NotImplementedError("Encoder-decoder prompt not supported")

280

277 - prompt_text: str | None

278 - prompt_token_ids: list[int]

279 - multi_modal_data: MultiModalDataDict | None

280 - if isinstance(prompt, str):

281 - prompt_text = prompt

282 - prompt_token_ids = []

283 - multi_modal_data = None

284 - else:

285 - prompt_text = prompt.get("prompt") # type: ignore

286 - prompt_token_ids = prompt.get("prompt_token_ids", []) # type: ignore

287 - multi_modal_data = prompt.get("multi_modal_data") # type: ignore

281 + prompt_text: str | None = prompt.get("prompt") # type: ignore

282 + prompt_token_ids: list[int] = prompt.get("prompt_token_ids", []) # type: ignore

283 + multi_modal_data: MultiModalDataDict | None = prompt.get("multi_modal_data") # type: ignore

284

285 mm_processor_kwargs: dict[str, Any] | None = None

286

@@ -963,22 +959,40 @@ class OpenAIServing:

959 request: RendererRequest,

960 prompt_input: str | list[str] | list[int] | list[list[int]] | None,

961 prompt_embeds: bytes | list[bytes] | None,

966 - ) -> list[TokensPrompt | EmbedsPrompt]:

962 + ) -> list[TokPrompt]:

963 renderer = self.renderer

968 - tok_params = request.build_tok_params(self.model_config)

964 + model_config = self.model_config

965

970 - in_prompts = await renderer.render_completions_async(

971 - prompt_input, prompt_embeds

972 - )

973 - engine_prompts = await renderer.tokenize_prompts_async(in_prompts, tok_params)

966 + tok_params = request.build_tok_params(model_config)

967 +

968 + prompts = list[SingletonPrompt | bytes]()

969 + if prompt_embeds is not None: # embeds take higher priority

970 + prompts.extend(prompt_to_seq(prompt_embeds))

971 + if prompt_input is not None:

972 + prompts.extend(prompt_to_seq(prompt_input))

973 +

974 + parsed_prompts = [

975 + (

976 + prompt

977 + if isinstance(prompt, bytes)

978 + else parse_model_prompt(model_config, prompt)

979 + )

980 + for prompt in prompts

981 + ]

982 + in_prompts = await renderer.render_prompts_async(parsed_prompts)

983

984 extra_items = {

985 k: v

986 for k in ("mm_processor_kwargs", "cache_salt")

987 if (v := getattr(request, k, None)) is not None

988 }

980 - for prompt in engine_prompts:

981 - prompt.update(extra_items) # type: ignore

989 + for in_prompt in in_prompts:

990 + target_prompt: SingletonDictPrompt = in_prompt.get( # type: ignore

991 + "encoder_prompt", in_prompt

992 + )

993 + target_prompt.update(extra_items) # type: ignore

994 +

995 + engine_prompts = await renderer.tokenize_prompts_async(in_prompts, tok_params)

996

997 return engine_prompts

998

@@ -991,7 +1005,7 @@ class OpenAIServing:

1005 default_template_kwargs: dict[str, Any] | None,

1006 tool_dicts: list[dict[str, Any]] | None = None,

1007 tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,

994 - ) -> tuple[list[ConversationMessage], list[TokensPrompt | EmbedsPrompt]]:

1008 + ) -> tuple[list[ConversationMessage], list[TokPrompt]]:

1009 from vllm.tokenizers.mistral import MistralTokenizer

1010

1011 renderer = self.renderer

@@ -1009,17 +1023,21 @@ class OpenAIServing:

1023 default_template, default_template_content_format

1024 ).with_defaults(default_template_kwargs)

1025

1012 - conversation, prompt = await renderer.render_messages_async(

1026 + conversation, in_prompt = await renderer.render_messages_async(

1027 messages, chat_params

1028 )

1015 - engine_prompt = await renderer.tokenize_prompt_async(prompt, tok_params)

1029 + target_prompt: SingletonDictPrompt = in_prompt.get( # type: ignore

1030 + "encoder_prompt", in_prompt

1031 + )

1032

1033 extra_items = {

1034 k: v

1035 for k in ("mm_processor_kwargs", "cache_salt")

1036 if (v := getattr(request, k, None)) is not None

1037 }

1022 - engine_prompt.update(extra_items) # type: ignore

1038 + target_prompt.update(extra_items) # type: ignore

1039 +

1040 + engine_prompt = await renderer.tokenize_prompt_async(target_prompt, tok_params)

1041

1042 # tool parsing is done only if a tool_parser has been set and if

1043 # tool_choice is not "none" (if tool_choice is "none" but a tool_parser

@@ -1040,6 +1058,15 @@ class OpenAIServing:

1058

1059 return conversation, [engine_prompt]

1060

1061 + def _extract_prompt_components(self, prompt: object):

1062 + return extract_prompt_components(self.model_config, prompt)

1063 +

1064 + def _extract_prompt_text(self, prompt: object):

1065 + return self._extract_prompt_components(prompt).text

1066 +

1067 + def _extract_prompt_len(self, prompt: object):

1068 + return extract_prompt_len(self.model_config, prompt)

1069 +

1070 async def _render_next_turn(

1071 self,

1072 request: ResponsesRequest,

@@ -1067,7 +1094,7 @@ class OpenAIServing:

1094 async def _generate_with_builtin_tools(

1095 self,

1096 request_id: str,

1070 - engine_prompt: TokensPrompt | EmbedsPrompt,

1097 + engine_prompt: TokPrompt,

1098 sampling_params: SamplingParams,

1099 tok_params: TokenizeParams,

1100 context: ConversationContext,

@@ -1075,7 +1102,7 @@ class OpenAIServing:

1102 priority: int = 0,

1103 trace_headers: Mapping[str, str] | None = None,

1104 ):

1078 - prompt_text = engine_prompt.get("prompt")

1105 + prompt_text = self._extract_prompt_text(engine_prompt)

1106

1107 orig_priority = priority

1108 sub_request = 0

@@ -1145,12 +1172,12 @@ class OpenAIServing:

1172 context.chat_template_content_format,

1173 )

1174 engine_prompt = engine_prompts[0]

1148 - prompt_text = engine_prompt.get("prompt")

1175 + prompt_text = self._extract_prompt_text(engine_prompt)

1176

1177 sampling_params.max_tokens = get_max_tokens(

1178 self.max_model_len,

1179 context.request,

1153 - engine_prompt,

1180 + self._extract_prompt_len(engine_prompt),

1181 self.default_sampling_params, # type: ignore

1182 )

1183

@@ -1161,20 +1188,20 @@ class OpenAIServing:

1188 def _log_inputs(

1189 self,

1190 request_id: str,

1164 - inputs: PromptType,

1191 + inputs: PromptType | TokPrompt,

1192 params: SamplingParams | PoolingParams | BeamSearchParams | None,

1193 lora_request: LoRARequest | None,

1194 ) -> None:

1195 if self.request_logger is None:

1196 return

1197

1171 - prompt, prompt_token_ids, prompt_embeds = get_prompt_components(inputs)

1198 + components = self._extract_prompt_components(inputs)

1199

1200 self.request_logger.log_inputs(

1201 request_id,

1175 - prompt,

1176 - prompt_token_ids,

1177 - prompt_embeds,

1202 + components.text,

1203 + components.token_ids,

1204 + components.embeds,

1205 params=params,

1206 lora_request=lora_request,

1207 )

vllm/entrypoints/openai/responses/serving.py

+5 -5

@@ -116,13 +116,13 @@ from vllm.entrypoints.openai.responses.utils import (

116 )

117 from vllm.entrypoints.utils import get_max_tokens

118 from vllm.exceptions import VLLMValidationError

119 -from vllm.inputs.data import EmbedsPrompt, TokensPrompt

120 -from vllm.inputs.parse import get_prompt_len

119 +from vllm.inputs.data import TokensPrompt

120 from vllm.logger import init_logger

121 from vllm.logprobs import Logprob as SampleLogprob

122 from vllm.logprobs import SampleLogprobs

123 from vllm.outputs import CompletionOutput

124 from vllm.parser import ParserManager

125 +from vllm.renderers.inputs import TokPrompt

126 from vllm.sampling_params import SamplingParams, StructuredOutputsParams

127 from vllm.tokenizers import TokenizerLike

128 from vllm.utils import random_uuid

@@ -292,10 +292,10 @@ class OpenAIServingResponses(OpenAIServing):

292

293 def _validate_generator_input(

294 self,

295 - engine_prompt: TokensPrompt | EmbedsPrompt,

295 + engine_prompt: TokPrompt,

296 ) -> ErrorResponse | None:

297 """Add validations to the input to the generator here."""

298 - prompt_len = get_prompt_len(engine_prompt)

298 + prompt_len = self._extract_prompt_len(engine_prompt)

299 if self.max_model_len <= prompt_len:

300 error_message = (

301 f"The engine prompt length {prompt_len} "

@@ -442,7 +442,7 @@ class OpenAIServingResponses(OpenAIServing):

442 default_max_tokens = get_max_tokens(

443 self.max_model_len,

444 request,

445 - engine_prompt,

445 + self._extract_prompt_len(engine_prompt),

446 self.default_sampling_params,

447 )

448

vllm/entrypoints/openai/speech_to_text/speech_to_text.py

+13 -21

@@ -7,7 +7,7 @@ import time

7 import zlib

8 from collections.abc import AsyncGenerator, Callable

9 from functools import cached_property

10 -from typing import Literal, TypeAlias, TypeVar, cast

10 +from typing import Final, Literal, TypeAlias, TypeVar, cast

12 import numpy as np

13 from fastapi import Request

@@ -37,12 +37,13 @@ from vllm.entrypoints.openai.speech_to_text.protocol import (

37 TranslationStreamResponse,

38 )

39 from vllm.exceptions import VLLMValidationError

40 -from vllm.inputs.data import ExplicitEncoderDecoderPrompt, PromptType

41 -from vllm.inputs.parse import is_explicit_encoder_decoder_prompt

40 +from vllm.inputs.data import PromptType

41 from vllm.logger import init_logger

42 from vllm.logprobs import FlatLogprobs, Logprob

43 from vllm.model_executor.models import SupportsTranscription, supports_transcription

44 from vllm.outputs import RequestOutput

45 +from vllm.renderers.inputs import EncoderDecoderDictPrompt

46 +from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt

47 from vllm.tokenizers import get_tokenizer

48 from vllm.utils.import_utils import PlaceholderModule

@@ -94,7 +95,7 @@ class OpenAISpeechToText(OpenAIServing):

95 )

97 self.default_sampling_params = self.model_config.get_diff_sampling_param()

97 - self.task_type = task_type

98 + self.task_type: Final = task_type

100 self.asr_config = self.model_cls.get_speech_to_text_config(

101 self.model_config, task_type

@@ -298,35 +299,26 @@ class OpenAISpeechToText(OpenAIServing):

299 to_language=to_language,

300 )

301 if request.response_format == "verbose_json":

301 - if not is_explicit_encoder_decoder_prompt(prompt):

302 - raise VLLMValidationError(

303 - "Expected prompt to be an encoder-decoder prompt",

304 - parameter="prompt",

305 - value=type(prompt).__name__,

306 - )

307 -

308 - prompt = self._preprocess_verbose_prompt(prompt)

302 + prompt = self._preprocess_verbose_prompt(parse_enc_dec_prompt(prompt))

303

304 prompts.append(prompt)

311 - return prompts, duration

305

313 - def _repl_verbose_text(self, text: str):

314 - return text.replace("<|notimestamps|>", "<|0.00|>")

306 + return prompts, duration

307

316 - def _preprocess_verbose_prompt(self, prompt: ExplicitEncoderDecoderPrompt):

308 + def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):

309 dec_prompt = prompt["decoder_prompt"]

310

319 - if isinstance(dec_prompt, str):

320 - prompt["decoder_prompt"] = self._repl_verbose_text(dec_prompt)

321 - elif isinstance(dec_prompt, dict) and "prompt" in dec_prompt:

322 - dec_prompt["prompt"] = self._repl_verbose_text(dec_prompt["prompt"])

323 - else:

311 + if not (isinstance(dec_prompt, dict) and "prompt" in dec_prompt):

312 raise VLLMValidationError(

313 "Expected decoder_prompt to contain text",

314 parameter="decoder_prompt",

315 value=type(dec_prompt).__name__,

316 )

317

318 + dec_prompt["prompt"] = dec_prompt["prompt"].replace(

319 + "<|notimestamps|>", "<|0.00|>"

320 + )

321 +

322 return prompt

323

324 def _get_verbose_segments(

vllm/entrypoints/pooling/embed/serving.py

+3 -2

@@ -28,10 +28,11 @@ from vllm.entrypoints.pooling.utils import (

28 encode_pooling_output_base64,

29 encode_pooling_output_float,

30 )

31 -from vllm.inputs.data import EmbedsPrompt, TokensPrompt

31 +from vllm.inputs.data import TokensPrompt

32 from vllm.logger import init_logger

33 from vllm.outputs import PoolingOutput, PoolingRequestOutput

34 from vllm.pooling_params import PoolingParams

35 +from vllm.renderers.inputs import TokPrompt

36 from vllm.utils.async_utils import merge_async_iterators

37 from vllm.utils.collection_utils import chunk_list

38 from vllm.utils.serial_utils import EmbedDType, Endianness

@@ -369,7 +370,7 @@ class OpenAIServingEmbedding(OpenAIServing):

370 async def _create_single_prompt_generator(

371 self,

372 ctx: EmbeddingServeContext,

372 - engine_prompt: TokensPrompt | EmbedsPrompt,

373 + engine_prompt: TokPrompt,

374 pooling_params: PoolingParams,

375 trace_headers: Mapping[str, str] | None,

376 prompt_index: int,

vllm/entrypoints/pooling/pooling/serving.py

+6 -6

@@ -33,8 +33,11 @@ from vllm.entrypoints.pooling.utils import (

33 encode_pooling_output_base64,

34 encode_pooling_output_float,

35 )

36 +from vllm.inputs import PromptType

37 from vllm.logger import init_logger

38 from vllm.outputs import PoolingRequestOutput

39 +from vllm.renderers.inputs import TokPrompt

40 +from vllm.renderers.inputs.preprocess import prompt_to_seq

41 from vllm.utils.async_utils import merge_async_iterators

42 from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness

@@ -91,6 +94,7 @@ class OpenAIServingPooling(OpenAIServing):

94 "dimensions is currently not supported"

95 )

97 + engine_prompts: Sequence[PromptType | TokPrompt]

98 if is_io_processor_request:

99 if self.io_processor is None:

100 raise ValueError(

@@ -102,14 +106,10 @@ class OpenAIServingPooling(OpenAIServing):

106

107 validated_prompt = self.io_processor.parse_request(request)

108

105 - engine_prompts = await self.io_processor.pre_process_async(

109 + raw_prompts = await self.io_processor.pre_process_async(

110 prompt=validated_prompt, request_id=request_id

111 )

108 - if not isinstance(engine_prompts, Sequence) or isinstance(

109 - engine_prompts, (str, bytes, bytearray)

110 - ):

111 - engine_prompts = [engine_prompts]

112 -

112 + engine_prompts = prompt_to_seq(raw_prompts)

113 elif isinstance(request, PoolingChatRequest):

114 error_check_ret = self._validate_chat_template(

115 request_chat_template=request.chat_template,

vllm/entrypoints/utils.py

+1 -4

@@ -17,8 +17,6 @@ from starlette.background import BackgroundTask, BackgroundTasks

18 from vllm import envs

19 from vllm.engine.arg_utils import EngineArgs

20 -from vllm.inputs import EmbedsPrompt, TokensPrompt

21 -from vllm.inputs.parse import get_prompt_len

20 from vllm.logger import current_formatter_type, init_logger

21 from vllm.platforms import current_platform

22 from vllm.utils.argparse_utils import FlexibleArgumentParser

@@ -189,7 +187,7 @@ def cli_env_setup():

187 def get_max_tokens(

188 max_model_len: int,

189 request: "CompletionRequest | ChatCompletionRequest | ResponsesRequest",

192 - prompt: TokensPrompt | EmbedsPrompt,

190 + input_length: int,

191 default_sampling_params: dict,

192 ) -> int:

193 # NOTE: Avoid isinstance() for better efficiency

@@ -204,7 +202,6 @@ def get_max_tokens(

202 # CompletionRequest (also a fallback for ChatCompletionRequest)

203 max_tokens = getattr(request, "max_tokens", None)

204

207 - input_length = get_prompt_len(prompt)

205 default_max_tokens = max_model_len - input_length

206 max_output_tokens = current_platform.get_max_output_tokens(input_length)

207

vllm/inputs/__init__.py

-6

@@ -16,11 +16,8 @@ from .data import (

16 TextPrompt,

17 TokenInputs,

18 TokensPrompt,

19 - build_explicit_enc_dec_prompt,

19 embeds_inputs,

21 - to_enc_dec_tuple_list,

20 token_inputs,

23 - zip_enc_dec_prompts,

21 )

23 __all__ = [

@@ -39,8 +36,5 @@ __all__ = [

36 "EncoderDecoderInputs",

37 "ProcessorInputs",

38 "SingletonInputs",

42 - "build_explicit_enc_dec_prompt",

43 - "to_enc_dec_tuple_list",

44 - "zip_enc_dec_prompts",

39 "StreamingInput",

40 ]

vllm/inputs/data.py

+133 -154

@@ -1,11 +1,10 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from collections.abc import Iterable

3 from dataclasses import dataclass

5 -from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, cast

4 +from typing import TYPE_CHECKING, Any, Literal, TypeAlias

6 import torch

8 -from typing_extensions import NotRequired, TypedDict, TypeVar

7 +from typing_extensions import NotRequired, TypedDict

9 from vllm.sampling_params import SamplingParams

@@ -23,7 +22,13 @@ else:

22 MultiModalUUIDDict = object

26 -class _CommonKeys(TypedDict):

25 +# Inputs to LLM API

26 +class _PromptOptions(TypedDict):

27 + """

28 + Additional options available to all

29 + [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt].

30 + """

31 +

32 multi_modal_data: NotRequired[MultiModalDataDict | None]

33 """

34 Optional multi-modal data to pass to the model,

@@ -53,14 +58,14 @@ class _CommonKeys(TypedDict):

58 """

56 -class TextPrompt(_CommonKeys):

61 +class TextPrompt(_PromptOptions):

62 """Schema for a text prompt."""

64 prompt: str

65 """The input text to be tokenized before passing to the model."""

63 -class TokensPrompt(_CommonKeys):

68 +class TokensPrompt(_PromptOptions):

69 """Schema for a tokenized prompt."""

71 prompt_token_ids: list[int]

@@ -73,7 +78,7 @@ class TokensPrompt(_CommonKeys):

78 """A list of token type IDs to pass to the cross encoder model."""

76 -class EmbedsPrompt(_CommonKeys):

81 +class EmbedsPrompt(_PromptOptions):

82 """Schema for a prompt provided via token embeddings."""

84 prompt_embeds: torch.Tensor

@@ -83,93 +88,113 @@ class EmbedsPrompt(_CommonKeys):

88 """The prompt text corresponding to the token embeddings, if available."""

86 -class DataPrompt(_CommonKeys):

87 - """Represents generic inputs handled by IO processor plugins."""

91 +DecoderOnlyPrompt: TypeAlias = (

92 + str | TextPrompt | list[int] | TokensPrompt | EmbedsPrompt

93 +)

94 +"""

95 +Schema of a prompt for a decoder-only model:

89 - data: Any

90 - """The input data"""

97 +- A text prompt (string or [`TextPrompt`][vllm.inputs.data.TextPrompt])

98 +- A tokenized prompt (list of token IDs, or

99 + [`TokensPrompt`][vllm.inputs.data.TokensPrompt])

100 +- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])

101

92 - data_format: str

93 - """The input data format"""

102 +For encoder-decoder models, passing a singleton prompt is shorthand for passing

103 +`ExplicitEncoderDecoderPrompt(encoder_prompt=prompt, decoder_prompt=None)`.

104 +"""

105

106

96 -SingletonPrompt: TypeAlias = str | TextPrompt | TokensPrompt | EmbedsPrompt

107 +EncoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt

108 +"""

109 +Schema of a prompt for the encoder part of a encoder-decoder model:

110 +

111 +- A text prompt (string or [`TextPrompt`][vllm.inputs.data.TextPrompt])

112 +- A tokenized prompt (list of token IDs, or

113 + [`TokensPrompt`][vllm.inputs.data.TokensPrompt])

114 """

98 -Set of possible schemas for a single prompt:

115

100 -- A text prompt ([`str`][] or [`TextPrompt`][vllm.inputs.data.TextPrompt])

101 -- A tokenized prompt ([`TokensPrompt`][vllm.inputs.data.TokensPrompt])

102 -- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])

116

104 -Note that "singleton" is as opposed to a data structure

105 -which encapsulates multiple prompts, i.e. of the sort

106 -which may be utilized for encoder/decoder models when

107 -the user desires to express both the encoder & decoder

108 -prompts explicitly, i.e.

109 -[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]

110 -

111 -A prompt of type [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] may be

112 -employed as (1) input to a decoder-only model, (2) input to

113 -the encoder of an encoder/decoder model, in the scenario

114 -where the decoder-prompt is not specified explicitly, or

115 -(3) as a member of a larger data structure encapsulating

116 -more than one prompt, i.e.

117 -[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]

117 +DecoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt

118 """

119 +Schema of a prompt for the decoder part of an encoder-decoder model:

120

121 +- A text prompt (string or [`TextPrompt`][vllm.inputs.data.TextPrompt])

122 +- A tokenized prompt (list of token IDs, or

123 + [`TokensPrompt`][vllm.inputs.data.TokensPrompt])

124

121 -_T1_co = TypeVar(

122 - "_T1_co", bound=SingletonPrompt, default=SingletonPrompt, covariant=True

123 -)

124 -_T2_co = TypeVar(

125 - "_T2_co", bound=SingletonPrompt, default=SingletonPrompt, covariant=True

126 -)

125 +Note:

126 + Multi-modal inputs are not supported for decoder prompts.

127 +"""

128

129

129 -# TODO: Make fields ReadOnly once mypy supports it

130 -class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):

130 +class ExplicitEncoderDecoderPrompt(TypedDict):

131 """

132 - Represents an encoder/decoder model input prompt,

133 - comprising an explicit encoder prompt and a decoder prompt.

134 -

135 - The encoder and decoder prompts, respectively, may be formatted

136 - according to any of the

137 - [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] schemas,

138 - and are not required to have the same schema.

139 -

140 - Only the encoder prompt may have multi-modal data. mm_processor_kwargs

141 - should be at the top-level, and should not be set in the encoder/decoder

142 - prompts, since they are agnostic to the encoder/decoder.

143 -

144 - Note that an

145 - [`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]

146 - may not be used as an input to a decoder-only model,

147 - and that the `encoder_prompt` and `decoder_prompt`

148 - fields of this data structure themselves must be

149 - [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] instances.

132 + Schema for a pair of encoder and decoder singleton prompts.

133 +

134 + Note:

135 + This schema is not valid for decoder-only models.

136 """

137

152 - encoder_prompt: _T1_co

138 + encoder_prompt: EncoderPrompt

139 + """The prompt for the encoder part of the model."""

140

154 - decoder_prompt: _T2_co | None

141 + decoder_prompt: DecoderPrompt | None

142 + """

143 + The prompt for the decoder part of the model.

144

156 - mm_processor_kwargs: NotRequired[dict[str, Any]]

145 + Passing `None` will cause the prompt to be inferred automatically.

146 + """

147

148

159 -PromptType: TypeAlias = SingletonPrompt | ExplicitEncoderDecoderPrompt[Any, Any]

149 +EncoderDecoderPrompt: TypeAlias = EncoderPrompt | ExplicitEncoderDecoderPrompt

150 """

161 -Set of possible schemas for an LLM input, including

162 -both decoder-only and encoder/decoder input types:

151 +Schema for a prompt for an encoder-decoder model.

152

164 -- A text prompt ([`str`][] or [`TextPrompt`][vllm.inputs.data.TextPrompt])

165 -- A tokenized prompt ([`TokensPrompt`][vllm.inputs.data.TokensPrompt])

166 -- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])

167 -- A single data structure containing both an encoder and a decoder prompt

168 - ([`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt])

153 +You can pass a singleton encoder prompt, in which case the decoder prompt is

154 +considered to be `None` (i.e., infer automatically).

155 +"""

156 +

157 +

158 +SingletonPrompt: TypeAlias = DecoderOnlyPrompt | EncoderPrompt | DecoderPrompt

159 +"""

160 +Schema for a single prompt. This is as opposed to a data structure

161 +which encapsulates multiple prompts, such as

162 +[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt].

163 """

164

165

172 -class TokenInputs(TypedDict):

166 +PromptType: TypeAlias = DecoderOnlyPrompt | EncoderDecoderPrompt

167 +"""

168 +Schema for any prompt, regardless of model type.

169 +

170 +This is the input format accepted by most [`LLM`][vllm.entrypoints.llm.LLM] APIs.

171 +"""

172 +

173 +

174 +class DataPrompt(_PromptOptions):

175 + """

176 + Represents generic inputs that are converted to

177 + [`PromptType`][vllm.inputs.data.PromptType] by IO processor plugins.

178 + """

179 +

180 + data: Any

181 + """The input data."""

182 +

183 + data_format: str

184 + """The input data format."""

185 +

186 +

187 +# Outputs of processor

188 +class _InputOptions(TypedDict):

189 + """

190 + Additional options available to all input types.

191 + """

192 +

193 + cache_salt: NotRequired[str]

194 + """Optional cache salt to be used for prefix caching."""

195 +

196 +

197 +class TokenInputs(_InputOptions):

198 """Represents token-based inputs."""

199

200 type: Literal["token"]

@@ -178,11 +203,6 @@ class TokenInputs(TypedDict):

203 prompt_token_ids: list[int]

204 """The token IDs of the prompt."""

205

181 - cache_salt: NotRequired[str]

182 - """

183 - Optional cache salt to be used for prefix caching.

184 - """

185 -

206

207 def token_inputs(

208 prompt_token_ids: list[int],

@@ -198,7 +218,7 @@ def token_inputs(

218 return inputs

219

220

201 -class EmbedsInputs(TypedDict):

221 +class EmbedsInputs(_InputOptions):

222 """Represents embeddings-based inputs."""

223

224 type: Literal["embeds"]

@@ -207,11 +227,6 @@ class EmbedsInputs(TypedDict):

227 prompt_embeds: torch.Tensor

228 """The embeddings of the prompt."""

229

210 - cache_salt: NotRequired[str]

211 - """

212 - Optional cache salt to be used for prefix caching.

213 - """

214 -

230

231 def embeds_inputs(

232 prompt_embeds: torch.Tensor,

@@ -229,96 +244,60 @@ def embeds_inputs(

244

245 DecoderOnlyInputs: TypeAlias = TokenInputs | EmbedsInputs | MultiModalInputs

246 """

232 -The inputs in [`LLMEngine`][vllm.engine.llm_engine.LLMEngine] before they are

233 -passed to the model executor.

234 -This specifies the data required for decoder-only models.

247 +A processed prompt from

248 +[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

249 +which can be passed to

250 +[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]

251 +for decoder-only models.

252 +"""

253 +

254 +

255 +EncoderInputs: TypeAlias = TokenInputs | MultiModalEncDecInputs

256 +"""

257 +A processed encoder prompt from

258 +[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

259 +which can be passed to

260 +[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]

261 +for encoder-decoder models.

262 +"""

263 +

264 +

265 +DecoderInputs: TypeAlias = TokenInputs | MultiModalInputs

266 +"""

267 +A processed decoder prompt from

268 +[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

269 +which can be passed to

270 +[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]

271 +for encoder-decoder models.

272 """

273

274

275 class EncoderDecoderInputs(TypedDict):

276 """

240 - The inputs in [`LLMEngine`][vllm.engine.llm_engine.LLMEngine] before they

241 - are passed to the model executor.

242 -

243 - This specifies the required data for encoder-decoder models.

277 + A processed pair of encoder and decoder singleton prompts.

278 + [`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

279 + which can be passed to

280 + [`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]

281 + for encoder-decoder models.

282 """

283

246 - encoder: TokenInputs | MultiModalEncDecInputs

284 + encoder: EncoderInputs

285 """The inputs for the encoder portion."""

286

249 - decoder: TokenInputs | MultiModalInputs

287 + decoder: DecoderInputs

288 """The inputs for the decoder portion."""

289

290

253 -SingletonInputs: TypeAlias = TokenInputs | EmbedsInputs | MultiModalInputs

254 -"""

255 -A processed [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] which can be

256 -passed to [`Sequence`][collections.abc.Sequence].

257 -"""

258 -

291 ProcessorInputs: TypeAlias = DecoderOnlyInputs | EncoderDecoderInputs

292 """

261 -The outputs from [`vllm.inputs.preprocess.InputPreprocessor`][].

293 +A processed prompt from

294 +[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

295 +which can be passed to

296 +[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor].

297 """

298

264 -_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)

265 -_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)

266 -

267 -

268 -def build_explicit_enc_dec_prompt(

269 - encoder_prompt: _T1,

270 - decoder_prompt: _T2 | None,

271 - mm_processor_kwargs: dict[str, Any] | None = None,

272 -) -> ExplicitEncoderDecoderPrompt[_T1, _T2]:

273 - if mm_processor_kwargs is None:

274 - mm_processor_kwargs = {}

275 - return ExplicitEncoderDecoderPrompt(

276 - encoder_prompt=encoder_prompt,

277 - decoder_prompt=decoder_prompt,

278 - mm_processor_kwargs=mm_processor_kwargs,

279 - )

280 -

281 -

282 -def zip_enc_dec_prompts(

283 - enc_prompts: Iterable[_T1],

284 - dec_prompts: Iterable[_T2 | None],

285 - mm_processor_kwargs: Iterable[dict[str, Any]] | dict[str, Any] | None = None,

286 -) -> list[ExplicitEncoderDecoderPrompt[_T1, _T2]]:

287 - """

288 - Zip encoder and decoder prompts together into a list of

289 - [`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]

290 - instances.

299

292 - `mm_processor_kwargs` may also be provided; if a dict is passed, the same

293 - dictionary will be used for every encoder/decoder prompt. If an iterable is

294 - provided, it will be zipped with the encoder/decoder prompts.

295 - """

296 - if mm_processor_kwargs is None:

297 - mm_processor_kwargs = cast(dict[str, Any], {})

298 - if isinstance(mm_processor_kwargs, dict):

299 - return [

300 - build_explicit_enc_dec_prompt(

301 - encoder_prompt,

302 - decoder_prompt,

303 - cast(dict[str, Any], mm_processor_kwargs),

304 - )

305 - for (encoder_prompt, decoder_prompt) in zip(enc_prompts, dec_prompts)

306 - ]

307 - return [

308 - build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt, mm_proc_kwargs)

309 - for (encoder_prompt, decoder_prompt, mm_proc_kwargs) in zip(

310 - enc_prompts, dec_prompts, mm_processor_kwargs

311 - )

312 - ]

313 -

314 -

315 -def to_enc_dec_tuple_list(

316 - enc_dec_prompts: Iterable[ExplicitEncoderDecoderPrompt[_T1, _T2]],

317 -) -> list[tuple[_T1, _T2 | None]]:

318 - return [

319 - (enc_dec_prompt["encoder_prompt"], enc_dec_prompt["decoder_prompt"])

320 - for enc_dec_prompt in enc_dec_prompts

321 - ]

300 +SingletonInputs: TypeAlias = DecoderOnlyInputs | MultiModalEncDecInputs

301

302

303 @dataclass

vllm/inputs/parse.py

+1 -109

@@ -1,88 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from typing import TYPE_CHECKING, Literal, NamedTuple, TypeAlias, TypedDict

5 -from typing_extensions import TypeIs

6 -

7 -from vllm.utils import length_from_prompt_token_ids_or_embeds

8 -

9 -from .data import (

10 - EmbedsPrompt,

11 - ExplicitEncoderDecoderPrompt,

12 - ProcessorInputs,

13 - PromptType,

14 - SingletonInputs,

15 - SingletonPrompt,

16 - TextPrompt,

17 - TokensPrompt,

18 -)

19 -

20 -if TYPE_CHECKING:

21 - import torch

22 -

23 -

24 -class ParsedStrPrompt(TypedDict):

25 - type: Literal["str"]

26 - content: str

27 -

28 -

29 -class ParsedTextPrompt(TypedDict):

30 - type: Literal["text"]

31 - content: TextPrompt

32 -

33 -

34 -class ParsedTokensPrompt(TypedDict):

35 - type: Literal["tokens"]

36 - content: TokensPrompt

37 -

38 -

39 -class ParsedEmbedsPrompt(TypedDict):

40 - type: Literal["embeds"]

41 - content: EmbedsPrompt

42 -

43 -

44 -ParsedSingletonPrompt: TypeAlias = (

45 - ParsedStrPrompt | ParsedTextPrompt | ParsedTokensPrompt | ParsedEmbedsPrompt

46 -)

47 -

48 -

49 -def parse_singleton_prompt(prompt: SingletonPrompt) -> ParsedSingletonPrompt:

50 - if isinstance(prompt, str):

51 - return ParsedStrPrompt(type="str", content=prompt)

52 - elif isinstance(prompt, dict):

53 - # Type ignores are because mypy does not correctly infer the TypedDicts

54 - # Pyright does succeed.

55 - if "prompt_embeds" in prompt:

56 - return ParsedEmbedsPrompt(type="embeds", content=prompt) # type: ignore[typeddict-item]

57 - elif "prompt_token_ids" in prompt:

58 - return ParsedTokensPrompt(type="tokens", content=prompt) # type: ignore[typeddict-item]

59 - elif "prompt" in prompt:

60 - return ParsedTextPrompt(type="text", content=prompt)

61 - raise TypeError(

62 - "inputs must be a string, TextPrompt, TokensPrompt, or EmbedsPrompt"

63 - )

64 -

65 -

66 -def is_explicit_encoder_decoder_prompt(

67 - prompt: PromptType,

68 -) -> TypeIs[ExplicitEncoderDecoderPrompt]:

69 - return isinstance(prompt, dict) and "encoder_prompt" in prompt

70 -

71 -

72 -def split_enc_dec_prompt(

73 - prompt: PromptType,

74 -) -> tuple[SingletonPrompt, SingletonPrompt | None]:

75 - if isinstance(prompt, str):

76 - return prompt, None

77 -

78 - if "encoder_prompt" in prompt and "decoder_prompt" in prompt:

79 - # NOTE: This passes pyright but not mypy

80 - return (

81 - prompt["encoder_prompt"], # type: ignore[typeddict-item]

82 - prompt["decoder_prompt"], # type: ignore[typeddict-item]

83 - )

84 -

85 - return prompt, None

4 +from .data import ProcessorInputs, SingletonInputs

7 def split_enc_dec_inputs(

@@ -96,30 +15,3 @@ def split_enc_dec_inputs(

15 )

17 return None, inputs

99 -

100 -

101 -class PromptComponents(NamedTuple):

102 - text: str | None = None

103 - token_ids: list[int] | None = None

104 - embeds: "torch.Tensor | None" = None

105 -

106 -

107 -def get_prompt_components(prompt: PromptType) -> PromptComponents:

108 - if isinstance(prompt, str):

109 - return PromptComponents(text=prompt)

110 -

111 - if encoder_prompt := prompt.get("encoder_prompt"):

112 - return get_prompt_components(encoder_prompt) # type: ignore[arg-type]

113 -

114 - return PromptComponents(

115 - text=prompt.get("prompt"), # type: ignore[arg-type]

116 - token_ids=prompt.get("prompt_token_ids"), # type: ignore[arg-type]

117 - embeds=prompt.get("prompt_embeds"),

118 - )

119 -

120 -

121 -def get_prompt_len(prompt: TokensPrompt | EmbedsPrompt):

122 - return length_from_prompt_token_ids_or_embeds(

123 - prompt.get("prompt_token_ids"), # type: ignore[arg-type]

124 - prompt.get("prompt_embeds"), # type: ignore[arg-type]

125 - )

vllm/inputs/preprocess.py

+66 -64

@@ -2,43 +2,51 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from collections.abc import Mapping

5 -from typing import Any

5 +from typing import Any, overload

7 from typing_extensions import assert_never

9 from vllm.config import ModelConfig, ObservabilityConfig

10 -from vllm.inputs.parse import split_enc_dec_prompt

10 from vllm.logger import init_logger

11 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

12 from vllm.multimodal.cache import BaseMultiModalProcessorCache

13 from vllm.multimodal.inputs import (

14 MultiModalDataDict,

16 - MultiModalEncDecInputs,

15 MultiModalInputs,

16 MultiModalUUIDDict,

17 )

18 from vllm.multimodal.processing import BaseMultiModalProcessor

19 from vllm.renderers import renderer_from_config

20 +from vllm.renderers.inputs import (

21 + DecoderDictPrompt,

22 + DecoderOnlyDictPrompt,

23 + DictPrompt,

24 + EncoderDecoderDictPrompt,

25 + EncoderDictPrompt,

26 + SingletonDictPrompt,

27 + TokPrompt,

28 +)

29 +from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt

30 from vllm.tokenizers import TokenizerLike

31 from vllm.utils.jsontree import json_iter_leaves

32 from vllm.v1.metrics.stats import MultiModalCacheStats

34 from .data import (

35 + DecoderInputs,

36 DecoderOnlyInputs,

37 EmbedsInputs,

38 EmbedsPrompt,

39 EncoderDecoderInputs,

40 + EncoderInputs,

41 ProcessorInputs,

42 PromptType,

43 SingletonInputs,

34 - SingletonPrompt,

44 TextPrompt,

45 TokenInputs,

46 TokensPrompt,

47 embeds_inputs,

48 token_inputs,

49 )

41 -from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt

51 logger = init_logger(__name__)

@@ -328,9 +336,36 @@ class InputPreprocessor:

336

337 return inputs

338

339 + @overload

340 def _prompt_to_llm_inputs(

341 self,

333 - prompt: SingletonPrompt,

342 + prompt: EncoderDictPrompt,

343 + tokenization_kwargs: dict[str, Any] | None = None,

344 + *,

345 + mm_uuids: MultiModalUUIDDict | None = None,

346 + ) -> EncoderInputs: ...

347 +

348 + @overload

349 + def _prompt_to_llm_inputs( # type: ignore[misc]

350 + self,

351 + prompt: DecoderDictPrompt,

352 + tokenization_kwargs: dict[str, Any] | None = None,

353 + *,

354 + mm_uuids: MultiModalUUIDDict | None = None,

355 + ) -> DecoderInputs: ...

356 +

357 + @overload

358 + def _prompt_to_llm_inputs( # type: ignore[misc]

359 + self,

360 + prompt: DecoderOnlyDictPrompt,

361 + tokenization_kwargs: dict[str, Any] | None = None,

362 + *,

363 + mm_uuids: MultiModalUUIDDict | None = None,

364 + ) -> DecoderOnlyInputs: ...

365 +

366 + def _prompt_to_llm_inputs(

367 + self,

368 + prompt: SingletonDictPrompt,

369 tokenization_kwargs: dict[str, Any] | None = None,

370 *,

371 mm_uuids: MultiModalUUIDDict | None = None,

@@ -346,34 +381,25 @@ class InputPreprocessor:

381

382 * [`SingletonInputs`][vllm.inputs.data.SingletonInputs] instance

383 """

349 - parsed = parse_singleton_prompt(prompt)

384 + if "prompt_embeds" in prompt:

385 + return self._process_embeds(prompt) # type: ignore[arg-type]

386

351 - if parsed["type"] == "embeds":

352 - return self._process_embeds(parsed["content"])

353 - if parsed["type"] == "tokens":

387 + if "prompt_token_ids" in prompt:

388 return self._process_tokens(

355 - parsed["content"],

389 + prompt, # type: ignore[arg-type]

390 mm_uuids=mm_uuids,

391 )

358 - if parsed["type"] == "text":

359 - return self._process_text(

360 - parsed["content"],

361 - tokenization_kwargs=tokenization_kwargs,

362 - mm_uuids=mm_uuids,

363 - )

364 - if parsed["type"] == "str":

392 +

393 + if "prompt" in prompt:

394 return self._process_text(

366 - TextPrompt(prompt=parsed["content"]),

395 + prompt, # type: ignore[arg-type]

396 tokenization_kwargs=tokenization_kwargs,

397 mm_uuids=mm_uuids,

398 )

399

371 - assert_never(parsed)

400 + assert_never(prompt) # type: ignore[arg-type]

401

373 - def _validate_enc_inputs(

374 - self,

375 - inputs: SingletonInputs,

376 - ) -> TokenInputs | MultiModalEncDecInputs:

402 + def _validate_enc_inputs(self, inputs: SingletonInputs) -> EncoderInputs:

403 if inputs["type"] == "embeds":

404 raise ValueError(

405 "Embedding inputs are not supported for encoder-decoder models"

@@ -387,10 +413,7 @@ class InputPreprocessor:

413

414 return inputs # type: ignore[return-value]

415

390 - def _validate_dec_inputs(

391 - self,

392 - inputs: SingletonInputs,

393 - ) -> TokenInputs | MultiModalInputs:

416 + def _validate_dec_inputs(self, inputs: SingletonInputs) -> DecoderInputs:

417 if inputs["type"] == "embeds":

418 raise ValueError(

419 "Embedding inputs are not supported for encoder-decoder models"

@@ -403,14 +426,15 @@ class InputPreprocessor:

426 encoder_inputs: SingletonInputs,

427 decoder_inputs: SingletonInputs | None = None,

428 ) -> EncoderDecoderInputs:

406 - if decoder_inputs is None:

407 - decoder_inputs = encoder_inputs

408 -

429 enc_inputs = self._validate_enc_inputs(encoder_inputs)

410 - dec_inputs = self._validate_dec_inputs(decoder_inputs)

430

412 - enc_inputs_new: TokenInputs | MultiModalEncDecInputs

413 - dec_inputs_new: TokenInputs | MultiModalInputs

431 + if decoder_inputs is None:

432 + dec_inputs: DecoderInputs = enc_inputs # type: ignore[assignment]

433 + else:

434 + dec_inputs = self._validate_dec_inputs(decoder_inputs)

435 +

436 + enc_inputs_new: EncoderInputs

437 + dec_inputs_new: DecoderInputs

438

439 if enc_inputs["type"] == "multimodal":

440 enc_inputs_new = token_inputs(enc_inputs["encoder_prompt_token_ids"])

@@ -437,7 +461,7 @@ class InputPreprocessor:

461

462 def _process_encoder_decoder_prompt(

463 self,

440 - prompt: PromptType,

464 + prompt: EncoderDecoderDictPrompt,

465 tokenization_kwargs: dict[str, Any] | None = None,

466 *,

467 mm_uuids: MultiModalUUIDDict | None = None,

@@ -448,24 +472,6 @@ class InputPreprocessor:

472 [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]

473 instance.

474

451 - There are two types of input prompts:

452 - singleton prompts which carry only the

453 - encoder prompt, and explicit encoder/decoder

454 - prompts which carry both the encoder and the

455 - decoder prompts as member variables.

456 -

457 - This function handles the following scenarios:

458 - * Singleton encoder prompt: extract encoder prompt

459 - token ids & infer default decoder prompt token ids

460 - * Explicit encoder/decoder prompt: extract encoder

461 - and decoder prompt token ids

462 -

463 - Note that for Explicit encoder/decoder prompts,

464 - each sub-prompt (encoder or decoder prompt) can

465 - have any possible singleton type; thus this

466 - method relies on helper functions to obtain

467 - token ids for the sub-prompts.

468 -

475 Arguments:

476

477 * prompt: an input prompt

@@ -475,7 +481,8 @@ class InputPreprocessor:

481 * [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]

482 instance

483 """

478 - encoder_prompt, decoder_prompt = split_enc_dec_prompt(prompt)

484 + encoder_prompt = prompt["encoder_prompt"]

485 + decoder_prompt = prompt["decoder_prompt"]

486

487 return self._build_enc_dec_inputs(

488 encoder_inputs=self._prompt_to_llm_inputs(

@@ -495,7 +502,7 @@ class InputPreprocessor:

502

503 def _process_decoder_only_prompt(

504 self,

498 - prompt: SingletonPrompt,

505 + prompt: DecoderOnlyDictPrompt,

506 tokenization_kwargs: dict[str, Any] | None = None,

507 *,

508 mm_uuids: MultiModalUUIDDict | None = None,

@@ -521,7 +528,7 @@ class InputPreprocessor:

528

529 def _preprocess(

530 self,

524 - prompt: PromptType,

531 + prompt: PromptType | DictPrompt | TokPrompt,

532 tokenization_kwargs: dict[str, Any] | None = None,

533 *,

534 mm_uuids: MultiModalUUIDDict | None = None,

@@ -530,25 +537,20 @@ class InputPreprocessor:

537 # Encoder-decoder model requires special mapping of

538 # input prompts to encoder & decoder.

539 return self._process_encoder_decoder_prompt(

533 - prompt,

540 + parse_enc_dec_prompt(prompt),

541 tokenization_kwargs,

542 mm_uuids=mm_uuids,

543 )

544

538 - if is_explicit_encoder_decoder_prompt(prompt):

539 - raise ValueError(

540 - "Cannot pass encoder-decoder prompt to decoder-only models"

541 - )

542 -

545 return self._process_decoder_only_prompt(

544 - prompt,

546 + parse_dec_only_prompt(prompt),

547 tokenization_kwargs=tokenization_kwargs,

548 mm_uuids=mm_uuids,

549 )

550

551 def preprocess(

552 self,

551 - prompt: PromptType,

553 + prompt: PromptType | DictPrompt | TokPrompt,

554 tokenization_kwargs: dict[str, Any] | None = None,

555 *,

556 mm_uuids: MultiModalUUIDDict | None = None,

vllm/multimodal/inputs.py

+6 -7

@@ -20,7 +20,7 @@ from typing import (

21 import numpy as np

22 from PIL.Image import Image

23 -from typing_extensions import NotRequired, TypeVar

23 +from typing_extensions import TypeVar

25 from vllm.utils.collection_utils import is_list_of

26 from vllm.utils.import_utils import LazyLoader

@@ -32,9 +32,13 @@ if TYPE_CHECKING:

32 import torch

33 import torch.types

34 from transformers.feature_extraction_utils import BatchFeature

35 +

36 + from vllm.inputs.data import _InputOptions

37 else:

38 torch = LazyLoader("torch", globals(), "torch")

40 + _InputOptions = dict

41 +

42 _T = TypeVar("_T")

44 HfImageItem: TypeAlias = Union["Image", np.ndarray, "torch.Tensor"]

@@ -1059,7 +1063,7 @@ A dictionary containing per-item placeholder ranges for each modality.

1063 """

1064

1065

1062 -class MultiModalInputs(TypedDict):

1066 +class MultiModalInputs(_InputOptions):

1067 """

1068 Represents the outputs of

1069 [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor],

@@ -1084,11 +1088,6 @@ class MultiModalInputs(TypedDict):

1088 `prompt_token_ids`.

1089 """

1090

1087 - cache_salt: NotRequired[str]

1088 - """

1089 - Optional cache salt to be used for prefix caching.

1090 - """

1091 -

1091

1092 class MultiModalEncDecInputs(MultiModalInputs):

1093 """

vllm/platforms/interface.py

+2 -1

@@ -19,6 +19,7 @@ if TYPE_CHECKING:

19 from vllm.config import VllmConfig

20 from vllm.inputs import ProcessorInputs, PromptType

21 from vllm.pooling_params import PoolingParams

22 + from vllm.renderers.inputs import DictPrompt, TokPrompt

23 from vllm.sampling_params import SamplingParams

24 from vllm.utils.argparse_utils import FlexibleArgumentParser

25 from vllm.v1.attention.selector import AttentionSelectorConfig

@@ -565,7 +566,7 @@ class Platform:

566 @classmethod

567 def validate_request(

568 cls,

568 - prompt: "PromptType",

569 + prompt: "PromptType | DictPrompt | TokPrompt",

570 params: "SamplingParams | PoolingParams",

571 processed_inputs: "ProcessorInputs",

572 ) -> None:

vllm/renderers/deepseek_v32.py

+6 -5

@@ -9,11 +9,12 @@ from vllm.entrypoints.chat_utils import (

9 parse_chat_messages,

10 parse_chat_messages_async,

11 )

12 -from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

12 from vllm.logger import init_logger

13 from vllm.tokenizers import cached_get_tokenizer

14 from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer

16 +from .inputs import DictPrompt

17 +from .inputs.preprocess import parse_dec_only_prompt

18 from .params import ChatParams

19 from .protocol import BaseRenderer

@@ -61,7 +62,7 @@ class DeepseekV32Renderer(BaseRenderer):

62 self,

63 messages: list[ChatCompletionMessageParam],

64 params: ChatParams,

64 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

65 + ) -> tuple[list[ConversationMessage], DictPrompt]:

66 tokenizer = self.get_tokenizer()

67 conversation, mm_data, mm_uuids = parse_chat_messages(

68 messages,

@@ -75,7 +76,7 @@ class DeepseekV32Renderer(BaseRenderer):

76 **params.get_apply_chat_template_kwargs(),

77 )

78 - prompt = self.render_completion(prompt_raw)

79 + prompt = parse_dec_only_prompt(prompt_raw)

80 if mm_data is not None:

81 prompt["multi_modal_data"] = mm_data

82 if mm_uuids is not None:

@@ -87,7 +88,7 @@ class DeepseekV32Renderer(BaseRenderer):

88 self,

89 messages: list[ChatCompletionMessageParam],

90 params: ChatParams,

90 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

91 + ) -> tuple[list[ConversationMessage], DictPrompt]:

92 tokenizer = self.get_tokenizer()

93 conversation, mm_data, mm_uuids = await parse_chat_messages_async(

94 messages,

@@ -101,7 +102,7 @@ class DeepseekV32Renderer(BaseRenderer):

102 **params.get_apply_chat_template_kwargs(),

103 )

104

104 - prompt = self.render_completion(prompt_raw)

105 + prompt = parse_dec_only_prompt(prompt_raw)

106 if mm_data is not None:

107 prompt["multi_modal_data"] = mm_data

108 if mm_uuids is not None:

vllm/renderers/grok2.py

+6 -5

@@ -9,11 +9,12 @@ from vllm.entrypoints.chat_utils import (

9 parse_chat_messages,

10 parse_chat_messages_async,

11 )

12 -from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

12 from vllm.logger import init_logger

13 from vllm.tokenizers import cached_get_tokenizer

14 from vllm.tokenizers.grok2 import Grok2Tokenizer

16 +from .inputs import DictPrompt

17 +from .inputs.preprocess import parse_dec_only_prompt

18 from .params import ChatParams

19 from .protocol import BaseRenderer

@@ -61,7 +62,7 @@ class Grok2Renderer(BaseRenderer):

62 self,

63 messages: list[ChatCompletionMessageParam],

64 params: ChatParams,

64 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

65 + ) -> tuple[list[ConversationMessage], DictPrompt]:

66 tokenizer = self.get_tokenizer()

67 conversation, mm_data, mm_uuids = parse_chat_messages(

68 messages,

@@ -75,7 +76,7 @@ class Grok2Renderer(BaseRenderer):

76 **params.get_apply_chat_template_kwargs(),

77 )

78 - prompt = self.render_completion(prompt_raw)

79 + prompt = parse_dec_only_prompt(prompt_raw)

80 if mm_data is not None:

81 prompt["multi_modal_data"] = mm_data

82 if mm_uuids is not None:

@@ -87,7 +88,7 @@ class Grok2Renderer(BaseRenderer):

88 self,

89 messages: list[ChatCompletionMessageParam],

90 params: ChatParams,

90 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

91 + ) -> tuple[list[ConversationMessage], DictPrompt]:

92 tokenizer = self.get_tokenizer()

93 conversation, mm_data, mm_uuids = await parse_chat_messages_async(

94 messages,

@@ -101,7 +102,7 @@ class Grok2Renderer(BaseRenderer):

102 **params.get_apply_chat_template_kwargs(),

103 )

104

104 - prompt = self.render_completion(prompt_raw)

105 + prompt = parse_dec_only_prompt(prompt_raw)

106 if mm_data is not None:

107 prompt["multi_modal_data"] = mm_data

108 if mm_uuids is not None:

vllm/renderers/hf.py

+6 -5

@@ -25,7 +25,6 @@ from vllm.entrypoints.chat_utils import (

25 parse_chat_messages,

26 parse_chat_messages_async,

27 )

28 -from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

28 from vllm.logger import init_logger

29 from vllm.tokenizers import cached_get_tokenizer

30 from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer

@@ -33,6 +32,8 @@ from vllm.transformers_utils.chat_templates import get_chat_template_fallback_pa

32 from vllm.transformers_utils.processor import cached_get_processor

33 from vllm.utils.func_utils import supports_kw

35 +from .inputs import DictPrompt

36 +from .inputs.preprocess import parse_dec_only_prompt

37 from .params import ChatParams

38 from .protocol import BaseRenderer

@@ -632,7 +633,7 @@ class HfRenderer(BaseRenderer):

633 self,

634 messages: list[ChatCompletionMessageParam],

635 params: ChatParams,

635 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

636 + ) -> tuple[list[ConversationMessage], DictPrompt]:

637 model_config = self.config

638 tokenizer = self.get_tokenizer()

639

@@ -674,7 +675,7 @@ class HfRenderer(BaseRenderer):

675 video_placeholder,

676 )

677

677 - prompt = self.render_completion(prompt_raw)

678 + prompt = parse_dec_only_prompt(prompt_raw)

679 if mm_data is not None:

680 prompt["multi_modal_data"] = mm_data

681 if mm_uuids is not None:

@@ -686,7 +687,7 @@ class HfRenderer(BaseRenderer):

687 self,

688 messages: list[ChatCompletionMessageParam],

689 params: ChatParams,

689 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

690 + ) -> tuple[list[ConversationMessage], DictPrompt]:

691 model_config = self.config

692 tokenizer = self.get_tokenizer()

693

@@ -726,7 +727,7 @@ class HfRenderer(BaseRenderer):

727 video_placeholder,

728 )

729

729 - prompt = self.render_completion(prompt_raw)

730 + prompt = parse_dec_only_prompt(prompt_raw)

731 if mm_data is not None:

732 prompt["multi_modal_data"] = mm_data

733 if mm_uuids is not None:

vllm/renderers/inputs/__init__.py new

+33

@@ -0,0 +1,33 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from .preprocess import (

4 + DecoderDictPrompt,

5 + DecoderOnlyDictPrompt,

6 + DictPrompt,

7 + EncoderDecoderDictPrompt,

8 + EncoderDictPrompt,

9 + SingletonDictPrompt,

10 +)

11 +from .tokenize import (

12 + DecoderOnlyTokPrompt,

13 + DecoderTokPrompt,

14 + EncoderDecoderTokPrompt,

15 + EncoderTokPrompt,

16 + SingletonTokPrompt,

17 + TokPrompt,

18 +)

19 +

20 +__all__ = [

21 + "DecoderOnlyDictPrompt",

22 + "EncoderDictPrompt",

23 + "DecoderDictPrompt",

24 + "EncoderDecoderDictPrompt",

25 + "SingletonDictPrompt",

26 + "DictPrompt",

27 + "DecoderOnlyTokPrompt",

28 + "EncoderTokPrompt",

29 + "DecoderTokPrompt",

30 + "EncoderDecoderTokPrompt",

31 + "SingletonTokPrompt",

32 + "TokPrompt",

33 +]

vllm/renderers/inputs/preprocess.py new

+255

@@ -0,0 +1,255 @@

1 +"""

2 +Schemas and utilites for preprocessing inputs.

3 +"""

4 +

5 +# SPDX-License-Identifier: Apache-2.0

6 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

7 +from collections.abc import Sequence

8 +from typing import TYPE_CHECKING, NamedTuple, TypeAlias, TypedDict, overload

9 +

10 +from vllm.inputs import (

11 + EmbedsPrompt,

12 + ExplicitEncoderDecoderPrompt,

13 + PromptType,

14 + SingletonPrompt,

15 + TextPrompt,

16 + TokensPrompt,

17 +)

18 +from vllm.utils import length_from_prompt_token_ids_or_embeds

19 +from vllm.utils.collection_utils import is_list_of

20 +

21 +if TYPE_CHECKING:

22 + import torch

23 +

24 + from vllm.config import ModelConfig

25 + from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

26 +

27 +

28 +@overload

29 +def prompt_to_seq(

30 + prompt_or_prompts: SingletonPrompt | bytes | Sequence[SingletonPrompt | bytes],

31 +) -> Sequence[SingletonPrompt]: ...

32 +

33 +

34 +@overload

35 +def prompt_to_seq( # type: ignore[misc]

36 + prompt_or_prompts: ExplicitEncoderDecoderPrompt

37 + | Sequence[ExplicitEncoderDecoderPrompt],

38 +) -> Sequence[ExplicitEncoderDecoderPrompt]: ...

39 +

40 +

41 +@overload

42 +def prompt_to_seq( # type: ignore[misc]

43 + prompt_or_prompts: PromptType | Sequence[PromptType],

44 +) -> Sequence[PromptType]: ...

45 +

46 +

47 +def prompt_to_seq(

48 + prompt_or_prompts: PromptType | bytes | Sequence[PromptType | bytes],

49 +) -> Sequence[PromptType]:

50 + if isinstance(prompt_or_prompts, (dict, str, bytes)) or (

51 + len(prompt_or_prompts) > 0 and is_list_of(prompt_or_prompts, int)

52 + ):

53 + return [prompt_or_prompts] # type: ignore[list-item]

54 +

55 + return prompt_or_prompts # type: ignore[return-value]

56 +

57 +

58 +def conversation_to_seq(

59 + conversation_or_conversations: list["ChatCompletionMessageParam"]

60 + | Sequence[list["ChatCompletionMessageParam"]],

61 +) -> Sequence[list["ChatCompletionMessageParam"]]:

62 + if len(conversation_or_conversations) > 0 and is_list_of(

63 + conversation_or_conversations, dict

64 + ):

65 + return [conversation_or_conversations] # type: ignore[list-item]

66 +

67 + return conversation_or_conversations # type: ignore[return-value]

68 +

69 +

70 +DecoderOnlyDictPrompt: TypeAlias = TextPrompt | TokensPrompt | EmbedsPrompt

71 +"""

72 +A [`DecoderOnlyPrompt`][vllm.inputs.data.DecoderOnlyPrompt]

73 +that has been standardized into a dictionary.

74 +"""

75 +

76 +

77 +EncoderDictPrompt: TypeAlias = TextPrompt | TokensPrompt

78 +"""

79 +A [`EncoderPrompt`][vllm.inputs.data.EncoderPrompt]

80 +that has been standardized into a dictionary.

81 +"""

82 +

83 +

84 +DecoderDictPrompt: TypeAlias = TextPrompt | TokensPrompt

85 +"""

86 +A [`DecoderPrompt`][vllm.inputs.data.DecoderPrompt]

87 +that has been standardized into a dictionary.

88 +"""

89 +

90 +

91 +class EncoderDecoderDictPrompt(TypedDict):

92 + """

93 + A [`EncoderDecoderPrompt`][vllm.inputs.data.EncoderDecoderPrompt]

94 + that has been standardized into a dictionary.

95 + """

96 +

97 + encoder_prompt: EncoderDictPrompt

98 +

99 + decoder_prompt: DecoderDictPrompt | None

100 +

101 +

102 +SingletonDictPrompt: TypeAlias = (

103 + DecoderOnlyDictPrompt | EncoderDictPrompt | DecoderDictPrompt

104 +)

105 +"""

106 +A [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt]

107 +that has been standardized into a dictionary.

108 +"""

109 +

110 +

111 +DictPrompt: TypeAlias = DecoderOnlyDictPrompt | EncoderDecoderDictPrompt

112 +"""

113 +A [`PromptType`][vllm.inputs.data.PromptType]

114 +that has been standardized into a dictionary.

115 +"""

116 +

117 +

118 +def parse_dec_only_prompt(prompt: object) -> DecoderOnlyDictPrompt:

119 + """

120 + Parse a prompt for a decoder-only model and normalize it to a dictionary.

121 + """

122 + if isinstance(prompt, str):

123 + return TextPrompt(prompt=prompt)

124 +

125 + if isinstance(prompt, list):

126 + if not is_list_of(prompt, int):

127 + raise TypeError("Token prompt should be a list of integers")

128 +

129 + return TokensPrompt(prompt_token_ids=prompt)

130 +

131 + if isinstance(prompt, dict):

132 + if "encoder_prompt" in prompt:

133 + raise TypeError("Cannot pass encoder-decoder prompt to decoder-only models")

134 +

135 + if (

136 + "prompt" in prompt

137 + or "prompt_token_ids" in prompt

138 + or "prompt_embeds" in prompt

139 + ):

140 + return prompt # type: ignore[return-value]

141 +

142 + raise TypeError("Prompt dictionary must contain text, tokens, or embeddings")

143 +

144 + raise TypeError("Prompt should be a string, list of tokens, or dictionary")

145 +

146 +

147 +def _parse_enc_prompt(prompt: object) -> EncoderDictPrompt:

148 + if isinstance(prompt, str):

149 + return TextPrompt(prompt=prompt)

150 +

151 + if isinstance(prompt, list):

152 + if not is_list_of(prompt, int):

153 + raise TypeError("Token prompt should be a list of integers")

154 +

155 + return TokensPrompt(prompt_token_ids=prompt)

156 +

157 + if isinstance(prompt, dict):

158 + if "prompt_embeds" in prompt:

159 + raise TypeError("Cannot pass embeddings prompt to encoder-decoder models")

160 +

161 + if "prompt" in prompt or "prompt_token_ids" in prompt:

162 + return prompt # type: ignore[return-value]

163 +

164 + raise TypeError("Prompt dictionary must contain text or tokens")

165 +

166 + raise TypeError("Prompt should be a string, list of tokens, or dictionary")

167 +

168 +

169 +def _parse_dec_prompt(prompt: object) -> DecoderDictPrompt:

170 + if isinstance(prompt, str):

171 + return TextPrompt(prompt=prompt)

172 +

173 + if isinstance(prompt, list):

174 + if not is_list_of(prompt, int):

175 + raise TypeError("Token prompt should be a list of integers")

176 +

177 + return TokensPrompt(prompt_token_ids=prompt)

178 +

179 + if isinstance(prompt, dict):

180 + if "prompt_embeds" in prompt:

181 + raise TypeError("Cannot pass embeddings prompt to encoder-decoder models")

182 +

183 + if (

184 + "multi_modal_data" in prompt

185 + or "mm_processor_kwargs" in prompt

186 + or "multi_modal_uuids" in prompt

187 + ):

188 + raise TypeError("Cannot pass multi-modal inputs to decoder prompt")

189 +

190 + if "prompt" in prompt or "prompt_token_ids" in prompt:

191 + return prompt # type: ignore[return-value]

192 +

193 + raise TypeError("Prompt dictionary must contain text or tokens")

194 +

195 + raise TypeError("Prompt should be a string, list of tokens, or dictionary")

196 +

197 +

198 +def parse_enc_dec_prompt(prompt: object) -> EncoderDecoderDictPrompt:

199 + """

200 + Parse a prompt for an encoder-decoder model and normalize it to a dictionary.

201 + """

202 + if isinstance(prompt, dict) and "encoder_prompt" in prompt:

203 + enc_prompt: object = prompt["encoder_prompt"] # type: ignore[typeddict-item]

204 + dec_prompt: object | None = prompt["decoder_prompt"] # type: ignore[typeddict-item]

205 + else:

206 + enc_prompt = prompt

207 + dec_prompt = None

208 +

209 + return EncoderDecoderDictPrompt(

210 + encoder_prompt=_parse_enc_prompt(enc_prompt),

211 + decoder_prompt=None if dec_prompt is None else _parse_dec_prompt(dec_prompt),

212 + )

213 +

214 +

215 +def parse_model_prompt(model_config: "ModelConfig", prompt: object):

216 + if model_config.is_encoder_decoder:

217 + return parse_enc_dec_prompt(prompt)

218 +

219 + return parse_dec_only_prompt(prompt)

220 +

221 +

222 +class PromptComponents(NamedTuple):

223 + text: str | None = None

224 + token_ids: list[int] | None = None

225 + embeds: "torch.Tensor | None" = None

226 +

227 +

228 +def extract_prompt_components(

229 + model_config: "ModelConfig",

230 + prompt: object,

231 +) -> PromptComponents:

232 + target_prompt = (

233 + parse_enc_dec_prompt(prompt)["encoder_prompt"]

234 + if model_config.is_encoder_decoder

235 + else parse_dec_only_prompt(prompt)

236 + )

237 +

238 + return PromptComponents(

239 + text=target_prompt.get("prompt"),

240 + token_ids=target_prompt.get("prompt_token_ids"), # type: ignore[arg-type]

241 + embeds=target_prompt.get("prompt_embeds"),

242 + )

243 +

244 +

245 +def extract_prompt_len(model_config: "ModelConfig", prompt: object):

246 + target_prompt = (

247 + parse_enc_dec_prompt(prompt)["encoder_prompt"]

248 + if model_config.is_encoder_decoder

249 + else parse_dec_only_prompt(prompt)

250 + )

251 +

252 + return length_from_prompt_token_ids_or_embeds(

253 + target_prompt.get("prompt_token_ids"), # type: ignore[arg-type]

254 + target_prompt.get("prompt_embeds"),

255 + )

vllm/renderers/inputs/tokenize.py new

+57

@@ -0,0 +1,57 @@

1 +"""

2 +Schemas and utilites for tokenization inputs.

3 +"""

4 +

5 +# SPDX-License-Identifier: Apache-2.0

6 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

7 +from typing import TypeAlias, TypedDict

8 +

9 +from vllm.inputs import EmbedsPrompt, TokensPrompt

10 +

11 +DecoderOnlyTokPrompt: TypeAlias = TokensPrompt | EmbedsPrompt

12 +"""

13 +A [`DecoderOnlyDictPrompt`][vllm.renderers.inputs.preprocess.DecoderOnlyDictPrompt]

14 +that has been tokenized.

15 +"""

16 +

17 +

18 +EncoderTokPrompt: TypeAlias = TokensPrompt

19 +"""

20 +A [`EncoderDictPrompt`][vllm.renderers.inputs.preprocess.EncoderDictPrompt]

21 +that has been tokenized.

22 +"""

23 +

24 +

25 +DecoderTokPrompt: TypeAlias = TokensPrompt

26 +"""

27 +A [`DecoderDictPrompt`][vllm.renderers.inputs.preprocess.DecoderDictPrompt]

28 +that has been tokenized.

29 +"""

30 +

31 +

32 +class EncoderDecoderTokPrompt(TypedDict):

33 + """

34 + A

35 + [`EncoderDecoderDictPrompt`][vllm.renderers.inputs.preprocess.EncoderDecoderDictPrompt]

36 + that has been tokenized.

37 + """

38 +

39 + encoder_prompt: EncoderTokPrompt

40 +

41 + decoder_prompt: DecoderTokPrompt | None

42 +

43 +

44 +SingletonTokPrompt: TypeAlias = (

45 + DecoderOnlyTokPrompt | EncoderTokPrompt | DecoderTokPrompt

46 +)

47 +"""

48 +A [`SingletonDictPrompt`][vllm.renderers.inputs.preprocess.SingletonDictPrompt]

49 +that has been tokenized.

50 +"""

51 +

52 +

53 +TokPrompt: TypeAlias = DecoderOnlyTokPrompt | EncoderDecoderTokPrompt

54 +"""

55 +A [`DictPrompt`][vllm.renderers.inputs.preprocess.DictPrompt]

56 +that has been tokenized.

57 +"""

vllm/renderers/mistral.py

+6 -5

@@ -10,12 +10,13 @@ from vllm.entrypoints.chat_utils import (

10 parse_chat_messages,

11 parse_chat_messages_async,

12 )

13 -from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

13 from vllm.logger import init_logger

14 from vllm.tokenizers import cached_get_tokenizer

15 from vllm.tokenizers.mistral import MistralTokenizer

16 from vllm.utils.async_utils import make_async

18 +from .inputs import DictPrompt

19 +from .inputs.preprocess import parse_dec_only_prompt

20 from .params import ChatParams

21 from .protocol import BaseRenderer

@@ -95,7 +96,7 @@ class MistralRenderer(BaseRenderer):

96 self,

97 messages: list[ChatCompletionMessageParam],

98 params: ChatParams,

98 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

99 + ) -> tuple[list[ConversationMessage], DictPrompt]:

100 tokenizer = self.get_tokenizer()

101 conversation, mm_data, mm_uuids = parse_chat_messages(

102 messages,

@@ -109,7 +110,7 @@ class MistralRenderer(BaseRenderer):

110 **params.get_apply_chat_template_kwargs(),

111 )

112

112 - prompt = self.render_completion(prompt_raw)

113 + prompt = parse_dec_only_prompt(prompt_raw)

114 if mm_data is not None:

115 prompt["multi_modal_data"] = mm_data

116 if mm_uuids is not None:

@@ -121,7 +122,7 @@ class MistralRenderer(BaseRenderer):

122 self,

123 messages: list[ChatCompletionMessageParam],

124 params: ChatParams,

124 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

125 + ) -> tuple[list[ConversationMessage], DictPrompt]:

126 tokenizer = self.get_tokenizer()

127 conversation, mm_data, mm_uuids = await parse_chat_messages_async(

128 messages,

@@ -135,7 +136,7 @@ class MistralRenderer(BaseRenderer):

136 **params.get_apply_chat_template_kwargs(),

137 )

138

138 - prompt = self.render_completion(prompt_raw)

139 + prompt = parse_dec_only_prompt(prompt_raw)

140 if mm_data is not None:

141 prompt["multi_modal_data"] = mm_data

142 if mm_uuids is not None:

vllm/renderers/protocol.py

+162 -79

@@ -2,14 +2,20 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import asyncio

4 from abc import ABC, abstractmethod

5 -from typing import TYPE_CHECKING, Any

5 +from collections.abc import Sequence

6 +from typing import TYPE_CHECKING, Any, overload

8 from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

9 from vllm.tokenizers import TokenizerLike

10 from vllm.utils.async_utils import AsyncMicrobatchTokenizer

10 -from vllm.utils.collection_utils import is_list_of

12 from .embed_utils import safe_load_prompt_embeds

13 +from .inputs import (

14 + DictPrompt,

15 + EncoderDecoderDictPrompt,

16 + EncoderDecoderTokPrompt,

17 + TokPrompt,

18 +)

19 from .params import ChatParams, TokenizeParams

21 if TYPE_CHECKING:

@@ -57,140 +63,217 @@ class BaseRenderer(ABC):

63 return self._async_tokenizer

65 # Step 1: Convert raw inputs to prompts

60 - def render_completion(

66 + def render_prompt(

67 self,

62 - prompt_raw: str | list[int] | bytes,

63 - ) -> TextPrompt | TokensPrompt | EmbedsPrompt:

64 - error_msg = "Each prompt must be a string or an array of tokens"

68 + prompt: DictPrompt | bytes,

69 + ) -> DictPrompt:

70 + if isinstance(prompt, bytes):

71 + embeds = safe_load_prompt_embeds(self.config, prompt)

72 + prompt = EmbedsPrompt(prompt_embeds=embeds)

66 - if isinstance(prompt_raw, str):

67 - return TextPrompt(prompt=prompt_raw)

74 + return prompt

69 - if isinstance(prompt_raw, list):

70 - if not is_list_of(prompt_raw, int):

71 - raise TypeError(error_msg)

72 -

73 - return TokensPrompt(prompt_token_ids=prompt_raw)

74 -

75 - if isinstance(prompt_raw, bytes):

76 - embeds = safe_load_prompt_embeds(self.config, prompt_raw)

77 - return EmbedsPrompt(prompt_embeds=embeds)

78 -

79 - raise TypeError(error_msg)

80 -

81 - def render_completions(

76 + def render_prompts(

77 self,

83 - prompt_input: str | list[str] | list[int] | list[list[int]] | None = None,

84 - prompt_embeds: bytes | list[bytes] | None = None,

85 - ) -> list[TextPrompt | TokensPrompt | EmbedsPrompt]:

86 - prompts_raw = list[str | list[int] | bytes]()

87 -

88 - if prompt_embeds is not None: # embeds take higher priority

89 - if isinstance(prompt_embeds, bytes):

90 - prompts_raw.append(prompt_embeds)

91 - else:

92 - prompts_raw.extend(prompt_embeds)

93 -

94 - if prompt_input is not None:

95 - if isinstance(prompt_input, str) or (

96 - len(prompt_input) > 0 and is_list_of(prompt_input, int)

97 - ):

98 - prompts_raw.append(prompt_input) # type: ignore[arg-type]

99 - else:

100 - prompts_raw.extend(prompt_input) # type: ignore[arg-type]

101 -

102 - if len(prompts_raw) == 0:

78 + prompts: Sequence[DictPrompt | bytes],

79 + ) -> list[DictPrompt]:

80 + if len(prompts) == 0:

81 raise ValueError("You must pass at least one prompt")

105 - return [self.render_completion(prompt) for prompt in prompts_raw]

83 + return [self.render_prompt(prompt) for prompt in prompts]

107 - async def render_completions_async(

85 + async def render_prompts_async(

86 self,

109 - prompt_input: str | list[str] | list[int] | list[list[int]] | None = None,

110 - prompt_embeds: bytes | list[bytes] | None = None,

111 - ) -> list[TextPrompt | TokensPrompt | EmbedsPrompt]:

112 - return self.render_completions(prompt_input, prompt_embeds)

87 + prompts: Sequence[DictPrompt | bytes],

88 + ) -> list[DictPrompt]:

89 + return self.render_prompts(prompts)

91 @abstractmethod

92 def render_messages(

93 self,

94 messages: list["ChatCompletionMessageParam"],

95 params: ChatParams,

119 - ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt | EmbedsPrompt]:

96 + ) -> tuple[list["ConversationMessage"], DictPrompt]:

97 raise NotImplementedError

99 async def render_messages_async(

100 self,

101 messages: list["ChatCompletionMessageParam"],

102 params: ChatParams,

126 - ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt | EmbedsPrompt]:

103 + ) -> tuple[list["ConversationMessage"], DictPrompt]:

104 return self.render_messages(messages, params)

105

106 # Step 2: Tokenize prompts if necessary

107 + def _tokenize_prompt(

108 + self,

109 + prompt: TextPrompt,

110 + params: TokenizeParams,

111 + ) -> TokensPrompt:

112 + tokenizer = self.get_tokenizer()

113 + prompt_token_ids = tokenizer.encode(

114 + prompt["prompt"],

115 + **params.get_encode_kwargs(),

116 + )

117 +

118 + return TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)

119 +

120 + async def _tokenize_prompt_async(

121 + self,

122 + prompt: TextPrompt,

123 + params: TokenizeParams,

124 + ) -> TokensPrompt:

125 + tokenizer = self.get_async_tokenizer()

126 + prompt_token_ids = await tokenizer.encode(

127 + prompt["prompt"],

128 + **params.get_encode_kwargs(),

129 + )

130 +

131 + return TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)

132 +

133 + def _detokenize_prompt(self, prompt: TokensPrompt) -> TokensPrompt:

134 + tokenizer = self.get_tokenizer()

135 + prompt["prompt"] = tokenizer.decode(prompt["prompt_token_ids"])

136 +

137 + return prompt

138 +

139 + async def _detokenize_prompt_async(self, prompt: TokensPrompt) -> TokensPrompt:

140 + tokenizer = self.get_async_tokenizer()

141 + prompt["prompt"] = await tokenizer.decode(prompt["prompt_token_ids"])

142 +

143 + return prompt

144 +

145 + def _tokenize_enc_dec_prompt(

146 + self,

147 + prompt: EncoderDecoderDictPrompt,

148 + params: TokenizeParams,

149 + ) -> EncoderDecoderTokPrompt:

150 + enc_prompt, dec_prompt = (

151 + self.tokenize_prompt(prompt["encoder_prompt"], params),

152 + (

153 + None

154 + if prompt["decoder_prompt"] is None

155 + else self.tokenize_prompt(prompt["decoder_prompt"], params)

156 + ),

157 + )

158 +

159 + return EncoderDecoderTokPrompt(

160 + encoder_prompt=enc_prompt,

161 + decoder_prompt=dec_prompt,

162 + )

163 +

164 + async def _tokenize_enc_dec_prompt_async(

165 + self,

166 + prompt: EncoderDecoderDictPrompt,

167 + params: TokenizeParams,

168 + ) -> EncoderDecoderTokPrompt:

169 + enc_prompt, dec_prompt = await asyncio.gather(

170 + self.tokenize_prompt_async(prompt["encoder_prompt"], params),

171 + (

172 + asyncio.sleep(0)

173 + if prompt["decoder_prompt"] is None

174 + else self.tokenize_prompt_async(prompt["decoder_prompt"], params)

175 + ),

176 + )

177 +

178 + return EncoderDecoderTokPrompt(

179 + encoder_prompt=enc_prompt,

180 + decoder_prompt=dec_prompt,

181 + )

182 +

183 + @overload

184 def tokenize_prompt(

185 self,

132 - prompt: TextPrompt | TokensPrompt | EmbedsPrompt,

186 + prompt: TextPrompt | TokensPrompt,

187 params: TokenizeParams,

134 - ) -> TokensPrompt | EmbedsPrompt:

135 - if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

136 - prompt = params.apply_pre_tokenization(self.tokenizer, prompt)

188 + ) -> TokensPrompt: ...

189

138 - tokenizer = self.get_tokenizer()

139 - prompt_token_ids = tokenizer.encode(

140 - prompt["prompt"],

141 - **params.get_encode_kwargs(),

142 - )

190 + @overload

191 + def tokenize_prompt( # type: ignore[misc]

192 + self,

193 + prompt: EmbedsPrompt,

194 + params: TokenizeParams,

195 + ) -> EmbedsPrompt: ...

196

144 - prompt = TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)

197 + @overload

198 + def tokenize_prompt( # type: ignore[misc]

199 + self,

200 + prompt: EncoderDecoderDictPrompt,

201 + params: TokenizeParams,

202 + ) -> EncoderDecoderTokPrompt: ...

203 +

204 + def tokenize_prompt(

205 + self,

206 + prompt: DictPrompt,

207 + params: TokenizeParams,

208 + ) -> TokPrompt:

209 + if "encoder_prompt" in prompt:

210 + return self._tokenize_enc_dec_prompt(prompt, params) # type: ignore[arg-type]

211 +

212 + if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

213 + prompt = params.apply_pre_tokenization(self.tokenizer, prompt)

214 + prompt = self._tokenize_prompt(prompt, params)

215

216 if params.needs_detokenization and "prompt" not in prompt:

217 if "prompt_token_ids" not in prompt:

218 raise RuntimeError("Cannot run detokenization on embeddings")

219

150 - tokenizer = self.get_tokenizer()

151 - prompt_text = tokenizer.decode(prompt["prompt_token_ids"]) # type: ignore[typeddict-item]

152 - prompt["prompt"] = prompt_text # type: ignore[typeddict-unknown-key]

220 + prompt = self._detokenize_prompt(prompt) # type: ignore[arg-type]

221

222 return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

223

224 def tokenize_prompts(

225 self,

158 - prompts: list[TextPrompt | TokensPrompt | EmbedsPrompt],

226 + prompts: Sequence[DictPrompt],

227 params: TokenizeParams,

160 - ) -> list[TokensPrompt | EmbedsPrompt]:

228 + ) -> list[TokPrompt]:

229 return [self.tokenize_prompt(prompt, params) for prompt in prompts]

230

231 + @overload

232 async def tokenize_prompt_async(

233 self,

165 - prompt: TextPrompt | TokensPrompt | EmbedsPrompt,

234 + prompt: TextPrompt | TokensPrompt,

235 params: TokenizeParams,

167 - ) -> TokensPrompt | EmbedsPrompt:

168 - if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

169 - prompt = params.apply_pre_tokenization(self.tokenizer, prompt)

236 + ) -> TokensPrompt: ...

237 +

238 + @overload

239 + async def tokenize_prompt_async( # type: ignore[misc]

240 + self,

241 + prompt: EmbedsPrompt,

242 + params: TokenizeParams,

243 + ) -> EmbedsPrompt: ...

244 +

245 + @overload

246 + async def tokenize_prompt_async( # type: ignore[misc]

247 + self,

248 + prompt: EncoderDecoderDictPrompt,

249 + params: TokenizeParams,

250 + ) -> EncoderDecoderTokPrompt: ...

251

171 - tokenizer = self.get_async_tokenizer()

172 - prompt_token_ids = await tokenizer.encode(

173 - prompt["prompt"],

174 - **params.get_encode_kwargs(),

175 - )

252 + async def tokenize_prompt_async(

253 + self,

254 + prompt: DictPrompt,

255 + params: TokenizeParams,

256 + ) -> TokPrompt:

257 + if "encoder_prompt" in prompt:

258 + return await self._tokenize_enc_dec_prompt_async(prompt, params) # type: ignore[arg-type]

259

177 - prompt = TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)

260 + if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

261 + prompt = params.apply_pre_tokenization(self.tokenizer, prompt)

262 + prompt = await self._tokenize_prompt_async(prompt, params)

263

264 if params.needs_detokenization and "prompt" not in prompt:

265 if "prompt_token_ids" not in prompt:

266 raise RuntimeError("Cannot run detokenization on embeddings")

267

183 - tokenizer = self.get_async_tokenizer()

184 - prompt_text = await tokenizer.decode(prompt["prompt_token_ids"]) # type: ignore[typeddict-item]

185 - prompt["prompt"] = prompt_text # type: ignore[typeddict-unknown-key]

268 + prompt = await self._detokenize_prompt_async(prompt) # type: ignore[arg-type]

269

270 return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

271

272 async def tokenize_prompts_async(

273 self,

191 - prompts: list[TextPrompt | TokensPrompt | EmbedsPrompt],

274 + prompts: Sequence[DictPrompt],

275 params: TokenizeParams,

193 - ) -> list[TokensPrompt | EmbedsPrompt]:

276 + ) -> list[TokPrompt]:

277 return await asyncio.gather(

278 *(self.tokenize_prompt_async(prompt, params) for prompt in prompts)

279 )

vllm/renderers/terratorch.py

+6 -5

@@ -9,10 +9,11 @@ from vllm.entrypoints.chat_utils import (

9 parse_chat_messages,

10 parse_chat_messages_async,

11 )

12 -from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

12 from vllm.logger import init_logger

13 from vllm.tokenizers import TokenizerLike

15 +from .inputs import DictPrompt

16 +from .inputs.preprocess import parse_dec_only_prompt

17 from .params import ChatParams

18 from .protocol import BaseRenderer

@@ -45,7 +46,7 @@ class TerratorchRenderer(BaseRenderer):

46 self,

47 messages: list[ChatCompletionMessageParam],

48 params: ChatParams,

48 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

49 + ) -> tuple[list[ConversationMessage], DictPrompt]:

50 model_config = self.config

52 conversation, mm_data, mm_uuids = parse_chat_messages(

@@ -54,7 +55,7 @@ class TerratorchRenderer(BaseRenderer):

55 content_format="string",

56 )

57 - prompt = self.render_completion([1]) # Dummy token IDs

58 + prompt = parse_dec_only_prompt([1]) # Dummy token IDs

59 if mm_data is not None:

60 prompt["multi_modal_data"] = mm_data

61 if mm_uuids is not None:

@@ -66,7 +67,7 @@ class TerratorchRenderer(BaseRenderer):

67 self,

68 messages: list[ChatCompletionMessageParam],

69 params: ChatParams,

69 - ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:

70 + ) -> tuple[list[ConversationMessage], DictPrompt]:

71 model_config = self.config

73 conversation, mm_data, mm_uuids = await parse_chat_messages_async(

@@ -75,7 +76,7 @@ class TerratorchRenderer(BaseRenderer):

76 content_format="string",

77 )

78 - prompt = self.render_completion([1]) # Dummy token IDs

79 + prompt = parse_dec_only_prompt([1]) # Dummy token IDs

80 if mm_data is not None:

81 prompt["multi_modal_data"] = mm_data

82 if mm_uuids is not None:

vllm/v1/engine/async_llm.py

+17 -6

@@ -28,6 +28,8 @@ from vllm.outputs import STREAM_FINISHED, PoolingRequestOutput, RequestOutput

28 from vllm.plugins.io_processors import get_io_processor

29 from vllm.pooling_params import PoolingParams

30 from vllm.renderers import BaseRenderer, merge_kwargs

31 +from vllm.renderers.inputs import DictPrompt, TokPrompt

32 +from vllm.renderers.inputs.preprocess import extract_prompt_components

33 from vllm.sampling_params import RequestOutputKind, SamplingParams

34 from vllm.tasks import SupportedTask

35 from vllm.tokenizers import TokenizerLike

@@ -42,7 +44,6 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError

44 from vllm.v1.engine.input_processor import InputProcessor

45 from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollector

46 from vllm.v1.engine.parallel_sampling import ParentRequest

45 -from vllm.v1.engine.utils import get_prompt_text

47 from vllm.v1.executor import Executor

48 from vllm.v1.metrics.loggers import (

49 StatLoggerFactory,

@@ -284,7 +285,11 @@ class AsyncLLM(EngineClient):

285 async def add_request(

286 self,

287 request_id: str,

287 - prompt: EngineCoreRequest | PromptType | AsyncGenerator[StreamingInput, None],

288 + prompt: EngineCoreRequest

289 + | PromptType

290 + | DictPrompt

291 + | TokPrompt

292 + | AsyncGenerator[StreamingInput, None],

293 params: SamplingParams | PoolingParams,

294 arrival_time: float | None = None,

295 lora_request: LoRARequest | None = None,

@@ -367,7 +372,7 @@ class AsyncLLM(EngineClient):

372 data_parallel_rank=data_parallel_rank,

373 supported_tasks=await self.get_supported_tasks(),

374 )

370 - prompt_text = get_prompt_text(prompt)

375 + prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)

376

377 self.input_processor.assign_request_id(request)

378

@@ -484,7 +489,9 @@ class AsyncLLM(EngineClient):

489 raise ValueError(

490 "prompt_embeds not supported for streaming inputs"

491 )

487 - prompt_text = get_prompt_text(input_chunk.prompt)

492 + prompt_text, _, _ = extract_prompt_components(

493 + self.model_config, input_chunk.prompt

494 + )

495 await self._add_request(req, prompt_text, None, 0, queue)

496 except (asyncio.CancelledError, GeneratorExit):

497 cancelled = True

@@ -528,7 +535,11 @@ class AsyncLLM(EngineClient):

535 # re-multiplexed in the API server anyhow.

536 async def generate(

537 self,

531 - prompt: EngineCoreRequest | PromptType | AsyncGenerator[StreamingInput, None],

538 + prompt: EngineCoreRequest

539 + | PromptType

540 + | DictPrompt

541 + | TokPrompt

542 + | AsyncGenerator[StreamingInput, None],

543 sampling_params: SamplingParams,

544 request_id: str,

545 *,

@@ -769,7 +780,7 @@ class AsyncLLM(EngineClient):

780

781 async def encode(

782 self,

772 - prompt: PromptType,

783 + prompt: PromptType | DictPrompt | TokPrompt,

784 pooling_params: PoolingParams,

785 request_id: str,

786 lora_request: LoRARequest | None = None,

vllm/v1/engine/input_processor.py

+18 -16

@@ -7,14 +7,13 @@ from typing import Any, Literal, cast

8 from vllm.config import VllmConfig

9 from vllm.exceptions import VLLMValidationError

10 -from vllm.inputs import (

10 +from vllm.inputs.data import (

11 ProcessorInputs,

12 PromptType,

13 SingletonInputs,

14 SingletonPrompt,

15 - TextPrompt,

15 )

17 -from vllm.inputs.parse import is_explicit_encoder_decoder_prompt, split_enc_dec_inputs

16 +from vllm.inputs.parse import split_enc_dec_inputs

17 from vllm.inputs.preprocess import InputPreprocessor

18 from vllm.logger import init_logger

19 from vllm.lora.request import LoRARequest

@@ -30,6 +29,7 @@ from vllm.multimodal.processing.context import set_request_id

29 from vllm.multimodal.utils import argsort_mm_positions

30 from vllm.pooling_params import PoolingParams

31 from vllm.renderers import BaseRenderer

32 +from vllm.renderers.inputs import DictPrompt, TokPrompt

33 from vllm.sampling_params import _SAMPLING_EPS, SamplingParams

34 from vllm.tasks import POOLING_TASKS, SupportedTask

35 from vllm.tokenizers import TokenizerLike

@@ -243,8 +243,8 @@ class InputProcessor:

243 return mm_processor.info.parse_mm_data(mm_data)

244

245 def _validate_singleton_mm_uuids(self, prompt: SingletonPrompt) -> None:

246 - if isinstance(prompt, str):

247 - prompt = TextPrompt(prompt=prompt)

246 + if not isinstance(prompt, dict):

247 + return

248

249 mm_data = cast(MultiModalDataDict, prompt.get("multi_modal_data") or {})

250 mm_uuids = cast(MultiModalUUIDDict, prompt.get("multi_modal_uuids") or {})

@@ -297,7 +297,7 @@ class InputProcessor:

297 f"multi_modal_uuids[{modality!r}] is missing."

298 )

299

300 - def _validate_mm_uuids(self, prompt: PromptType) -> None:

300 + def _validate_mm_uuids(self, prompt: PromptType | DictPrompt | TokPrompt) -> None:

301 """

302 Validate that user-provided multi_modal_uuids align with

303 multi_modal_data in the incoming request prompt(s).

@@ -305,10 +305,10 @@ class InputProcessor:

305 auto-hashed downstream.

306 """

307

308 - if is_explicit_encoder_decoder_prompt(prompt):

309 - self._validate_singleton_mm_uuids(prompt["encoder_prompt"])

308 + if isinstance(prompt, dict) and "encoder_prompt" in prompt:

309 + self._validate_singleton_mm_uuids(prompt["encoder_prompt"]) # type: ignore[typeddict-item]

310

311 - if (dec_prompt := prompt["decoder_prompt"]) is not None:

311 + if (dec_prompt := prompt["decoder_prompt"]) is not None: # type: ignore[typeddict-item]

312 self._validate_singleton_mm_uuids(dec_prompt)

313 else:

314 self._validate_singleton_mm_uuids(prompt)

@@ -449,21 +449,23 @@ class InputProcessor:

449 def _extract_singleton_mm_data(

450 self, prompt: SingletonPrompt

451 ) -> MultiModalDataDict | None:

452 - if isinstance(prompt, str):

452 + if not isinstance(prompt, dict):

453 return None

454

455 - return prompt.get("multi_modal_data") # type: ignore[return-value]

455 + return prompt.get("multi_modal_data")

456

457 - def _extract_mm_data(self, prompt: PromptType) -> MultiModalDataDict | None:

458 - if is_explicit_encoder_decoder_prompt(prompt):

459 - return self._extract_singleton_mm_data(prompt["encoder_prompt"])

457 + def _extract_mm_data(

458 + self, prompt: PromptType | DictPrompt | TokPrompt

459 + ) -> MultiModalDataDict | None:

460 + if isinstance(prompt, dict) and "encoder_prompt" in prompt:

461 + return self._extract_singleton_mm_data(prompt["encoder_prompt"]) # type: ignore[typeddict-item]

462 else:

463 return self._extract_singleton_mm_data(prompt)

464

465 def _maybe_build_mm_uuids(

466 self,

467 request_id: str,

466 - prompt: PromptType,

468 + prompt: PromptType | DictPrompt | TokPrompt,

469 ) -> MultiModalUUIDDict | None:

470 """Build per-item multimodal hash overrides when enabled. In this case,

471 multimodal data items are identified by their request id, modality and

@@ -519,7 +521,7 @@ class InputProcessor:

521 def process_inputs(

522 self,

523 request_id: str,

522 - prompt: PromptType,

524 + prompt: PromptType | DictPrompt | TokPrompt,

525 params: SamplingParams | PoolingParams,

526 arrival_time: float | None = None,

527 lora_request: LoRARequest | None = None,

vllm/v1/engine/llm_engine.py

+4 -3

@@ -22,6 +22,8 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput

22 from vllm.plugins.io_processors import get_io_processor

23 from vllm.pooling_params import PoolingParams

24 from vllm.renderers import BaseRenderer

25 +from vllm.renderers.inputs import DictPrompt, TokPrompt

26 +from vllm.renderers.inputs.preprocess import extract_prompt_components

27 from vllm.sampling_params import SamplingParams

28 from vllm.tasks import SupportedTask

29 from vllm.tokenizers import TokenizerLike

@@ -32,7 +34,6 @@ from vllm.v1.engine.core_client import EngineCoreClient

34 from vllm.v1.engine.input_processor import InputProcessor

35 from vllm.v1.engine.output_processor import OutputProcessor

36 from vllm.v1.engine.parallel_sampling import ParentRequest

35 -from vllm.v1.engine.utils import get_prompt_text

37 from vllm.v1.executor import Executor

38 from vllm.v1.metrics.loggers import StatLoggerFactory, StatLoggerManager

39 from vllm.v1.metrics.reader import Metric, get_metrics_snapshot

@@ -216,7 +217,7 @@ class LLMEngine:

217 def add_request(

218 self,

219 request_id: str,

219 - prompt: EngineCoreRequest | PromptType,

220 + prompt: EngineCoreRequest | PromptType | DictPrompt | TokPrompt,

221 params: SamplingParams | PoolingParams,

222 arrival_time: float | None = None,

223 lora_request: LoRARequest | None = None,

@@ -251,7 +252,7 @@ class LLMEngine:

252 priority,

253 supported_tasks=self.get_supported_tasks(),

254 )

254 - prompt_text = get_prompt_text(prompt)

255 + prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)

256

257 self.input_processor.assign_request_id(request)

258

vllm/v1/engine/utils.py

-6

@@ -17,8 +17,6 @@ import zmq

18 from vllm import envs

19 from vllm.config import CacheConfig, ParallelConfig, VllmConfig

20 -from vllm.inputs import PromptType

21 -from vllm.inputs.parse import get_prompt_components

20 from vllm.logger import init_logger

21 from vllm.platforms import current_platform

22 from vllm.ray.ray_env import get_env_vars_to_copy

@@ -226,10 +224,6 @@ def get_device_indices(

224 return value

225

226

229 -def get_prompt_text(prompt: PromptType) -> str | None:

230 - return get_prompt_components(prompt)[0]

231 -

232 -

227 class CoreEngineActorManager:

228 """

229 Utility class to handle creation, readiness, and shutdown