@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 574fe75245

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 05:00 UTC

[Renderer] Move InputPreprocessor into Renderer (2/2) (#34560)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 17, 2026 at 21:29 UTC 574fe75245fdadfa61b9c00b24dc84177540e3a5

32 files changed +983 -1053

tests/entrypoints/llm/test_chat.py

+6 -9

@@ -195,18 +195,15 @@ def test_chat_batch_failure_cleanup(llm_for_failure_test):

195 valid_msg = [{"role": "user", "content": "Hello"}]

196 long_text = "This is a very long text to test the error " * 50

197 invalid_msg = [{"role": "user", "content": long_text}]

198 - batch_1 = [

199 - valid_msg,

200 - valid_msg,

201 - invalid_msg,

202 - ]

203 - batch_2 = [

204 - valid_msg,

205 - valid_msg,

206 - ]

198 +

199 + batch_1 = [valid_msg, valid_msg, invalid_msg]

200 + batch_2 = [valid_msg, valid_msg]

201 sampling_params = SamplingParams(temperature=0, max_tokens=10)

202 +

203 with pytest.raises(ValueError, match="context length is only"):

204 llm.chat(batch_1, sampling_params=sampling_params)

205 + assert llm.llm_engine.get_num_unfinished_requests() == 0

206 +

207 outputs_2 = llm.chat(batch_2, sampling_params=sampling_params)

208 assert len(outputs_2) == len(batch_2)

209 assert llm.llm_engine.get_num_unfinished_requests() == 0

tests/models/multimodal/processing/test_common.py

+3 -2

@@ -489,8 +489,9 @@ def _assert_inputs_equal(

489 if ignore_mm_keys is None:

490 ignore_mm_keys = set()

491

492 - a_rest = {k: v for k, v in a.items() if k != "mm_kwargs"}

493 - b_rest = {k: v for k, v in b.items() if k != "mm_kwargs"}

492 + ignore_prompt_keys = ("prompt", "mm_kwargs")

493 + a_rest = {k: v for k, v in a.items() if k not in ignore_prompt_keys}

494 + b_rest = {k: v for k, v in b.items() if k not in ignore_prompt_keys}

495

496 assert a_rest == b_rest, msg

497

tests/renderers/test_process_multi_modal_uuids.py new

+165

@@ -0,0 +1,165 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import pytest

5 +

6 +from vllm.assets.image import ImageAsset

7 +from vllm.assets.video import VideoAsset

8 +from vllm.config import CacheConfig, ModelConfig, VllmConfig

9 +from vllm.renderers.hf import HfRenderer

10 +from vllm.tokenizers.registry import tokenizer_args_from_config

11 +

12 +cherry_pil_image = ImageAsset("cherry_blossom").pil_image

13 +stop_pil_image = ImageAsset("stop_sign").pil_image

14 +baby_reading_np_ndarrays = VideoAsset("baby_reading").np_ndarrays

15 +

16 +

17 +def _build_renderer(

18 + *, mm_cache_gb: float = 4.0, enable_prefix_caching: bool = True

19 +) -> HfRenderer:

20 + model_config = ModelConfig(

21 + model="Qwen/Qwen2.5-VL-3B-Instruct",

22 + max_model_len=128,

23 + mm_processor_cache_gb=mm_cache_gb,

24 + )

25 +

26 + vllm_config = VllmConfig(

27 + model_config=model_config,

28 + cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching),

29 + )

30 +

31 + _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)

32 +

33 + return HfRenderer.from_config(

34 + vllm_config,

35 + tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},

36 + )

37 +

38 +

39 +def test_multi_modal_uuids_length_mismatch_raises():

40 + renderer = _build_renderer()

41 +

42 + mm_data = {"image": [cherry_pil_image, stop_pil_image]}

43 +

44 + # Mismatch: 2 items but only 1 uuid provided

45 + mm_uuids = {"image": ["hash_cherry"]}

46 +

47 + mm_processor = renderer.get_mm_processor()

48 + mm_items = mm_processor.info.parse_mm_data(mm_data)

49 +

50 + with pytest.raises(ValueError, match="must have same length as"):

51 + renderer._process_mm_uuids(mm_data, mm_items, mm_uuids, "req-1")

52 +

53 +

54 +def test_multi_modal_uuids_missing_modality_raises():

55 + renderer = _build_renderer()

56 +

57 + mm_data = {

58 + "image": [cherry_pil_image],

59 + "video": None,

60 + }

61 +

62 + # Only image uuids provided; video missing should raise

63 + mm_uuids = {"image": ["hash_cherry"]}

64 +

65 + mm_processor = renderer.get_mm_processor()

66 + mm_items = mm_processor.info.parse_mm_data(mm_data)

67 +

68 + with pytest.raises(ValueError, match="is empty but .* is missing"):

69 + renderer._process_mm_uuids(mm_data, mm_items, mm_uuids, "req-2")

70 +

71 +

72 +@pytest.mark.parametrize(

73 + "mm_cache_gb, enable_prefix_caching",

74 + [

75 + (4.0, True), # default behavior

76 + (4.0, False), # prefix caching disabled

77 + (0.0, True), # processor cache disabled

78 + ],

79 +)

80 +def test_multi_modal_uuids_accepts_none_and_passes_through(

81 + monkeypatch, mm_cache_gb: float, enable_prefix_caching: bool

82 +):

83 + renderer = _build_renderer(

84 + mm_cache_gb=mm_cache_gb,

85 + enable_prefix_caching=enable_prefix_caching,

86 + )

87 +

88 + mm_data = {

89 + "image": [cherry_pil_image, stop_pil_image],

90 + "video": baby_reading_np_ndarrays,

91 + }

92 +

93 + # Use a consistent two-image scenario across all configurations

94 + mm_uuids = {"image": [None, "hash_stop"], "video": None}

95 +

96 + mm_processor = renderer.get_mm_processor()

97 + mm_items = mm_processor.info.parse_mm_data(mm_data)

98 + processed_mm_uuids = renderer._process_mm_uuids(

99 + mm_data, mm_items, mm_uuids, "req-3"

100 + )

101 +

102 + assert processed_mm_uuids == mm_uuids

103 +

104 +

105 +@pytest.mark.parametrize(

106 + "mm_cache_gb, enable_prefix_caching",

107 + [

108 + (4.0, True), # default behavior

109 + (4.0, False), # prefix caching disabled

110 + (0.0, True), # processor cache disabled

111 + ],

112 +)

113 +def test_multi_modal_uuids_accepts_empty(

114 + monkeypatch, mm_cache_gb: float, enable_prefix_caching: bool

115 +):

116 + renderer = _build_renderer(

117 + mm_cache_gb=mm_cache_gb,

118 + enable_prefix_caching=enable_prefix_caching,

119 + )

120 +

121 + # While None means cached multi-modal input requiring UUIDs

122 + # an empty list means no multi-modal input

123 + mm_data = {"image": [], "video": []} # type: ignore[var-annotated]

124 + mm_uuids = {"image": [], "video": None} # type: ignore[var-annotated]

125 +

126 + mm_processor = renderer.get_mm_processor()

127 + mm_items = mm_processor.info.parse_mm_data(mm_data)

128 + processed_mm_uuids = renderer._process_mm_uuids(

129 + mm_data, mm_items, mm_uuids, "req-4"

130 + )

131 +

132 + assert processed_mm_uuids == mm_uuids

133 +

134 +

135 +def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):

136 + # When both processor cache is 0 and prefix caching disabled, the

137 + # processor builds overrides from request id instead of using user UUIDs.

138 + renderer = _build_renderer(mm_cache_gb=0.0, enable_prefix_caching=False)

139 +

140 + request_id = "req-42"

141 + mm_data = {

142 + "image": [cherry_pil_image, stop_pil_image],

143 + "video": baby_reading_np_ndarrays,

144 + }

145 + mm_uuids = {"image": ["hash_cherry", "hash_stop"], "video": ["hash_video"]}

146 +

147 + mm_processor = renderer.get_mm_processor()

148 + mm_items = mm_processor.info.parse_mm_data(mm_data)

149 + processed_mm_uuids = renderer._process_mm_uuids(

150 + mm_data, mm_items, mm_uuids, request_id

151 + )

152 +

153 + # Expect request-id-based overrides are passed through

154 + assert set(mm_uuids.keys()) == {"image", "video"}

155 + assert len(mm_uuids["image"]) == 2

156 + assert len(mm_uuids["video"]) == 1

157 + assert processed_mm_uuids["image"][0].startswith(

158 + f"{request_id}-image-"

159 + ) and processed_mm_uuids["image"][0].endswith("-0")

160 + assert processed_mm_uuids["image"][1].startswith(

161 + f"{request_id}-image-"

162 + ) and processed_mm_uuids["image"][1].endswith("-1")

163 + assert processed_mm_uuids["video"][0].startswith(

164 + f"{request_id}-video-"

165 + ) and processed_mm_uuids["video"][0].endswith("-0")

tests/samplers/test_beam_search.py

-2

@@ -20,7 +20,6 @@ MM_BEAM_WIDTHS = [2]

20 MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]

23 -@pytest.mark.skip_v1 # V1 engine does not yet support beam search

23 @pytest.mark.parametrize("model", MODELS)

24 @pytest.mark.parametrize("dtype", ["half"])

25 @pytest.mark.parametrize("max_tokens", MAX_TOKENS)

@@ -62,7 +61,6 @@ def test_beam_search_single_input(

61 )

65 -@pytest.mark.skip_v1 # V1 engine does not yet support beam search

64 @pytest.mark.parametrize("model", MODELS)

65 @pytest.mark.parametrize("dtype", ["half"])

66 @pytest.mark.parametrize("max_tokens", MAX_TOKENS)

tests/v1/engine/test_process_multi_modal_uuids.py deleted

-174

@@ -1,174 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import pytest

5 -

6 -from vllm.assets.image import ImageAsset

7 -from vllm.assets.video import VideoAsset

8 -from vllm.config import CacheConfig, ModelConfig, VllmConfig

9 -from vllm.multimodal import MultiModalUUIDDict

10 -from vllm.sampling_params import SamplingParams

11 -from vllm.v1.engine.input_processor import InputProcessor

12 -

13 -cherry_pil_image = ImageAsset("cherry_blossom").pil_image

14 -stop_pil_image = ImageAsset("stop_sign").pil_image

15 -baby_reading_np_ndarrays = VideoAsset("baby_reading").np_ndarrays

16 -

17 -

18 -def _build_input_processor(

19 - *, mm_cache_gb: float = 4.0, enable_prefix_caching: bool = True

20 -) -> InputProcessor:

21 - model_config = ModelConfig(

22 - model="Qwen/Qwen2.5-VL-3B-Instruct",

23 - max_model_len=128,

24 - mm_processor_cache_gb=mm_cache_gb,

25 - )

26 -

27 - vllm_config = VllmConfig(

28 - model_config=model_config,

29 - cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching),

30 - )

31 -

32 - return InputProcessor(vllm_config)

33 -

34 -

35 -def test_multi_modal_uuids_length_mismatch_raises():

36 - input_processor = _build_input_processor()

37 -

38 - prompt = {

39 - "prompt": "USER: <image>\nDescribe\nASSISTANT:",

40 - "multi_modal_data": {"image": [cherry_pil_image, stop_pil_image]},

41 - # Mismatch: 2 items but only 1 uuid provided

42 - "multi_modal_uuids": {"image": ["hash_cherry"]},

43 - }

44 -

45 - with pytest.raises(ValueError, match="must have same length as"):

46 - input_processor.process_inputs(

47 - request_id="req-1",

48 - prompt=prompt, # type: ignore[arg-type]

49 - params=SamplingParams(),

50 - )

51 -

52 -

53 -def test_multi_modal_uuids_missing_modality_raises():

54 - input_processor = _build_input_processor()

55 -

56 - prompt = {

57 - "prompt": "USER: <image><video>\nDescribe\nASSISTANT:",

58 - # Two modalities provided in data

59 - "multi_modal_data": {

60 - "image": [cherry_pil_image],

61 - "video": None,

62 - },

63 - # Only image uuids provided; video missing should raise

64 - "multi_modal_uuids": {"image": ["hash_cherry"]},

65 - }

66 -

67 - with pytest.raises(ValueError, match="is empty but .* is missing"):

68 - input_processor.process_inputs(

69 - request_id="req-2",

70 - prompt=prompt, # type: ignore[arg-type]

71 - params=SamplingParams(),

72 - )

73 -

74 -

75 -@pytest.mark.parametrize(

76 - "mm_cache_gb, enable_prefix_caching",

77 - [

78 - (4.0, True), # default behavior

79 - (4.0, False), # prefix caching disabled

80 - (0.0, True), # processor cache disabled

81 - ],

82 -)

83 -def test_multi_modal_uuids_accepts_none_and_passes_through(

84 - monkeypatch, mm_cache_gb: float, enable_prefix_caching: bool

85 -):

86 - input_processor = _build_input_processor(

87 - mm_cache_gb=mm_cache_gb,

88 - enable_prefix_caching=enable_prefix_caching,

89 - )

90 -

91 - # Capture the overrides passed to InputPreprocessor.preprocess

92 - captured: dict[str, object] = {}

93 -

94 - def fake_preprocess(

95 - prompt, *, tokenization_kwargs=None, lora_request=None, mm_uuids=None

96 - ):

97 - captured["mm_uuids"] = mm_uuids

98 - # Minimal processed inputs for decoder-only flow

99 - return {"type": "token", "prompt_token_ids": [1]}

100 -

101 - # Monkeypatch only the bound preprocess method on this instance

102 - monkeypatch.setattr(

103 - input_processor.input_preprocessor, "preprocess", fake_preprocess, raising=True

104 - )

105 -

106 - # Use a consistent two-image scenario across all configurations

107 - mm_uuids = {"image": [None, "hash_stop"], "video": None}

108 - prompt = {

109 - "prompt": "USER: <image><image>\nTwo images\nASSISTANT:",

110 - "multi_modal_data": {

111 - "image": [cherry_pil_image, stop_pil_image],

112 - "video": baby_reading_np_ndarrays,

113 - },

114 - "multi_modal_uuids": mm_uuids,

115 - }

116 -

117 - input_processor.process_inputs(

118 - request_id="req-3",

119 - prompt=prompt, # type: ignore[arg-type]

120 - params=SamplingParams(),

121 - )

122 -

123 - assert captured["mm_uuids"] == mm_uuids

124 -

125 -

126 -def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):

127 - # When both processor cache is 0 and prefix caching disabled, the

128 - # processor builds overrides from request id instead of using user UUIDs.

129 - input_processor = _build_input_processor(

130 - mm_cache_gb=0.0, enable_prefix_caching=False

131 - )

132 -

133 - captured: dict[str, MultiModalUUIDDict] = {}

134 -

135 - def fake_preprocess(

136 - prompt, *, tokenization_kwargs=None, lora_request=None, mm_uuids=None

137 - ):

138 - captured["mm_uuids"] = mm_uuids

139 - return {"type": "token", "prompt_token_ids": [1]}

140 -

141 - monkeypatch.setattr(

142 - input_processor.input_preprocessor, "preprocess", fake_preprocess, raising=True

143 - )

144 -

145 - request_id = "req-42"

146 - mm_uuids = {"image": ["hash_cherry", "hash_stop"], "video": ["hash_video"]}

147 - prompt = {

148 - "prompt": "USER: <image><image><video>\nDescribe\nASSISTANT:",

149 - "multi_modal_data": {

150 - "image": [cherry_pil_image, stop_pil_image],

151 - "video": [baby_reading_np_ndarrays],

152 - },

153 - "multi_modal_uuids": mm_uuids,

154 - }

155 -

156 - input_processor.process_inputs(

157 - request_id=request_id,

158 - prompt=prompt, # type: ignore[arg-type]

159 - params=SamplingParams(),

160 - )

161 -

162 - # Expect request-id-based overrides are passed through

163 - assert set(mm_uuids.keys()) == {"image", "video"}

164 - assert len(mm_uuids["image"]) == 2

165 - assert len(mm_uuids["video"]) == 1

166 - assert captured["mm_uuids"]["image"][0].startswith(

167 - f"{request_id}-image-"

168 - ) and captured["mm_uuids"]["image"][0].endswith("-0")

169 - assert captured["mm_uuids"]["image"][1].startswith(

170 - f"{request_id}-image-"

171 - ) and captured["mm_uuids"]["image"][1].endswith("-1")

172 - assert captured["mm_uuids"]["video"][0].startswith(

173 - f"{request_id}-video-"

174 - ) and captured["mm_uuids"]["video"][0].endswith("-0")

vllm/beam_search.py

+29 -8

@@ -2,13 +2,11 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from dataclasses import dataclass

5 -from typing import TYPE_CHECKING, Any

6 +from vllm.inputs import TokenInputs, token_inputs

7 from vllm.logprobs import Logprob

8 from vllm.lora.request import LoRARequest

9 -

10 -if TYPE_CHECKING:

11 - from vllm.multimodal import MultiModalDataDict

9 +from vllm.multimodal.inputs import MultiModalInputs, mm_inputs

12 @dataclass

@@ -19,6 +17,8 @@ class BeamSearchSequence:

17 about to be returned to the user.

18 """

20 + orig_prompt: TokenInputs | MultiModalInputs

21 +

22 # The tokens include the prompt.

23 tokens: list[int]

24 logprobs: list[dict[int, Logprob]]

@@ -27,8 +27,28 @@ class BeamSearchSequence:

27 text: str | None = None

28 finish_reason: str | None = None

29 stop_reason: int | str | None = None

30 - multi_modal_data: "MultiModalDataDict | None" = None

31 - mm_processor_kwargs: dict[str, Any] | None = None

30 +

31 + def get_prompt(self):

32 + prompt = self.orig_prompt

33 +

34 + prompt_text = prompt.get("prompt")

35 + cache_salt = prompt.get("cache_salt")

36 +

37 + if prompt["type"] == "token":

38 + return token_inputs(

39 + self.tokens,

40 + prompt=prompt_text,

41 + cache_salt=cache_salt,

42 + )

43 +

44 + return mm_inputs(

45 + prompt_token_ids=self.tokens,

46 + mm_kwargs=prompt["mm_kwargs"],

47 + mm_hashes=prompt["mm_hashes"],

48 + mm_placeholders=prompt["mm_placeholders"],

49 + prompt=prompt_text,

50 + cache_salt=cache_salt,

51 + )

54 @dataclass

@@ -44,14 +64,15 @@ class BeamSearchOutput:

64 class BeamSearchInstance:

65 def __init__(

66 self,

47 - prompt_tokens: list[int],

67 + prompt: TokenInputs | MultiModalInputs,

68 lora_request: LoRARequest | None = None,

69 logprobs: list[dict[int, Logprob]] | None = None,

70 **kwargs,

71 ):

72 self.beams: list[BeamSearchSequence] = [

73 BeamSearchSequence(

54 - tokens=prompt_tokens,

74 + orig_prompt=prompt,

75 + tokens=prompt["prompt_token_ids"],

76 logprobs=[] if logprobs is None else list(logprobs),

77 lora_request=lora_request,

78 **kwargs,

vllm/engine/protocol.py

+6 -6

@@ -11,13 +11,12 @@ from vllm.distributed.weight_transfer.base import (

11 WeightTransferInitRequest,

12 WeightTransferUpdateRequest,

13 )

14 -from vllm.inputs.data import PromptType

14 +from vllm.inputs.data import ProcessorInputs, PromptType

15 from vllm.lora.request import LoRARequest

16 from vllm.outputs import PoolingRequestOutput, RequestOutput

17 from vllm.plugins.io_processors import IOProcessor

18 from vllm.pooling_params import PoolingParams

19 from vllm.renderers import BaseRenderer

20 -from vllm.renderers.inputs import DictPrompt, TokPrompt

20 from vllm.sampling_params import SamplingParams

21 from vllm.tasks import SupportedTask

22 from vllm.v1.engine import EngineCoreRequest

@@ -35,7 +34,7 @@ class StreamingInput:

34 where inputs are provided via an async generator.

35 """

38 - prompt: PromptType

37 + prompt: ProcessorInputs

38 sampling_params: SamplingParams | None = None

@@ -69,8 +68,7 @@ class EngineClient(ABC):

68 self,

69 prompt: EngineCoreRequest

70 | PromptType

72 - | DictPrompt

73 - | TokPrompt

71 + | ProcessorInputs

72 | AsyncGenerator[StreamingInput, None],

73 sampling_params: SamplingParams,

74 request_id: str,

@@ -81,6 +79,7 @@ class EngineClient(ABC):

79 trace_headers: Mapping[str, str] | None = None,

80 priority: int = 0,

81 data_parallel_rank: int | None = None,

82 + reasoning_ended: bool | None = None,

83 ) -> AsyncGenerator[RequestOutput, None]:

84 """Generate outputs for a request."""

85 ...

@@ -88,13 +87,14 @@ class EngineClient(ABC):

87 @abstractmethod

88 def encode(

89 self,

91 - prompt: PromptType | DictPrompt | TokPrompt,

90 + prompt: PromptType | ProcessorInputs,

91 pooling_params: PoolingParams,

92 request_id: str,

93 lora_request: LoRARequest | None = None,

94 trace_headers: Mapping[str, str] | None = None,

95 priority: int = 0,

96 tokenization_kwargs: dict[str, Any] | None = None,

97 + reasoning_ended: bool | None = None,

98 ) -> AsyncGenerator[PoolingRequestOutput, None]:

99 """Generate outputs for a request from a pooling model."""

100 ...

vllm/entrypoints/llm.py

+203 -221

@@ -3,8 +3,8 @@

4 import itertools

5 import warnings

6 -from collections.abc import Callable, Sequence

7 -from typing import TYPE_CHECKING, Any, cast

6 +from collections.abc import Callable, Iterable, Sequence

7 +from typing import TYPE_CHECKING, Any

9 import cloudpickle

10 import torch.nn as nn

@@ -55,6 +55,7 @@ from vllm.entrypoints.pooling.score.utils import (

55 from vllm.entrypoints.utils import log_non_default_args

56 from vllm.inputs.data import (

57 DataPrompt,

58 + ProcessorInputs,

59 PromptType,

60 SingletonPrompt,

61 TextPrompt,

@@ -73,10 +74,8 @@ from vllm.outputs import (

74 from vllm.platforms import current_platform

75 from vllm.pooling_params import PoolingParams

76 from vllm.renderers import ChatParams, merge_kwargs

76 -from vllm.renderers.inputs import DictPrompt, TokPrompt

77 from vllm.renderers.inputs.preprocess import (

78 conversation_to_seq,

79 - extract_prompt_components,

79 parse_model_prompt,

80 prompt_to_seq,

81 )

@@ -86,6 +85,7 @@ from vllm.tokenizers import TokenizerLike

85 from vllm.tokenizers.mistral import MistralTokenizer

86 from vllm.usage.usage_lib import UsageContext

87 from vllm.utils.counter import Counter

88 +from vllm.utils.tqdm_utils import maybe_tqdm

89 from vllm.v1.engine.llm_engine import LLMEngine

90 from vllm.v1.sample.logits_processor import LogitsProcessor

@@ -400,7 +400,7 @@ class LLM:

400 sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,

401 *,

402 use_tqdm: bool | Callable[..., tqdm] = True,

403 - lora_request: list[LoRARequest] | LoRARequest | None = None,

403 + lora_request: Sequence[LoRARequest] | LoRARequest | None = None,

404 priority: list[int] | None = None,

405 tokenization_kwargs: dict[str, Any] | None = None,

406 ) -> list[RequestOutput]:

@@ -462,7 +462,7 @@ class LLM:

462 self,

463 prompts: PromptType | Sequence[PromptType],

464 sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,

465 - lora_request: list[LoRARequest] | LoRARequest | None = None,

465 + lora_request: Sequence[LoRARequest] | LoRARequest | None = None,

466 priority: list[int] | None = None,

467 use_tqdm: bool | Callable[..., tqdm] = True,

468 tokenization_kwargs: dict[str, Any] | None = None,

@@ -495,34 +495,32 @@ class LLM:

495 # Use the same preprocessing as _run_completion

496 seq_prompts = prompt_to_seq(prompts)

497 seq_params = self._params_to_seq(sampling_params, len(seq_prompts))

498 -

499 - if any(param.truncate_prompt_tokens is not None for param in seq_params):

500 - engine_prompts: Sequence[DictPrompt | TokPrompt] = [

501 - engine_prompt

502 - for prompt, param in zip(seq_prompts, seq_params)

503 - for engine_prompt in self._preprocess_cmpl(

504 - [prompt],

505 - tokenization_kwargs=merge_kwargs(

506 - tokenization_kwargs,

507 - dict(truncate_prompt_tokens=param.truncate_prompt_tokens),

498 + seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts))

499 + seq_tok_kwargs = [

500 + merge_kwargs(

501 + tokenization_kwargs,

502 + dict(truncate_prompt_tokens=param.truncate_prompt_tokens),

503 + )

504 + for param in seq_params

505 + ]

506 + seq_priority = self._priority_to_seq(priority, len(prompts))

507 +

508 + request_ids = self._render_and_add_requests(

509 + prompts=(

510 + self._preprocess_cmpl_one(prompt, tok_kwargs)

511 + for prompt, tok_kwargs in zip(

512 + maybe_tqdm(

513 + seq_prompts,

514 + use_tqdm=use_tqdm,

515 + desc="Rendering prompts",

516 ),

517 + seq_tok_kwargs,

518 )

510 - ]

511 - else:

512 - engine_prompts = self._preprocess_cmpl(

513 - seq_prompts,

514 - tokenization_kwargs=tokenization_kwargs,

515 - )

516 -

517 - request_ids = self._validate_and_add_requests(

518 - prompts=engine_prompts,

519 - params=seq_params,

520 - use_tqdm=use_tqdm,

521 - lora_request=self._get_modality_specific_lora_reqs(

522 - engine_prompts, lora_request

519 ),

520 + params=seq_params,

521 + lora_requests=seq_lora_requests,

522 tokenization_kwargs=tokenization_kwargs,

525 - priority=priority,

523 + priorities=seq_priority,

524 )

525

526 return request_ids

@@ -545,53 +543,41 @@ class LLM:

543 outputs = self._run_engine(use_tqdm=use_tqdm)

544 return self.engine_class.validate_outputs(outputs, RequestOutput)

545

548 - def _get_modality_specific_lora_reqs(

546 + def _resolve_lora_reqs(

547 self,

550 - prompts: Sequence[DictPrompt | TokPrompt],

551 - lora_request: list[LoRARequest] | LoRARequest | None,

548 + prompts: Sequence[ProcessorInputs],

549 + lora_request: Sequence[LoRARequest | None] | LoRARequest | None,

550 ):

553 - # Grab the lora config off the vllm config on the engine,

554 - # since this is the same for both v0 & v1.

551 lora_config = self.llm_engine.vllm_config.lora_config

552 + seq_lora_requests = self._lora_request_to_seq(lora_request, len(prompts))

553

557 - # If there's no lora config / default_mm_loras, or the model

558 - # isn't multimodal, leave the lora as is.

554 if (

555 lora_config is None

556 or not self.model_config.is_multimodal_model

557 or (lora_config and lora_config.default_mm_loras is None)

558 ):

564 - return lora_request

565 -

566 - optional_loras = (

567 - [lora_request] * len(prompts)

568 - if not isinstance(lora_request, Sequence)

569 - else lora_request

570 - )

559 + return seq_lora_requests

560

561 return [

562 self._resolve_single_prompt_mm_lora(

563 prompt,

575 - opt_lora_req,

564 + lora_req,

565 lora_config.default_mm_loras,

566 )

578 - for prompt, opt_lora_req in zip(prompts, optional_loras)

567 + for prompt, lora_req in zip(prompts, seq_lora_requests)

568 ]

569

570 def _resolve_single_prompt_mm_lora(

571 self,

583 - prompt: DictPrompt | TokPrompt,

572 + prompt: ProcessorInputs,

573 lora_request: LoRARequest | None,

574 default_mm_loras: dict[str, str] | None,

575 ):

587 - if not default_mm_loras or not (

588 - mm_data := prompt.get("multi_modal_data") or {}

589 - ):

576 + if not default_mm_loras or prompt["type"] != "multimodal":

577 return lora_request

578

592 - intersection = set(

593 - mm_data.keys() # type: ignore

594 - ).intersection(default_mm_loras.keys())

579 + prompt_modalities = prompt["mm_placeholders"].keys()

580 + intersection = set(prompt_modalities).intersection(default_mm_loras.keys())

581 if not intersection:

582 return lora_request

583 if len(intersection) > 1:

@@ -674,22 +660,6 @@ class LLM:

660 """

661 return self.llm_engine.apply_model(func)

662

677 - def _get_beam_search_lora_requests(

678 - self,

679 - lora_request: list[LoRARequest] | LoRARequest | None,

680 - prompts: list[TokensPrompt | TextPrompt],

681 - ) -> list[LoRARequest | None]:

682 - """Get the optional lora request corresponding to each prompt."""

683 - if isinstance(lora_request, Sequence) and len(lora_request) != len(prompts):

684 - raise ValueError(

685 - "Lora request list should be the same length as the prompts"

686 - )

687 -

688 - if lora_request is None or isinstance(lora_request, LoRARequest):

689 - return [lora_request] * len(prompts)

690 -

691 - raise TypeError(f"Invalid lora_request type {type(lora_request)}")

692 -

663 def beam_search(

664 self,

665 prompts: list[TokensPrompt | TextPrompt],

@@ -718,13 +688,12 @@ class LLM:

688 ignore_eos = params.ignore_eos

689 length_penalty = params.length_penalty

690

721 - lora_requests = self._get_beam_search_lora_requests(lora_request, prompts)

691 + tokenizer = self.renderer.get_tokenizer()

692 + eos_token_id = tokenizer.eos_token_id

693 + sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)

694

723 - tokenizer = self.get_tokenizer()

724 - sort_beams_key = create_sort_beams_key_function(

725 - tokenizer.eos_token_id,

726 - length_penalty,

727 - )

695 + engine_prompts = self._preprocess_cmpl(prompts)

696 + lora_requests = self._lora_request_to_seq(lora_request, len(engine_prompts))

697

698 if use_tqdm and concurrency_limit is not None:

699 logger.warning(

@@ -734,21 +703,12 @@ class LLM:

703 use_tqdm = False

704

705 if concurrency_limit is None:

737 - concurrency_limit = len(prompts)

738 -

739 - def create_tokens_prompt_from_beam(beam: BeamSearchSequence) -> TokensPrompt:

740 - token_prompt_kwargs: TokensPrompt = {"prompt_token_ids": beam.tokens}

741 - if beam.multi_modal_data is not None:

742 - token_prompt_kwargs["multi_modal_data"] = beam.multi_modal_data

743 -

744 - if beam.mm_processor_kwargs is not None:

745 - token_prompt_kwargs["mm_processor_kwargs"] = beam.mm_processor_kwargs

746 - return TokensPrompt(**token_prompt_kwargs)

706 + concurrency_limit = len(engine_prompts)

707

708 # generate 2 * beam_width candidates at each step

709 # following the huggingface transformers implementation

710 # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa

751 - beam_search_params = SamplingParams(

711 + sampling_params = SamplingParams(

712 logprobs=2 * beam_width,

713 max_tokens=1,

714 temperature=temperature,

@@ -756,30 +716,25 @@ class LLM:

716 )

717 instances: list[BeamSearchInstance] = []

718

759 - for lora_req, prompt in zip(lora_requests, prompts):

760 - # Add multimodal processor kwargs & data

761 - mm_kwargs = {}

762 - if "multi_modal_data" in prompt:

763 - mm_kwargs["multi_modal_data"] = prompt["multi_modal_data"]

764 - if "mm_processor_kwargs" in prompt:

765 - mm_kwargs["mm_processor_kwargs"] = prompt["mm_processor_kwargs"]

766 -

767 - if "prompt_token_ids" in prompt:

768 - prompt = cast(TokensPrompt, prompt) # Needed for mypy

769 - prompt_tokens = prompt["prompt_token_ids"]

770 - else:

771 - prompt_tokens = tokenizer.encode(prompt["prompt"])

719 + for lora_req, prompt in zip(lora_requests, engine_prompts):

720 + if prompt["type"] == "embeds":

721 + raise NotImplementedError(

722 + "Embedding prompt not supported for beam search"

723 + )

724 + if prompt["type"] == "enc_dec":

725 + raise NotImplementedError(

726 + "Encoder-decoder prompt not supported for beam search"

727 + )

728

729 instances.append(

730 BeamSearchInstance(

775 - prompt_tokens,

731 + prompt,

732 lora_request=lora_req,

733 logprobs=None,

778 - **mm_kwargs,

734 ),

735 )

736

782 - for prompt_start in range(0, len(prompts), concurrency_limit):

737 + for prompt_start in range(0, len(instances), concurrency_limit):

738 instances_batch = instances[prompt_start : prompt_start + concurrency_limit]

739

740 token_iter = range(max_tokens)

@@ -808,22 +763,15 @@ class LLM:

763 if len(all_beams) == 0:

764 break

765

811 - # create corresponding batch entries for prompt & optional lora

812 - prompts_batch, lora_req_batch = zip(

813 - *[

814 - (create_tokens_prompt_from_beam(beam), beam.lora_request)

815 - for beam in all_beams

816 - ]

817 - )

818 -

766 # only runs for one step

767 # we don't need to use tqdm here

821 - output = self.generate(

822 - prompts_batch,

823 - sampling_params=beam_search_params,

768 + raw_output = self._render_and_run_requests(

769 + prompts=(beam.get_prompt() for beam in all_beams),

770 + params=self._params_to_seq(sampling_params, len(all_beams)),

771 + lora_requests=[beam.lora_request for beam in all_beams],

772 use_tqdm=False,

825 - lora_request=lora_req_batch,

773 )

774 + output = self.engine_class.validate_outputs(raw_output, RequestOutput)

775

776 for (start, end), instance in zip(

777 instance_start_and_end, instances_batch

@@ -841,19 +789,15 @@ class LLM:

789 logprobs = result.outputs[0].logprobs[0]

790 for token_id, logprob_obj in logprobs.items():

791 new_beam = BeamSearchSequence(

792 + current_beam.orig_prompt,

793 tokens=current_beam.tokens + [token_id],

794 logprobs=current_beam.logprobs + [logprobs],

795 lora_request=current_beam.lora_request,

796 cum_logprob=current_beam.cum_logprob

797 + logprob_obj.logprob,

849 - multi_modal_data=current_beam.multi_modal_data,

850 - mm_processor_kwargs=current_beam.mm_processor_kwargs,

798 )

799

853 - if (

854 - token_id == tokenizer.eos_token_id

855 - and not ignore_eos

856 - ):

800 + if token_id == eos_token_id and not ignore_eos:

801 instance.completed.append(new_beam)

802 else:

803 instance_new_beams.append(new_beam)

@@ -872,6 +816,7 @@ class LLM:

816

817 for beam in best_beams:

818 beam.text = tokenizer.decode(beam.tokens)

819 +

820 outputs.append(BeamSearchOutput(sequences=best_beams))

821

822 return outputs

@@ -880,7 +825,7 @@ class LLM:

825 self,

826 prompts: Sequence[PromptType],

827 tokenization_kwargs: dict[str, Any] | None = None,

883 - ) -> Sequence[DictPrompt | TokPrompt]:

828 + ) -> Sequence[ProcessorInputs]:

829 """

830 Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into

831 a format that can be passed to `_add_request`.

@@ -888,8 +833,7 @@ class LLM:

833 Refer to [LLM.generate][] for a complete description of the arguments.

834

835 Returns:

891 - A list of `TokPrompt` objects containing the tokenized prompt

892 - after chat template interpolation, and the raw multi-modal inputs.

836 + A list of `ProcessorInputs` objects ready to be passed into LLMEngine.

837 """

838 renderer = self.renderer

839 model_config = self.model_config

@@ -903,6 +847,14 @@ class LLM:

847

848 return renderer.render_cmpl(parsed_prompts, tok_params)

849

850 + def _preprocess_cmpl_one(

851 + self,

852 + prompt: PromptType,

853 + tokenization_kwargs: dict[str, Any] | None = None,

854 + ) -> ProcessorInputs:

855 + (engine_prompt,) = self._preprocess_cmpl([prompt], tokenization_kwargs)

856 + return engine_prompt

857 +

858 def _preprocess_chat(

859 self,

860 conversations: Sequence[list[ChatCompletionMessageParam]],

@@ -914,7 +866,7 @@ class LLM:

866 tools: list[dict[str, Any]] | None = None,

867 tokenization_kwargs: dict[str, Any] | None = None,

868 mm_processor_kwargs: dict[str, Any] | None = None,

917 - ) -> Sequence[TokPrompt]:

869 + ) -> Sequence[ProcessorInputs]:

870 """

871 Convert a list of conversations into prompts so that they can then

872 be used as input for other LLM APIs.

@@ -922,8 +874,7 @@ class LLM:

874 Refer to [LLM.chat][] for a complete description of the arguments.

875

876 Returns:

925 - A list of `TokPrompt` objects containing the tokenized prompt

926 - after chat template interpolation, and the raw multi-modal inputs.

877 + A list of `ProcessorInputs` objects ready to be passed into LLMEngine.

878 """

879 renderer = self.renderer

880

@@ -953,13 +904,39 @@ class LLM:

904

905 return engine_prompts

906

907 + def _preprocess_chat_one(

908 + self,

909 + conversation: list[ChatCompletionMessageParam],

910 + chat_template: str | None = None,

911 + chat_template_content_format: ChatTemplateContentFormatOption = "auto",

912 + chat_template_kwargs: dict[str, Any] | None = None,

913 + add_generation_prompt: bool = True,

914 + continue_final_message: bool = False,

915 + tools: list[dict[str, Any]] | None = None,

916 + tokenization_kwargs: dict[str, Any] | None = None,

917 + mm_processor_kwargs: dict[str, Any] | None = None,

918 + ) -> ProcessorInputs:

919 + (engine_prompt,) = self._preprocess_chat(

920 + [conversation],

921 + chat_template=chat_template,

922 + chat_template_content_format=chat_template_content_format,

923 + chat_template_kwargs=chat_template_kwargs,

924 + add_generation_prompt=add_generation_prompt,

925 + continue_final_message=continue_final_message,

926 + tools=tools,

927 + tokenization_kwargs=tokenization_kwargs,

928 + mm_processor_kwargs=mm_processor_kwargs,

929 + )

930 +

931 + return engine_prompt

932 +

933 def chat(

934 self,

935 messages: list[ChatCompletionMessageParam]

936 | Sequence[list[ChatCompletionMessageParam]],

937 sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,

938 use_tqdm: bool | Callable[..., tqdm] = True,

962 - lora_request: LoRARequest | None = None,

939 + lora_request: Sequence[LoRARequest] | LoRARequest | None = None,

940 chat_template: str | None = None,

941 chat_template_content_format: ChatTemplateContentFormatOption = "auto",

942 add_generation_prompt: bool = True,

@@ -1805,47 +1782,41 @@ class LLM:

1782 | Sequence[SamplingParams | PoolingParams],

1783 *,

1784 use_tqdm: bool | Callable[..., tqdm] = True,

1808 - lora_request: list[LoRARequest] | LoRARequest | None = None,

1785 + lora_request: Sequence[LoRARequest] | LoRARequest | None = None,

1786 priority: list[int] | None = None,

1787 tokenization_kwargs: dict[str, Any] | None = None,

1788 ):

1789 seq_prompts = prompt_to_seq(prompts)

1790 seq_params = self._params_to_seq(params, len(seq_prompts))

1814 -

1815 - if any(param.truncate_prompt_tokens is not None for param in seq_params):

1816 - # TODO: Remove this after deprecating `param.truncate_prompt_tokens`

1817 - # Then, move the code from the `else` block to the top and let

1818 - # `self._preprocess_cmpl` handle prompt normalization

1819 - engine_prompts: Sequence[DictPrompt | TokPrompt] = [

1820 - engine_prompt

1821 - for prompt, param in zip(seq_prompts, seq_params)

1822 - for engine_prompt in self._preprocess_cmpl(

1823 - [prompt],

1824 - tokenization_kwargs=merge_kwargs(

1825 - tokenization_kwargs,

1826 - dict(truncate_prompt_tokens=param.truncate_prompt_tokens),

1791 + seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts))

1792 + seq_tok_kwargs = [

1793 + merge_kwargs(

1794 + tokenization_kwargs,

1795 + dict(truncate_prompt_tokens=param.truncate_prompt_tokens),

1796 + )

1797 + for param in seq_params

1798 + ]

1799 + seq_priority = self._priority_to_seq(priority, len(prompts))

1800 +

1801 + return self._render_and_run_requests(

1802 + prompts=(

1803 + self._preprocess_cmpl_one(prompt, tok_kwargs)

1804 + for prompt, tok_kwargs in zip(

1805 + maybe_tqdm(

1806 + seq_prompts,

1807 + use_tqdm=use_tqdm,

1808 + desc="Rendering prompts",

1809 ),

1810 + seq_tok_kwargs,

1811 )

1829 - ]

1830 - else:

1831 - engine_prompts = self._preprocess_cmpl(

1832 - seq_prompts,

1833 - tokenization_kwargs=tokenization_kwargs,

1834 - )

1835 -

1836 - self._validate_and_add_requests(

1837 - prompts=engine_prompts,

1812 + ),

1813 params=seq_params,

1814 use_tqdm=use_tqdm,

1840 - lora_request=self._get_modality_specific_lora_reqs(

1841 - engine_prompts, lora_request

1842 - ),

1815 + lora_requests=seq_lora_requests,

1816 tokenization_kwargs=tokenization_kwargs,

1844 - priority=priority,

1817 + priorities=seq_priority,

1818 )

1819

1847 - return self._run_engine(use_tqdm=use_tqdm)

1848 -

1820 def _run_chat(

1821 self,

1822 messages: list[ChatCompletionMessageParam]

@@ -1855,7 +1826,7 @@ class LLM:

1826 | Sequence[SamplingParams | PoolingParams],

1827 *,

1828 use_tqdm: bool | Callable[..., tqdm] = True,

1858 - lora_request: LoRARequest | None = None,

1829 + lora_request: Sequence[LoRARequest] | LoRARequest | None = None,

1830 chat_template: str | None = None,

1831 chat_template_content_format: ChatTemplateContentFormatOption = "auto",

1832 add_generation_prompt: bool = True,

@@ -1865,68 +1836,94 @@ class LLM:

1836 tokenization_kwargs: dict[str, Any] | None = None,

1837 mm_processor_kwargs: dict[str, Any] | None = None,

1838 ):

1868 - engine_prompts = self._preprocess_chat(

1869 - conversation_to_seq(messages),

1870 - chat_template=chat_template,

1871 - chat_template_content_format=chat_template_content_format,

1872 - chat_template_kwargs=chat_template_kwargs,

1873 - add_generation_prompt=add_generation_prompt,

1874 - continue_final_message=continue_final_message,

1875 - tools=tools,

1839 + seq_convs = conversation_to_seq(messages)

1840 + seq_params = self._params_to_seq(params, len(seq_convs))

1841 + seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_convs))

1842 + seq_tok_kwargs = [

1843 + merge_kwargs(

1844 + tokenization_kwargs,

1845 + dict(truncate_prompt_tokens=param.truncate_prompt_tokens),

1846 + )

1847 + for param in seq_params

1848 + ]

1849 +

1850 + return self._render_and_run_requests(

1851 + prompts=(

1852 + self._preprocess_chat_one(

1853 + conversation,

1854 + chat_template=chat_template,

1855 + chat_template_content_format=chat_template_content_format,

1856 + chat_template_kwargs=chat_template_kwargs,

1857 + add_generation_prompt=add_generation_prompt,

1858 + continue_final_message=continue_final_message,

1859 + tools=tools,

1860 + tokenization_kwargs=tok_kwargs,

1861 + mm_processor_kwargs=mm_processor_kwargs,

1862 + )

1863 + for conversation, tok_kwargs in zip(

1864 + maybe_tqdm(

1865 + seq_convs,

1866 + use_tqdm=use_tqdm,

1867 + desc="Rendering conversations",

1868 + ),

1869 + seq_tok_kwargs,

1870 + )

1871 + ),

1872 + params=seq_params,

1873 + lora_requests=seq_lora_requests,

1874 + use_tqdm=use_tqdm,

1875 tokenization_kwargs=tokenization_kwargs,

1877 - mm_processor_kwargs=mm_processor_kwargs,

1876 )

1877

1880 - self._validate_and_add_requests(

1881 - prompts=engine_prompts,

1878 + def _render_and_run_requests(

1879 + self,

1880 + prompts: Iterable[ProcessorInputs],

1881 + params: Sequence[SamplingParams | PoolingParams],

1882 + *,

1883 + lora_requests: Sequence[LoRARequest | None] | None = None,

1884 + tokenization_kwargs: dict[str, Any] | None = None,

1885 + priorities: Sequence[int] | None = None,

1886 + use_tqdm: bool | Callable[..., tqdm] = True,

1887 + ):

1888 + if isinstance(prompts, (list, tuple)):

1889 + logger.warning_once(

1890 + "Rendering all prompts before adding them to the engine "

1891 + "is less efficient than performing both on the same prompt "

1892 + "before processing the next prompt. You should instead pass "

1893 + "a generator that renders one prompt per iteration, as that allows "

1894 + "engine execution to begin for the first prompt while processing "

1895 + "the next prompt."

1896 + )

1897 +

1898 + self._render_and_add_requests(

1899 + prompts=prompts,

1900 params=params,

1883 - use_tqdm=use_tqdm,

1884 - lora_request=self._get_modality_specific_lora_reqs(

1885 - engine_prompts, lora_request

1886 - ),

1901 + lora_requests=lora_requests,

1902 tokenization_kwargs=tokenization_kwargs,

1903 + priorities=priorities,

1904 )

1905

1906 return self._run_engine(use_tqdm=use_tqdm)

1907

1892 - def _validate_and_add_requests(

1908 + def _render_and_add_requests(

1909 self,

1894 - prompts: Sequence[DictPrompt | TokPrompt],

1895 - params: SamplingParams

1896 - | PoolingParams

1897 - | Sequence[SamplingParams | PoolingParams],

1910 + prompts: Iterable[ProcessorInputs],

1911 + params: Sequence[SamplingParams | PoolingParams],

1912 *,

1899 - use_tqdm: bool | Callable[..., tqdm] = True,

1900 - lora_request: Sequence[LoRARequest | None] | LoRARequest | None,

1913 + lora_requests: Sequence[LoRARequest | None] | None = None,

1914 tokenization_kwargs: dict[str, Any] | None = None,

1902 - priority: list[int] | None = None,

1915 + priorities: Sequence[int] | None = None,

1916 ) -> list[str]:

1904 - num_requests = len(prompts)

1905 - seq_params = self._params_to_seq(params, num_requests)

1906 - seq_lora_requests = self._lora_request_to_seq(lora_request, num_requests)

1907 - seq_priority = self._priority_to_seq(priority, num_requests)

1908 -

1909 - for sp in seq_params:

1910 - if isinstance(sp, SamplingParams):

1911 - # We only care about the final output

1912 - sp.output_kind = RequestOutputKind.FINAL_ONLY

1913 -

1914 - # Add requests to the engine.

1915 - it = prompts

1916 - if use_tqdm:

1917 - tqdm_func = use_tqdm if callable(use_tqdm) else tqdm

1918 - it = tqdm_func(it, desc="Adding requests")

1919 -

1917 added_request_ids: list[str] = []

1918

1919 try:

1923 - for i, prompt in enumerate(it):

1920 + for i, prompt in enumerate(prompts):

1921 request_id = self._add_request(

1922 prompt,

1926 - seq_params[i],

1927 - lora_request=seq_lora_requests[i],

1923 + params[i],

1924 + lora_request=None if lora_requests is None else lora_requests[i],

1925 tokenization_kwargs=tokenization_kwargs,

1929 - priority=seq_priority[i],

1926 + priority=0 if priorities is None else priorities[i],

1927 )

1928 added_request_ids.append(request_id)

1929 except Exception as e:

@@ -1938,13 +1935,16 @@ class LLM:

1935

1936 def _add_request(

1937 self,

1941 - prompt: PromptType | DictPrompt | TokPrompt,

1938 + prompt: ProcessorInputs,

1939 params: SamplingParams | PoolingParams,

1940 lora_request: LoRARequest | None = None,

1941 tokenization_kwargs: dict[str, Any] | None = None,

1942 priority: int = 0,

1943 ) -> str:

1947 - prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)

1944 + if isinstance(params, SamplingParams):

1945 + # We only care about the final output

1946 + params.output_kind = RequestOutputKind.FINAL_ONLY

1947 +

1948 request_id = str(next(self.request_counter))

1949

1950 if params.truncate_prompt_tokens is not None:

@@ -1962,32 +1962,14 @@ class LLM:

1962 dict(truncate_prompt_tokens=params.truncate_prompt_tokens),

1963 )

1964

1965 - renderer = self.renderer

1966 - tok_params = renderer.default_cmpl_tok_params.with_kwargs(

1967 - **(tokenization_kwargs or {})

1968 - )

1969 -

1970 - tokenization_kwargs = tok_params.get_encode_kwargs()

1971 - engine_request = self.input_processor.process_inputs(

1965 + return self.llm_engine.add_request(

1966 request_id,

1967 prompt,

1968 params,

1969 lora_request=lora_request,

1970 tokenization_kwargs=tokenization_kwargs,

1971 priority=priority,

1978 - supported_tasks=self.supported_tasks,

1979 - )

1980 -

1981 - self.llm_engine.add_request(

1982 - request_id,

1983 - engine_request,

1984 - params,

1985 - lora_request=lora_request,

1986 - tokenization_kwargs=tokenization_kwargs,

1987 - priority=priority,

1988 - prompt_text=prompt_text,

1972 )

1990 - return engine_request.request_id

1973

1974 def _run_engine(

1975 self,

vllm/entrypoints/openai/chat_completion/serving.py

+12 -25

@@ -67,13 +67,12 @@ from vllm.entrypoints.openai.parser.harmony_utils import (

67 )

68 from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls

69 from vllm.entrypoints.utils import get_max_tokens, should_include_usage

70 -from vllm.inputs.data import TokensPrompt

70 +from vllm.inputs.data import ProcessorInputs, TokensPrompt

71 from vllm.logger import init_logger

72 from vllm.logprobs import Logprob

73 from vllm.outputs import CompletionOutput, RequestOutput

74 from vllm.parser import ParserManager

75 from vllm.reasoning import ReasoningParser

76 -from vllm.renderers.inputs import TokPrompt

76 from vllm.sampling_params import BeamSearchParams, SamplingParams

77 from vllm.tokenizers import TokenizerLike

78 from vllm.tokenizers.mistral import (

@@ -221,7 +220,7 @@ class OpenAIServingChat(OpenAIServing):

220 async def render_chat_request(

221 self,

222 request: ChatCompletionRequest,

224 - ) -> tuple[list[ConversationMessage], list[TokPrompt]] | ErrorResponse:

223 + ) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:

224 """

225 render chat request by validating and preprocessing inputs.

226

@@ -380,7 +379,9 @@ class OpenAIServingChat(OpenAIServing):

379 generators: list[AsyncGenerator[RequestOutput, None]] = []

380 try:

381 for i, engine_prompt in enumerate(engine_prompts):

383 - prompt_text = self._extract_prompt_text(engine_prompt)

382 + prompt_token_ids = self._extract_prompt_components(

383 + engine_prompt

384 + ).token_ids

385

386 # If we are creating sub requests for multiple prompts, ensure that they

387 # have unique request ids.

@@ -431,35 +432,21 @@ class OpenAIServingChat(OpenAIServing):

432 trace_headers=trace_headers,

433 )

434 else:

434 - tok_params = request.build_tok_params(self.model_config)

435 - tokenization_kwargs = tok_params.get_encode_kwargs()

436 -

437 - engine_request = self.input_processor.process_inputs(

438 - sub_request_id,

439 - engine_prompt,

440 - sampling_params,

441 - lora_request=lora_request,

442 - tokenization_kwargs=tokenization_kwargs,

443 - trace_headers=trace_headers,

444 - priority=request.priority,

445 - data_parallel_rank=data_parallel_rank,

435 + reasoning_ended = (

436 + reasoning_parser.is_reasoning_end(prompt_token_ids or [])

437 + if reasoning_parser

438 + else None

439 )

447 - reasoning_ended = None

448 - if reasoning_parser:

449 - reasoning_ended = reasoning_parser.is_reasoning_end(

450 - engine_request.prompt_token_ids or [] # type: ignore[attr-defined]

451 - )

452 - engine_request.reasoning_ended = reasoning_ended

440 +

441 generator = self.engine_client.generate(

454 - engine_request,

442 + engine_prompt,

443 sampling_params,

444 sub_request_id,

445 lora_request=lora_request,

446 trace_headers=trace_headers,

447 priority=request.priority,

460 - prompt_text=prompt_text,

461 - tokenization_kwargs=tokenization_kwargs,

448 data_parallel_rank=data_parallel_rank,

449 + reasoning_ended=reasoning_ended,

450 )

451

452 generators.append(generator)

vllm/entrypoints/openai/completion/serving.py

+4 -22

@@ -34,10 +34,10 @@ from vllm.entrypoints.openai.engine.serving import (

34 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

35 from vllm.entrypoints.utils import get_max_tokens, should_include_usage

36 from vllm.exceptions import VLLMValidationError

37 +from vllm.inputs.data import ProcessorInputs

38 from vllm.logger import init_logger

39 from vllm.logprobs import Logprob

40 from vllm.outputs import RequestOutput

40 -from vllm.renderers.inputs import TokPrompt

41 from vllm.sampling_params import BeamSearchParams, SamplingParams

42 from vllm.tokenizers import TokenizerLike

43 from vllm.utils.async_utils import merge_async_iterators

@@ -80,7 +80,7 @@ class OpenAIServingCompletion(OpenAIServing):

80 async def render_completion_request(

81 self,

82 request: CompletionRequest,

83 - ) -> list[TokPrompt] | ErrorResponse:

83 + ) -> list[ProcessorInputs] | ErrorResponse:

84 """

85 render completion request by validating and preprocessing inputs.

@@ -163,8 +163,6 @@ class OpenAIServingCompletion(OpenAIServing):

163 generators: list[AsyncGenerator[RequestOutput, None]] = []

164 try:

165 for i, engine_prompt in enumerate(engine_prompts):

166 - prompt_text = self._extract_prompt_text(engine_prompt)

167 -

166 max_tokens = get_max_tokens(

167 max_model_len,

168 request.max_tokens,

@@ -208,29 +206,13 @@ class OpenAIServingCompletion(OpenAIServing):

206 trace_headers=trace_headers,

207 )

208 else:

211 - tok_params = request.build_tok_params(self.model_config)

212 - tokenization_kwargs = tok_params.get_encode_kwargs()

213 -

214 - engine_request = self.input_processor.process_inputs(

215 - request_id_item,

216 - engine_prompt,

217 - sampling_params,

218 - lora_request=lora_request,

219 - tokenization_kwargs=tokenization_kwargs,

220 - trace_headers=trace_headers,

221 - priority=request.priority,

222 - data_parallel_rank=data_parallel_rank,

223 - )

224 -

209 generator = self.engine_client.generate(

226 - engine_request,

210 + engine_prompt,

211 sampling_params,

212 request_id_item,

213 lora_request=lora_request,

214 trace_headers=trace_headers,

215 priority=request.priority,

232 - prompt_text=prompt_text,

233 - tokenization_kwargs=tokenization_kwargs,

216 data_parallel_rank=data_parallel_rank,

217 )

218

@@ -312,7 +294,7 @@ class OpenAIServingCompletion(OpenAIServing):

294 async def completion_stream_generator(

295 self,

296 request: CompletionRequest,

315 - engine_prompts: list[TokPrompt],

297 + engine_prompts: list[ProcessorInputs],

298 result_generator: AsyncIterator[tuple[int, RequestOutput]],

299 request_id: str,

300 created_time: int,

vllm/entrypoints/openai/engine/serving.py

+41 -87

@@ -96,15 +96,19 @@ from vllm.entrypoints.serve.tokenize.protocol import (

96 )

97 from vllm.entrypoints.utils import get_max_tokens, sanitize_message

98 from vllm.exceptions import VLLMValidationError

99 -from vllm.inputs.data import PromptType, SingletonPrompt, TokensPrompt

99 +from vllm.inputs.data import (

100 + ProcessorInputs,

101 + PromptType,

102 + SingletonPrompt,

103 + TokensPrompt,

104 + token_inputs,

105 +)

106 from vllm.logger import init_logger

107 from vllm.logprobs import Logprob, PromptLogprobs

108 from vllm.lora.request import LoRARequest

103 -from vllm.multimodal import MultiModalDataDict

109 from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput

110 from vllm.pooling_params import PoolingParams

111 from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs

107 -from vllm.renderers.inputs import TokPrompt

112 from vllm.renderers.inputs.preprocess import (

113 extract_prompt_components,

114 extract_prompt_len,

@@ -206,7 +210,7 @@ class ServeContext(Generic[RequestT]):

210 request_id: str

211 created_time: int = field(default_factory=lambda: int(time.time()))

212 lora_request: LoRARequest | None = None

209 - engine_prompts: list[TokPrompt] | None = None

213 + engine_prompts: list[ProcessorInputs] | None = None

214

215 result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (

216 None

@@ -249,7 +253,7 @@ class OpenAIServing:

253

254 async def beam_search(

255 self,

252 - prompt: TokPrompt,

256 + prompt: ProcessorInputs,

257 request_id: str,

258 params: BeamSearchParams,

259 lora_request: LoRARequest | None = None,

@@ -262,86 +266,53 @@ class OpenAIServing:

266 length_penalty = params.length_penalty

267 include_stop_str_in_output = params.include_stop_str_in_output

268

265 - input_processor = self.input_processor

266 - tokenizer = input_processor.tokenizer

267 - if tokenizer is None:

268 - raise VLLMValidationError(

269 - "You cannot use beam search when `skip_tokenizer_init=True`",

270 - parameter="skip_tokenizer_init",

271 - value=True,

272 - )

273 -

274 - eos_token_id: int = tokenizer.eos_token_id # type: ignore

275 -

276 - if isinstance(prompt, dict) and "encoder_prompt" in prompt:

277 - raise NotImplementedError("Encoder-decoder prompt not supported")

278 -

279 - prompt_text: str | None = prompt.get("prompt") # type: ignore

280 - prompt_token_ids: list[int] = prompt.get("prompt_token_ids", []) # type: ignore

281 - multi_modal_data: MultiModalDataDict | None = prompt.get("multi_modal_data") # type: ignore

282 -

283 - mm_processor_kwargs: dict[str, Any] | None = None

269 + tokenizer = self.renderer.get_tokenizer()

270 + eos_token_id = tokenizer.eos_token_id

271 + sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)

272

285 - # This is a workaround to fix multimodal beam search; this is a

286 - # bandaid fix for 2 small problems:

287 - # 1. Multi_modal_data on the processed_inputs currently resolves to

288 - # `None`.

289 - # 2. preprocessing above expands the multimodal placeholders. However,

290 - # this happens again in generation, so the double expansion causes

291 - # a mismatch.

292 - # TODO - would be ideal to handle this more gracefully.

273 + if prompt["type"] == "embeds":

274 + raise NotImplementedError("Embedding prompt not supported for beam search")

275 + if prompt["type"] == "enc_dec":

276 + raise NotImplementedError(

277 + "Encoder-decoder prompt not supported for beam search"

278 + )

279

280 + prompt_text = prompt.get("prompt")

281 + prompt_token_ids = prompt["prompt_token_ids"]

282 tokenized_length = len(prompt_token_ids)

283

296 - sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)

297 -

284 logprobs_num = 2 * beam_width

299 - beam_search_params = SamplingParams(

285 + sampling_params = SamplingParams(

286 logprobs=logprobs_num,

287 max_tokens=1,

288 temperature=temperature,

289 )

290 all_beams = [

291 BeamSearchSequence(

292 + orig_prompt=prompt,

293 tokens=prompt_token_ids,

294 cum_logprob=0,

295 logprobs=[],

309 - multi_modal_data=multi_modal_data,

310 - mm_processor_kwargs=mm_processor_kwargs,

296 lora_request=lora_request,

297 )

298 ]

299 completed = []

300

301 for _ in range(max_tokens):

317 - prompts_batch, lora_req_batch = zip(

318 - *[

319 - (

320 - TokensPrompt(

321 - prompt_token_ids=beam.tokens,

322 - multi_modal_data=beam.multi_modal_data,

323 - mm_processor_kwargs=beam.mm_processor_kwargs,

324 - ),

325 - beam.lora_request,

326 - )

327 - for beam in all_beams

328 - ]

329 - )

330 -

302 tasks = []

303 request_id_batch = f"{request_id}-{random_uuid()}"

304

334 - for i, (individual_prompt, lora_req) in enumerate(

335 - zip(prompts_batch, lora_req_batch)

336 - ):

305 + for i, beam in enumerate(all_beams):

306 + prompt_item = beam.get_prompt()

307 + lora_request_item = beam.lora_request

308 request_id_item = f"{request_id_batch}-beam-{i}"

309 task = asyncio.create_task(

310 collect_from_async_generator(

311 self.engine_client.generate(

341 - individual_prompt,

342 - beam_search_params,

312 + prompt_item,

313 + sampling_params,

314 request_id_item,

344 - lora_request=lora_req,

315 + lora_request=lora_request_item,

316 trace_headers=trace_headers,

317 )

318 )

@@ -406,6 +377,7 @@ class OpenAIServing:

377 logprobs_entry = result.outputs[0].logprobs[0]

378 completed.append(

379 BeamSearchSequence(

380 + orig_prompt=prompt,

381 tokens=current_beam.tokens + [eos_token_id]

382 if include_stop_str_in_output

383 else current_beam.tokens,

@@ -433,12 +405,11 @@ class OpenAIServing:

405 logprobs_entry = result.outputs[0].logprobs[0]

406 new_beams.append(

407 BeamSearchSequence(

408 + orig_prompt=prompt,

409 tokens=current_beam.tokens + [token_id],

410 logprobs=current_beam.logprobs + [logprobs_entry],

411 lora_request=current_beam.lora_request,

412 cum_logprob=float(all_beams_logprob[idx]),

440 - multi_modal_data=current_beam.multi_modal_data,

441 - mm_processor_kwargs=current_beam.mm_processor_kwargs,

413 )

414 )

415

@@ -958,7 +929,7 @@ class OpenAIServing:

929 request: RendererRequest,

930 prompt_input: str | list[str] | list[int] | list[list[int]] | None,

931 prompt_embeds: bytes | list[bytes] | None,

961 - ) -> list[TokPrompt]:

932 + ) -> list[ProcessorInputs]:

933 prompts = list[SingletonPrompt | bytes]()

934 if prompt_embeds is not None: # embeds take higher priority

935 prompts.extend(prompt_to_seq(prompt_embeds))

@@ -971,7 +942,7 @@ class OpenAIServing:

942 self,

943 request: RendererRequest,

944 prompts: Sequence[PromptType | bytes],

974 - ) -> list[TokPrompt]:

945 + ) -> list[ProcessorInputs]:

946 renderer = self.renderer

947 model_config = self.model_config

948

@@ -1004,7 +975,7 @@ class OpenAIServing:

975 default_template_kwargs: dict[str, Any] | None,

976 tool_dicts: list[dict[str, Any]] | None = None,

977 tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,

1007 - ) -> tuple[list[ConversationMessage], list[TokPrompt]]:

978 + ) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:

979 from vllm.tokenizers.mistral import MistralTokenizer

980

981 renderer = self.renderer

@@ -1052,13 +1023,13 @@ class OpenAIServing:

1023

1024 return conversation, [engine_prompt]

1025

1055 - def _extract_prompt_components(self, prompt: object):

1026 + def _extract_prompt_components(self, prompt: PromptType | ProcessorInputs):

1027 return extract_prompt_components(self.model_config, prompt)

1028

1058 - def _extract_prompt_text(self, prompt: object):

1029 + def _extract_prompt_text(self, prompt: ProcessorInputs):

1030 return self._extract_prompt_components(prompt).text

1031

1061 - def _extract_prompt_len(self, prompt: object):

1032 + def _extract_prompt_len(self, prompt: ProcessorInputs):

1033 return extract_prompt_len(self.model_config, prompt)

1034

1035 async def _render_next_turn(

@@ -1088,16 +1059,14 @@ class OpenAIServing:

1059 async def _generate_with_builtin_tools(

1060 self,

1061 request_id: str,

1091 - engine_prompt: TokPrompt,

1062 + engine_prompt: ProcessorInputs,

1063 sampling_params: SamplingParams,

1093 - tok_params: TokenizeParams,

1064 context: ConversationContext,

1065 lora_request: LoRARequest | None = None,

1066 priority: int = 0,

1067 trace_headers: Mapping[str, str] | None = None,

1068 ):

1069 max_model_len = self.model_config.max_model_len

1100 - prompt_text = self._extract_prompt_text(engine_prompt)

1070

1071 orig_priority = priority

1072 sub_request = 0

@@ -1112,26 +1081,13 @@ class OpenAIServing:

1081 lora_request=lora_request,

1082 )

1083

1115 - tokenization_kwargs = tok_params.get_encode_kwargs()

1116 - engine_request = self.input_processor.process_inputs(

1117 - sub_request_id,

1118 - engine_prompt,

1119 - sampling_params,

1120 - lora_request=lora_request,

1121 - tokenization_kwargs=tokenization_kwargs,

1122 - trace_headers=trace_headers,

1123 - priority=priority,

1124 - )

1125 -

1084 generator = self.engine_client.generate(

1127 - engine_request,

1085 + engine_prompt,

1086 sampling_params,

1087 sub_request_id,

1088 lora_request=lora_request,

1089 trace_headers=trace_headers,

1090 priority=priority,

1133 - prompt_text=prompt_text,

1134 - tokenization_kwargs=tokenization_kwargs,

1091 )

1092

1093 async for res in generator:

@@ -1154,11 +1110,11 @@ class OpenAIServing:

1110 # Render the next prompt token ids and update sampling_params.

1111 if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):

1112 token_ids = context.render_for_completion()

1157 - engine_prompt = TokensPrompt(prompt_token_ids=token_ids)

1113 + engine_prompt = token_inputs(token_ids)

1114

1115 sampling_params.max_tokens = max_model_len - len(token_ids)

1116 elif isinstance(context, ParsableContext):

1161 - engine_prompts = await self._render_next_turn(

1117 + (engine_prompt,) = await self._render_next_turn(

1118 context.request,

1119 context.parser.response_messages,

1120 context.tool_dicts,

@@ -1166,8 +1122,6 @@ class OpenAIServing:

1122 context.chat_template,

1123 context.chat_template_content_format,

1124 )

1169 - engine_prompt = engine_prompts[0]

1170 - prompt_text = self._extract_prompt_text(engine_prompt)

1125

1126 sampling_params.max_tokens = get_max_tokens(

1127 max_model_len,

@@ -1184,7 +1138,7 @@ class OpenAIServing:

1138 def _log_inputs(

1139 self,

1140 request_id: str,

1187 - inputs: PromptType | TokPrompt,

1141 + inputs: PromptType | ProcessorInputs,

1142 params: SamplingParams | PoolingParams | BeamSearchParams | None,

1143 lora_request: LoRARequest | None,

1144 ) -> None:

vllm/entrypoints/openai/realtime/serving.py

+8 -2

@@ -15,6 +15,7 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels

15 from vllm.inputs.data import PromptType

16 from vllm.logger import init_logger

17 from vllm.model_executor.models.interfaces import SupportsRealtime

18 +from vllm.renderers.inputs.preprocess import parse_model_prompt

20 logger = init_logger(__name__)

@@ -70,15 +71,20 @@ class OpenAIServingRealtime(OpenAIServing):

71 Yields:

72 StreamingInput objects containing audio prompts for the engine

73 """

74 + model_config = self.model_config

75 + renderer = self.renderer

77 # mypy is being stupid

78 # TODO(Patrick) - fix this

79 stream_input_iter = cast(

80 AsyncGenerator[PromptType, None],

81 self.model_cls.buffer_realtime_audio(

79 - audio_stream, input_stream, self.model_config

82 + audio_stream, input_stream, model_config

83 ),

84 )

86 async for prompt in stream_input_iter:

84 - yield StreamingInput(prompt=prompt)

87 + parsed_prompt = parse_model_prompt(model_config, prompt)

88 + (engine_prompt,) = await renderer.render_cmpl_async([parsed_prompt])

89 +

90 + yield StreamingInput(prompt=engine_prompt)

vllm/entrypoints/openai/responses/context.py

+2 -2

@@ -9,7 +9,7 @@ from abc import ABC, abstractmethod

9 from collections.abc import Callable

10 from contextlib import AsyncExitStack

11 from dataclasses import replace

12 -from typing import TYPE_CHECKING, Union

12 +from typing import TYPE_CHECKING, Final, Union

14 from openai.types.responses.response_function_tool_call_output_item import (

15 ResponseFunctionToolCallOutputItem,

@@ -304,7 +304,7 @@ class ParsableContext(ConversationContext):

304

305 self.tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)

306 self.chat_template = chat_template

307 - self.chat_template_content_format = chat_template_content_format

307 + self.chat_template_content_format: Final = chat_template_content_format

308

309 self.input_messages: list[ResponseRawMessageAndToken] = []

310 self.output_messages: list[ResponseRawMessageAndToken] = []

vllm/entrypoints/openai/responses/serving.py

+3 -6

@@ -116,13 +116,12 @@ from vllm.entrypoints.openai.responses.utils import (

116 )

117 from vllm.entrypoints.utils import get_max_tokens

118 from vllm.exceptions import VLLMValidationError

119 -from vllm.inputs.data import TokensPrompt

119 +from vllm.inputs.data import ProcessorInputs, token_inputs

120 from vllm.logger import init_logger

121 from vllm.logprobs import Logprob as SampleLogprob

122 from vllm.logprobs import SampleLogprobs

123 from vllm.outputs import CompletionOutput

124 from vllm.parser import ParserManager

125 -from vllm.renderers.inputs import TokPrompt

125 from vllm.sampling_params import SamplingParams, StructuredOutputsParams

126 from vllm.tokenizers import TokenizerLike

127 from vllm.utils import random_uuid

@@ -298,7 +297,7 @@ class OpenAIServingResponses(OpenAIServing):

297

298 def _validate_generator_input(

299 self,

301 - engine_prompt: TokPrompt,

300 + engine_prompt: ProcessorInputs,

301 ) -> ErrorResponse | None:

302 """Add validations to the input to the generator here."""

303 prompt_len = self._extract_prompt_len(engine_prompt)

@@ -458,7 +457,6 @@ class OpenAIServingResponses(OpenAIServing):

457 sampling_params = request.to_sampling_params(

458 default_max_tokens, self.default_sampling_params

459 )

461 - tok_params = request.build_tok_params(self.model_config)

460

461 trace_headers = (

462 None

@@ -512,7 +510,6 @@ class OpenAIServingResponses(OpenAIServing):

510 request_id=request.request_id,

511 engine_prompt=engine_prompt,

512 sampling_params=sampling_params,

515 - tok_params=tok_params,

513 context=context,

514 lora_request=lora_request,

515 priority=request.priority,

@@ -647,7 +644,7 @@ class OpenAIServingResponses(OpenAIServing):

644

645 messages = self._construct_input_messages_with_harmony(request, prev_response)

646 prompt_token_ids = render_for_completion(messages)

650 - engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)

647 + engine_prompt = token_inputs(prompt_token_ids)

648

649 # Add cache_salt if provided in the request

650 if request.cache_salt is not None:

vllm/entrypoints/openai/speech_to_text/speech_to_text.py

+49 -56

@@ -36,14 +36,15 @@ from vllm.entrypoints.openai.speech_to_text.protocol import (

36 TranslationSegment,

37 TranslationStreamResponse,

38 )

39 +from vllm.entrypoints.utils import get_max_tokens

40 from vllm.exceptions import VLLMValidationError

40 -from vllm.inputs.data import PromptType

41 +from vllm.inputs import ProcessorInputs

42 from vllm.logger import init_logger

43 from vllm.logprobs import FlatLogprobs, Logprob

44 from vllm.model_executor.models import SupportsTranscription, supports_transcription

45 from vllm.outputs import RequestOutput

45 -from vllm.renderers.inputs import EncoderDecoderDictPrompt

46 -from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt

46 +from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt

47 +from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt

48 from vllm.tokenizers import get_tokenizer

49 from vllm.utils.import_utils import PlaceholderModule

@@ -202,8 +203,6 @@ class OpenAISpeechToText(OpenAIServing):

203 return

204

205 try:

205 - from vllm.sampling_params import SamplingParams

206 -

206 warmup_start = time.perf_counter()

207 logger.info("Warming up multimodal input processor...")

208

@@ -221,21 +220,11 @@ class OpenAISpeechToText(OpenAIServing):

220 request_prompt="",

221 to_language=None,

222 )

224 -

225 - # Create minimal sampling params

226 - dummy_params = SamplingParams(

227 - max_tokens=1,

228 - temperature=0.0,

229 - skip_clone=True, # Internal warmup, safe to skip clone

230 - )

223 + parsed_prompt = parse_model_prompt(self.model_config, dummy_prompt)

224

225 # Process the dummy input through the input processor

226 # This will trigger all the multimodal processing initialization

234 - _ = self.input_processor.process_inputs(

235 - request_id="warmup",

236 - prompt=dummy_prompt,

237 - params=dummy_params,

238 - )

227 + _ = self.renderer.render_cmpl([parsed_prompt])

228

229 warmup_elapsed = time.perf_counter() - warmup_start

230 logger.info("Input processor warmup completed in %.2fs", warmup_elapsed)

@@ -257,7 +246,7 @@ class OpenAISpeechToText(OpenAIServing):

246 self,

247 request: SpeechToTextRequest,

248 audio_data: bytes,

260 - ) -> tuple[list[PromptType], float]:

249 + ) -> tuple[list[ProcessorInputs], float]:

250 # Validate request

251 language = self.model_cls.validate_language(request.language)

252 # Skip to_language validation to avoid extra logging for Whisper.

@@ -285,7 +274,7 @@ class OpenAISpeechToText(OpenAIServing):

274 and duration > self.asr_config.max_audio_clip_s

275 )

276 chunks = [y] if not do_split_audio else self._split_audio(y, int(sr))

288 - prompts = []

277 + parsed_prompts: list[DictPrompt] = []

278 for chunk in chunks:

279 # The model has control over the construction, as long as it

280 # returns a valid PromptType.

@@ -298,12 +287,19 @@ class OpenAISpeechToText(OpenAIServing):

287 request_prompt=request.prompt,

288 to_language=to_language,

289 )

290 +

291 + parsed_prompt: DictPrompt

292 if request.response_format == "verbose_json":

302 - prompt = self._preprocess_verbose_prompt(parse_enc_dec_prompt(prompt))

293 + parsed_prompt = parse_enc_dec_prompt(prompt)

294 + parsed_prompt = self._preprocess_verbose_prompt(parsed_prompt)

295 + else:

296 + parsed_prompt = parse_model_prompt(self.model_config, prompt)

297 +

298 + parsed_prompts.append(parsed_prompt)

299

304 - prompts.append(prompt)

300 + engine_prompts = await self.renderer.render_cmpl_async(parsed_prompts)

301

306 - return prompts, duration

302 + return engine_prompts, duration

303

304 def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):

305 dec_prompt = prompt["decoder_prompt"]

@@ -436,7 +432,7 @@ class OpenAISpeechToText(OpenAIServing):

432 try:

433 lora_request = self._maybe_get_adapters(request)

434

439 - prompts, duration_s = await self._preprocess_speech_to_text(

435 + engine_prompts, duration_s = await self._preprocess_speech_to_text(

436 request=request,

437 audio_data=audio_data,

438 )

@@ -445,57 +441,54 @@ class OpenAISpeechToText(OpenAIServing):

441 logger.exception("Error in preprocessing prompt inputs")

442 return self.create_error_response(e)

443

444 + # Schedule the request and get the result generator.

445 + max_model_len = self.model_config.max_model_len

446 list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None

447 try:

448 # Unlike most decoder-only models, whisper generation length is not

449 # constrained by the size of the input audio, which is mapped to a

450 # fixed-size log-mel-spectogram. Still, allow for fewer tokens to be

451 # generated by respecting the extra completion tokens arg.

454 - if request.max_completion_tokens is None:

455 - default_max_tokens = self.model_config.max_model_len

456 - else:

457 - default_max_tokens = min(

458 - self.model_config.max_model_len, request.max_completion_tokens

459 - )

452 + max_tokens = get_max_tokens(

453 + max_model_len,

454 + request.max_completion_tokens,

455 + 0,

456 + self.default_sampling_params,

457 + )

458 +

459 sampling_params = request.to_sampling_params(

461 - default_max_tokens, self.default_sampling_params

460 + max_tokens,

461 + self.default_sampling_params,

462 )

463 if request.response_format == "verbose_json":

464 sampling_params.logprobs = 1

465

466 - self._log_inputs(

467 - request_id,

468 - # It will not display special tokens like <|startoftranscript|>

469 - request.prompt,

470 - params=sampling_params,

471 - lora_request=lora_request,

472 - )

473 -

474 - trace_headers = (

475 - None

476 - if raw_request is None

477 - else await self._get_trace_headers(raw_request.headers)

478 - )

479 -

466 list_result_generator = []

481 - for i, prompt in enumerate(prompts):

467 + for i, engine_prompt in enumerate(engine_prompts):

468 request_id_item = f"{request_id}_{i}"

483 - engine_request = self.input_processor.process_inputs(

469 +

470 + self._log_inputs(

471 request_id_item,

485 - prompt,

472 + engine_prompt,

473 + params=sampling_params,

474 + lora_request=lora_request,

475 + )

476 +

477 + trace_headers = (

478 + None

479 + if raw_request is None

480 + else await self._get_trace_headers(raw_request.headers)

481 + )

482 +

483 + generator = self.engine_client.generate(

484 + engine_prompt,

485 sampling_params,

486 + request_id_item,

487 lora_request=lora_request,

488 trace_headers=trace_headers,

489 - priority=0,

490 - )

491 - list_result_generator.append(

492 - self.engine_client.generate(

493 - engine_request,

494 - sampling_params,

495 - request_id_item,

496 - lora_request=lora_request,

497 - )

489 )

490 +

491 + list_result_generator.append(generator)

492 except ValueError as e:

493 return self.create_error_response(e)

494

vllm/entrypoints/pooling/embed/serving.py

+3 -12

@@ -28,11 +28,10 @@ from vllm.entrypoints.pooling.utils import (

28 encode_pooling_output_base64,

29 encode_pooling_output_float,

30 )

31 -from vllm.inputs.data import TokensPrompt

31 +from vllm.inputs.data import ProcessorInputs, TokensPrompt, token_inputs

32 from vllm.logger import init_logger

33 from vllm.outputs import PoolingOutput, PoolingRequestOutput

34 from vllm.pooling_params import PoolingParams

35 -from vllm.renderers.inputs import TokPrompt

35 from vllm.utils.async_utils import merge_async_iterators

36 from vllm.utils.collection_utils import chunk_list

37 from vllm.utils.serial_utils import EmbedDType, Endianness

@@ -256,7 +255,7 @@ class OpenAIServingEmbedding(OpenAIServing):

255 chunk_request_id = f"{ctx.request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"

256

257 # Create engine prompt for this chunk

259 - chunk_engine_prompt = TokensPrompt(prompt_token_ids=chunk_tokens)

258 + chunk_engine_prompt = token_inputs(chunk_tokens)

259

260 # Log the chunk

261 self._log_inputs(

@@ -266,16 +265,12 @@ class OpenAIServingEmbedding(OpenAIServing):

265 lora_request=ctx.lora_request,

266 )

267

269 - tok_params = ctx.request.build_tok_params(self.model_config)

270 - tokenization_kwargs = tok_params.get_encode_kwargs()

271 -

268 # Create generator for this chunk and wrap it to return indices

269 original_generator = self.engine_client.encode(

270 chunk_engine_prompt,

271 pooling_params,

272 chunk_request_id,

273 lora_request=ctx.lora_request,

278 - tokenization_kwargs=tokenization_kwargs,

274 trace_headers=trace_headers,

275 priority=ctx.request.priority,

276 )

@@ -362,7 +357,7 @@ class OpenAIServingEmbedding(OpenAIServing):

357 async def _create_single_prompt_generator(

358 self,

359 ctx: EmbeddingServeContext,

365 - engine_prompt: TokPrompt,

360 + engine_prompt: ProcessorInputs,

361 pooling_params: PoolingParams,

362 trace_headers: Mapping[str, str] | None,

363 prompt_index: int,

@@ -377,16 +372,12 @@ class OpenAIServingEmbedding(OpenAIServing):

372 lora_request=ctx.lora_request,

373 )

374

380 - tok_params = ctx.request.build_tok_params(self.model_config)

381 - tokenization_kwargs = tok_params.get_encode_kwargs()

382 -

375 # Return the original generator without wrapping

376 return self.engine_client.encode(

377 engine_prompt,

378 pooling_params,

379 request_id_item,

380 lora_request=ctx.lora_request,

389 - tokenization_kwargs=tokenization_kwargs,

381 trace_headers=trace_headers,

382 priority=ctx.request.priority,

383 )

vllm/entrypoints/pooling/pooling/serving.py

+2 -7

@@ -33,10 +33,9 @@ from vllm.entrypoints.pooling.utils import (

33 encode_pooling_output_base64,

34 encode_pooling_output_float,

35 )

36 -from vllm.inputs import PromptType

36 +from vllm.inputs import ProcessorInputs

37 from vllm.logger import init_logger

38 from vllm.outputs import PoolingRequestOutput

39 -from vllm.renderers.inputs import TokPrompt

39 from vllm.renderers.inputs.preprocess import prompt_to_seq

40 from vllm.utils.async_utils import merge_async_iterators

41 from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness

@@ -93,7 +92,7 @@ class OpenAIServingPooling(OpenAIServing):

92 "dimensions is currently not supported"

93 )

96 - engine_prompts: Sequence[PromptType | TokPrompt]

95 + engine_prompts: Sequence[ProcessorInputs]

96 if use_io_processor := isinstance(request, IOProcessorRequest):

97 if self.io_processor is None:

98 raise ValueError(

@@ -152,9 +151,6 @@ class OpenAIServingPooling(OpenAIServing):

151 else:

152 pooling_params = request.to_pooling_params() # type: ignore

153

155 - tok_params = request.build_tok_params(self.model_config)

156 - tokenization_kwargs = tok_params.get_encode_kwargs()

157 -

154 for i, engine_prompt in enumerate(engine_prompts):

155 request_id_item = f"{request_id}-{i}"

156

@@ -176,7 +172,6 @@ class OpenAIServingPooling(OpenAIServing):

172 pooling_params,

173 request_id_item,

174 lora_request=lora_request,

179 - tokenization_kwargs=tokenization_kwargs,

175 trace_headers=trace_headers,

176 priority=request.priority,

177 )

vllm/entrypoints/pooling/score/serving.py

+13 -7

@@ -35,7 +35,7 @@ from vllm.entrypoints.pooling.score.utils import (

35 get_score_prompt,

36 validate_score_input,

37 )

38 -from vllm.inputs.data import TokensPrompt

38 +from vllm.inputs.data import ProcessorInputs, TokensPrompt, token_inputs

39 from vllm.logger import init_logger

40 from vllm.lora.request import LoRARequest

41 from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput

@@ -108,12 +108,15 @@ class ServingScores(OpenAIServing):

108 *(encode_async(t, **tokenization_kwargs) for t in input_texts)

109 )

110

111 - engine_prompts: list[TokensPrompt] = []

111 + engine_prompts: list[ProcessorInputs] = []

112 for tok_result, input_text in zip(tokenized_prompts, input_texts):

113 text_token_prompt = self._validate_input(request, tok_result, input_text)

114

115 engine_prompts.append(

116 - TokensPrompt(prompt_token_ids=text_token_prompt["prompt_token_ids"])

116 + token_inputs(

117 + text_token_prompt["prompt_token_ids"],

118 + prompt=input_text,

119 + )

120 )

121

122 # Schedule the request and get the result generator.

@@ -125,7 +128,7 @@ class ServingScores(OpenAIServing):

128

129 self._log_inputs(

130 request_id_item,

128 - input_texts[i],

131 + engine_prompt,

132 params=pooling_params,

133 lora_request=lora_request,

134 )

@@ -207,12 +210,15 @@ class ServingScores(OpenAIServing):

210 *(encode_async(t, **tokenization_kwargs) for t in input_texts)

211 )

212

210 - engine_prompts: list[TokensPrompt] = []

213 + engine_prompts: list[ProcessorInputs] = []

214 for tok_result, input_text in zip(tokenized_prompts, input_texts):

215 text_token_prompt = self._validate_input(request, tok_result, input_text)

216

217 engine_prompts.append(

215 - TokensPrompt(prompt_token_ids=text_token_prompt["prompt_token_ids"])

218 + token_inputs(

219 + text_token_prompt["prompt_token_ids"],

220 + prompt=input_text,

221 + )

222 )

223

224 # Schedule the request and get the result generator.

@@ -225,7 +231,7 @@ class ServingScores(OpenAIServing):

231

232 self._log_inputs(

233 request_id_item,

228 - input_texts[i],

234 + engine_prompt,

235 params=pooling_params,

236 lora_request=lora_request,

237 )

vllm/entrypoints/serve/disagg/serving.py

+2 -17

@@ -29,7 +29,6 @@ from vllm.entrypoints.serve.disagg.protocol import (

29 GenerateResponse,

30 GenerateResponseChoice,

31 )

32 -from vllm.inputs.data import TokensPrompt

32 from vllm.logger import init_logger

33 from vllm.logprobs import Logprob

34 from vllm.outputs import RequestOutput

@@ -116,7 +115,7 @@ class ServingTokens(OpenAIServing):

115

116 self._log_inputs(

117 request_id,

119 - TokensPrompt(prompt_token_ids=request.token_ids),

118 + engine_prompt,

119 params=sampling_params,

120 lora_request=lora_request,

121 )

@@ -127,27 +126,13 @@ class ServingTokens(OpenAIServing):

126 else await self._get_trace_headers(raw_request.headers)

127 )

128

130 - tok_params = request.build_tok_params(self.model_config)

131 - tokenization_kwargs = tok_params.get_encode_kwargs()

132 -

133 - engine_request = self.input_processor.process_inputs(

134 - request_id,

135 - engine_prompt,

136 - sampling_params,

137 - lora_request=lora_request,

138 - tokenization_kwargs=tokenization_kwargs,

139 - trace_headers=trace_headers,

140 - priority=request.priority,

141 - )

142 -

129 result_generator = self.engine_client.generate(

144 - engine_request,

130 + engine_prompt,

131 sampling_params,

132 request_id,

133 lora_request=lora_request,

134 trace_headers=trace_headers,

135 priority=request.priority,

150 - tokenization_kwargs=tokenization_kwargs,

136 )

137

138 except ValueError as e:

vllm/entrypoints/serve/tokenize/serving.py

+2 -2

@@ -20,7 +20,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (

20 TokenizeResponse,

21 TokenizerInfoResponse,

22 )

23 -from vllm.inputs import TokensPrompt

23 +from vllm.inputs import TokensPrompt, token_inputs

24 from vllm.logger import init_logger

25 from vllm.tokenizers import TokenizerLike

@@ -135,7 +135,7 @@ class OpenAIServingTokenization(OpenAIServing):

135

136 self._log_inputs(

137 request_id,

138 - TokensPrompt(prompt_token_ids=request.tokens),

138 + token_inputs(request.tokens),

139 params=None,

140 lora_request=lora_request,

141 )

vllm/inputs/data.py

@@ -187,6 +187,9 @@ class _InputOptions(TypedDict):

187 Additional options available to all input types.

188 """

189

190 + arrival_time: NotRequired[float]

191 + """The time when the input was received (before rendering)."""

192 +

193 cache_salt: NotRequired[str]

194 """Optional cache salt to be used for prefix caching."""

195

@@ -300,6 +303,9 @@ class EncoderDecoderInputs(TypedDict):

303 decoder_prompt: DecoderInputs

304 """The inputs for the decoder portion."""

305

306 + arrival_time: NotRequired[float]

307 + """The time when the input was received (before rendering)."""

308 +

309

310 ProcessorInputs: TypeAlias = DecoderOnlyInputs | EncoderDecoderInputs

311 """

vllm/inputs/preprocess.py

+15 -85

@@ -19,11 +19,9 @@ from vllm.renderers import BaseRenderer, renderer_from_config

19 from vllm.renderers.inputs import (

20 DecoderDictPrompt,

21 DecoderOnlyDictPrompt,

22 - DictPrompt,

22 EncoderDecoderDictPrompt,

23 EncoderDictPrompt,

24 SingletonDictPrompt,

26 - TokPrompt,

25 )

26 from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt

27 from vllm.tokenizers import TokenizerLike

@@ -41,7 +39,6 @@ from .data import (

39 TextPrompt,

40 TokenInputs,

41 TokensPrompt,

44 - embeds_inputs,

42 token_inputs,

43 )

@@ -83,7 +80,7 @@ class InputPreprocessor:

80 **(tokenization_kwargs or {})

81 )

86 - tok_prompt = renderer.tokenize_prompt(

83 + tok_prompt = renderer._tokenize_singleton_prompt(

84 TextPrompt(prompt=prompt),

85 tok_params,

86 )

@@ -103,17 +100,10 @@ class InputPreprocessor:

100 Apply the model's multi-modal processor to a multi-modal prompt,

101 returning the corresponding token IDs and metadata.

102 """

106 - mm_processor = self.renderer.get_mm_processor()

107 -

108 - if mm_processor_kwargs is None:

109 - mm_processor_kwargs = {}

110 -

111 - mm_items = mm_processor.info.parse_mm_data(mm_data)

112 -

113 - return mm_processor.apply(

103 + return self.renderer._process_multimodal(

104 prompt,

115 - mm_items,

116 - hf_processor_mm_kwargs=mm_processor_kwargs,

105 + mm_data,

106 + mm_processor_kwargs=mm_processor_kwargs,

107 tokenization_kwargs=tokenization_kwargs,

108 mm_uuids=mm_uuids,

109 )

@@ -122,31 +112,7 @@ class InputPreprocessor:

112 self,

113 parsed_content: EmbedsPrompt,

114 ) -> EmbedsInputs:

125 - if not self.model_config.enable_prompt_embeds:

126 - raise ValueError(

127 - "You must set `--enable-prompt-embeds` to input `prompt_embeds`."

128 - )

129 -

130 - prompt_embeds = parsed_content["prompt_embeds"]

131 -

132 - # prompt_embeds must be (seq_len, hidden_size), but if the user

133 - # passes in a batch of size 1, i.e. (1, seq_len, hidden_size),

134 - # we can unambiguously process the intent by squeezing the batch

135 - # dimension.

136 - if prompt_embeds.ndim == 3:

137 - prompt_embeds = prompt_embeds.squeeze(dim=0)

138 -

139 - if prompt_embeds.ndim != 2:

140 - raise ValueError("prompt_embeds must be of shape (seq_len, hidden_size).")

141 -

142 - # Tensors must be on CPU for serialization between processes

143 - # in the MsgpackEncoder. Casting to CPU here ensures that there is no

144 - # hidden device transfer in the critical path of generation.

145 - prompt_embeds = prompt_embeds.cpu()

146 -

147 - return embeds_inputs(

148 - prompt_embeds=prompt_embeds, cache_salt=parsed_content.get("cache_salt")

149 - )

115 + return self.renderer._process_embeds(parsed_content)

116

117 def _truncate_inputs(

118 self, inputs: list[int], tokenization_kwargs: dict[str, Any] | None = None

@@ -157,7 +123,7 @@ class InputPreprocessor:

123 **(tokenization_kwargs or {})

124 )

125

160 - tok_prompt = renderer.tokenize_prompt(

126 + tok_prompt = renderer._tokenize_singleton_prompt(

127 TokensPrompt(prompt_token_ids=inputs),

128 tok_params,

129 )

@@ -168,8 +134,6 @@ class InputPreprocessor:

134 self,

135 parsed_content: TokensPrompt,

136 tokenization_kwargs: dict[str, Any] | None = None,

171 - *,

172 - mm_uuids: MultiModalUUIDDict | None = None,

137 ) -> TokenInputs | MultiModalInputs:

138 prompt_token_ids = self._truncate_inputs(

139 parsed_content["prompt_token_ids"], tokenization_kwargs

@@ -182,11 +146,13 @@ class InputPreprocessor:

146 multi_modal_data,

147 parsed_content.get("mm_processor_kwargs") or {},

148 tokenization_kwargs=tokenization_kwargs,

185 - mm_uuids=mm_uuids,

149 + mm_uuids=parsed_content.get("multi_modal_uuids"),

150 )

151 else:

152 inputs = token_inputs(prompt_token_ids)

153

154 + if prompt_text := parsed_content.get("prompt"):

155 + inputs["prompt"] = prompt_text

156 if cache_salt := parsed_content.get("cache_salt"):

157 inputs["cache_salt"] = cache_salt

158

@@ -196,8 +162,6 @@ class InputPreprocessor:

162 self,

163 parsed_content: TextPrompt,

164 tokenization_kwargs: dict[str, Any] | None = None,

199 - *,

200 - mm_uuids: MultiModalUUIDDict | None = None,

165 ) -> TokenInputs | MultiModalInputs:

166 prompt_text = parsed_content["prompt"]

167

@@ -208,7 +172,6 @@ class InputPreprocessor:

172 multi_modal_data,

173 parsed_content.get("mm_processor_kwargs") or {},

174 tokenization_kwargs=tokenization_kwargs,

211 - mm_uuids=mm_uuids,

175 )

176 else:

177 prompt_token_ids = self._tokenize_prompt(

@@ -217,6 +180,8 @@ class InputPreprocessor:

180 )

181 inputs = token_inputs(prompt_token_ids)

182

183 + inputs["prompt"] = prompt_text

184 +

185 if cache_salt := parsed_content.get("cache_salt"):

186 inputs["cache_salt"] = cache_salt

187

@@ -227,8 +192,6 @@ class InputPreprocessor:

192 self,

193 prompt: EncoderDictPrompt,

194 tokenization_kwargs: dict[str, Any] | None = None,

230 - *,

231 - mm_uuids: MultiModalUUIDDict | None = None,

195 ) -> EncoderInputs: ...

196

197 @overload

@@ -236,8 +199,6 @@ class InputPreprocessor:

199 self,

200 prompt: DecoderDictPrompt,

201 tokenization_kwargs: dict[str, Any] | None = None,

239 - *,

240 - mm_uuids: MultiModalUUIDDict | None = None,

202 ) -> DecoderInputs: ...

203

204 @overload

@@ -245,16 +206,12 @@ class InputPreprocessor:

206 self,

207 prompt: DecoderOnlyDictPrompt,

208 tokenization_kwargs: dict[str, Any] | None = None,

248 - *,

249 - mm_uuids: MultiModalUUIDDict | None = None,

209 ) -> DecoderOnlyInputs: ...

210

211 def _prompt_to_llm_inputs(

212 self,

213 prompt: SingletonDictPrompt,

214 tokenization_kwargs: dict[str, Any] | None = None,

256 - *,

257 - mm_uuids: MultiModalUUIDDict | None = None,

215 ) -> SingletonInputs:

216 """

217 Extract the singleton inputs from a prompt.

@@ -271,16 +228,12 @@ class InputPreprocessor:

228 return self._process_embeds(prompt) # type: ignore[arg-type]

229

230 if "prompt_token_ids" in prompt:

274 - return self._process_tokens(

275 - prompt, # type: ignore[arg-type]

276 - mm_uuids=mm_uuids,

277 - )

231 + return self._process_tokens(prompt) # type: ignore[arg-type]

232

233 if "prompt" in prompt:

234 return self._process_text(

235 prompt, # type: ignore[arg-type]

236 tokenization_kwargs=tokenization_kwargs,

283 - mm_uuids=mm_uuids,

237 )

238

239 assert_never(prompt) # type: ignore[arg-type]

@@ -289,8 +242,6 @@ class InputPreprocessor:

242 self,

243 prompt: EncoderDecoderDictPrompt,

244 tokenization_kwargs: dict[str, Any] | None = None,

292 - *,

293 - mm_uuids: MultiModalUUIDDict | None = None,

245 ) -> EncoderDecoderInputs:

246 """

247 For encoder/decoder models only:

@@ -314,7 +265,6 @@ class InputPreprocessor:

265 encoder_inputs=self._prompt_to_llm_inputs(

266 encoder_prompt,

267 tokenization_kwargs=tokenization_kwargs,

317 - mm_uuids=mm_uuids,

268 ),

269 decoder_inputs=(

270 None

@@ -331,8 +281,6 @@ class InputPreprocessor:

281 self,

282 prompt: DecoderOnlyDictPrompt,

283 tokenization_kwargs: dict[str, Any] | None = None,

334 - *,

335 - mm_uuids: MultiModalUUIDDict | None = None,

284 ) -> DecoderOnlyInputs:

285 """

286 For decoder-only models:

@@ -350,41 +298,23 @@ class InputPreprocessor:

298 return self._prompt_to_llm_inputs(

299 prompt,

300 tokenization_kwargs=tokenization_kwargs,

353 - mm_uuids=mm_uuids,

301 )

302

356 - def _preprocess(

303 + def preprocess(

304 self,

358 - prompt: PromptType | DictPrompt | TokPrompt,

305 + prompt: PromptType,

306 tokenization_kwargs: dict[str, Any] | None = None,

360 - *,

361 - mm_uuids: MultiModalUUIDDict | None = None,

307 ) -> ProcessorInputs:

308 + """Preprocess the input prompt."""

309 if self.model_config.is_encoder_decoder:

310 # Encoder-decoder model requires special mapping of

311 # input prompts to encoder & decoder.

312 return self._process_encoder_decoder_prompt(

313 parse_enc_dec_prompt(prompt),

314 tokenization_kwargs,

369 - mm_uuids=mm_uuids,

315 )

316

317 return self._process_decoder_only_prompt(

318 parse_dec_only_prompt(prompt),

319 tokenization_kwargs=tokenization_kwargs,

375 - mm_uuids=mm_uuids,

320 )

377 -

378 - def preprocess(

379 - self,

380 - prompt: PromptType | DictPrompt | TokPrompt,

381 - tokenization_kwargs: dict[str, Any] | None = None,

382 - *,

383 - mm_uuids: MultiModalUUIDDict | None = None,

384 - ) -> ProcessorInputs:

385 - """Preprocess the input prompt."""

386 - res = self._preprocess(prompt, tokenization_kwargs, mm_uuids=mm_uuids)

387 -

388 - self.renderer.update_mm_cache_stats()

389 -

390 - return res

vllm/model_executor/models/funasr.py

+3 -15

@@ -48,7 +48,6 @@ from vllm.multimodal.processing import (

48 BaseProcessingInfo,

49 PromptReplacement,

50 PromptUpdate,

51 - PromptUpdateDetails,

51 )

52 from vllm.transformers_utils.processor import cached_processor_from_config

53 from vllm.transformers_utils.processors.funasr_processor import FunASRFeatureExtractor

@@ -810,13 +809,7 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):

809 out_mm_kwargs: MultiModalKwargsItems,

810 ) -> Sequence[PromptUpdate]:

811 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

813 - tokenizer = self.info.get_tokenizer()

814 - vocab = tokenizer.get_vocab()

815 -

816 - # Use getattr with default to be compatible with transformers<4.48

817 - audio_token = getattr(processor, "audio_token", "<|AUDIO|>")

818 -

819 - audio_token_id = vocab[audio_token]

812 + audio_token_id = processor.audio_token_id

813

814 out_mm_data = out_mm_kwargs.get_data()

815

@@ -836,17 +829,12 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):

829 assert len(audio_embeds.shape) == 2, "audio_embeds must be a 2D tensor"

830 num_features = audio_embeds.shape[0]

831

839 - audio_tokens = [audio_token_id] * num_features

840 -

841 - return PromptUpdateDetails.select_token_id(

842 - audio_tokens,

843 - embed_token_id=audio_token_id,

844 - )

832 + return [audio_token_id] * num_features

833

834 return [

835 PromptReplacement(

836 modality="audio",

849 - target=audio_token,

837 + target=[audio_token_id],

838 replacement=get_replacement_qwen2_audio,

839 )

840 ]

vllm/model_executor/models/qwen2_audio.py

+6 -20

@@ -59,7 +59,6 @@ from vllm.multimodal.processing import (

59 BaseProcessingInfo,

60 PromptReplacement,

61 PromptUpdate,

62 - PromptUpdateDetails,

62 )

63 from vllm.sequence import IntermediateTensors

64 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -187,8 +186,10 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn

186

187 hf_processor = self.info.get_hf_processor()

188 audio_token = hf_processor.audio_token

189 + audio_bos_token = hf_processor.audio_bos_token

190 + audio_eos_token = hf_processor.audio_eos_token

191

191 - return audio_token * num_audios

192 + return (audio_bos_token + audio_token + audio_eos_token) * num_audios

193

194 def get_dummy_mm_data(

195 self,

@@ -262,17 +263,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing

263 out_mm_kwargs: MultiModalKwargsItems,

264 ) -> Sequence[PromptUpdate]:

265 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

265 - tokenizer = self.info.get_tokenizer()

266 - vocab = tokenizer.get_vocab()

267 -

268 - # Use getattr with default to be compatible with transformers<4.48

269 - audio_token = getattr(processor, "audio_token", "<|AUDIO|>")

270 - audio_bos_token = getattr(processor, "audio_bos_token", "<|audio_bos|>")

271 - audio_eos_token = getattr(processor, "audio_eos_token", "<|audio_eos|>")

272 -

273 - audio_token_id = vocab[audio_token]

274 - audio_bos_id = vocab[audio_bos_token]

275 - audio_eos_id = vocab[audio_eos_token]

266 + audio_token_id = processor.audio_token_id

267

268 out_mm_data = out_mm_kwargs.get_data()

269 feature_attention_mask = out_mm_data.get("feature_attention_mask")

@@ -303,17 +294,12 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing

294 "to be represented inside the model"

295 )

296

306 - audio_tokens = [audio_token_id] * num_features

307 -

308 - return PromptUpdateDetails.select_token_id(

309 - [audio_bos_id] + audio_tokens + [audio_eos_id],

310 - embed_token_id=audio_token_id,

311 - )

297 + return [audio_token_id] * num_features

298

299 return [

300 PromptReplacement(

301 modality="audio",

316 - target=audio_token,

302 + target=[audio_token_id],

303 replacement=get_replacement_qwen2_audio,

304 )

305 ]

vllm/multimodal/processing/processor.py

@@ -1843,15 +1843,18 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1843 tokenizer = self.info.get_tokenizer()

1844 decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)

1845 if isinstance(decoder_prompt_raw, str):

1846 + decoder_prompt_text = decoder_prompt_raw

1847 decoder_prompt_ids = tokenizer.encode(

1848 decoder_prompt_raw, add_special_tokens=False

1849 )

1850 else:

1851 + decoder_prompt_text = None

1852 decoder_prompt_ids = decoder_prompt_raw

1853

1854 return mm_enc_dec_inputs(

1855 encoder_inputs,

1856 decoder_prompt_ids,

1857 + decoder_prompt=decoder_prompt_text,

1858 )

1859

1860 def apply(

vllm/platforms/interface.py

+1 -2

@@ -19,7 +19,6 @@ if TYPE_CHECKING:

19 from vllm.config import VllmConfig

20 from vllm.inputs import ProcessorInputs, PromptType

21 from vllm.pooling_params import PoolingParams

22 - from vllm.renderers.inputs import DictPrompt, TokPrompt

22 from vllm.sampling_params import SamplingParams

23 from vllm.utils.argparse_utils import FlexibleArgumentParser

24 from vllm.v1.attention.selector import AttentionSelectorConfig

@@ -569,7 +568,7 @@ class Platform:

568 @classmethod

569 def validate_request(

570 cls,

572 - prompt: "PromptType | DictPrompt | TokPrompt",

571 + prompt: "PromptType | ProcessorInputs",

572 params: "SamplingParams | PoolingParams",

573 processed_inputs: "ProcessorInputs",

574 ) -> None:

vllm/renderers/base.py

+311 -79

@@ -1,17 +1,29 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import asyncio

4 +import time

5 from abc import ABC, abstractmethod

5 -from collections.abc import Sequence

6 +from collections.abc import Mapping, Sequence

7 from functools import cached_property

8 from typing import TYPE_CHECKING, Any, Generic, overload

10 from typing_extensions import TypeVar

11 -from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

12 +from vllm.inputs import (

13 + EmbedsInputs,

14 + EmbedsPrompt,

15 + EncoderDecoderInputs,

16 + ProcessorInputs,

17 + SingletonInputs,

18 + TextPrompt,

19 + TokenInputs,

20 + TokensPrompt,

21 +)

22 +from vllm.inputs.data import build_enc_dec_inputs, embeds_inputs, token_inputs

23 from vllm.logger import init_logger

24 from vllm.tokenizers import TokenizerLike

25 from vllm.utils.async_utils import AsyncMicrobatchTokenizer

26 +from vllm.utils.counter import AtomicCounter

27 from vllm.utils.torch_utils import set_default_torch_num_threads

28 from vllm.v1.metrics.stats import MultiModalCacheStats

@@ -20,6 +32,8 @@ from .inputs import (

32 DictPrompt,

33 EncoderDecoderDictPrompt,

34 EncoderDecoderTokPrompt,

35 + SingletonDictPrompt,

36 + SingletonTokPrompt,

37 TokPrompt,

38 )

39 from .inputs.preprocess import extract_target_prompt

@@ -32,6 +46,12 @@ if TYPE_CHECKING:

46 ConversationMessage,

47 )

48 from vllm.multimodal.cache import BaseMultiModalProcessorCache

49 + from vllm.multimodal.inputs import (

50 + MultiModalDataDict,

51 + MultiModalInputs,

52 + MultiModalUUIDDict,

53 + )

54 + from vllm.multimodal.parse import MultiModalDataItems

55 from vllm.multimodal.processing import BaseMultiModalProcessor

57 logger = init_logger(__name__)

@@ -79,6 +99,10 @@ class BaseRenderer(ABC, Generic[_T]):

99 if mm_processor_cache:

100 self._mm_cache_stats = MultiModalCacheStats()

101

102 + # This is used to generate internal request ID for MM processing

103 + # It has no relation to the request ID for engine core

104 + self._mm_req_counter = AtomicCounter()

105 +

106 def get_tokenizer(self) -> _T:

107 tokenizer = self.tokenizer

108 if tokenizer is None:

@@ -284,17 +308,79 @@ class BaseRenderer(ABC, Generic[_T]):

308

309 return prompt

310

311 + @overload

312 + def _tokenize_singleton_prompt(

313 + self,

314 + prompt: TextPrompt | TokensPrompt,

315 + params: TokenizeParams,

316 + ) -> TokensPrompt: ...

317 +

318 + @overload

319 + def _tokenize_singleton_prompt( # type: ignore[misc]

320 + self,

321 + prompt: EmbedsPrompt,

322 + params: TokenizeParams,

323 + ) -> EmbedsPrompt: ...

324 +

325 + def _tokenize_singleton_prompt(

326 + self,

327 + prompt: SingletonDictPrompt,

328 + params: TokenizeParams,

329 + ) -> SingletonTokPrompt:

330 + if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

331 + prompt = params.apply_pre_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

332 + prompt = self._tokenize_prompt(prompt, params)

333 +

334 + if params.needs_detokenization and "prompt" not in prompt:

335 + if "prompt_token_ids" not in prompt:

336 + raise RuntimeError("Cannot run detokenization on embeddings")

337 +

338 + prompt = self._detokenize_prompt(prompt) # type: ignore[arg-type]

339 +

340 + return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

341 +

342 + @overload

343 + async def _tokenize_singleton_prompt_async(

344 + self,

345 + prompt: TextPrompt | TokensPrompt,

346 + params: TokenizeParams,

347 + ) -> TokensPrompt: ...

348 +

349 + @overload

350 + async def _tokenize_singleton_prompt_async( # type: ignore[misc]

351 + self,

352 + prompt: EmbedsPrompt,

353 + params: TokenizeParams,

354 + ) -> EmbedsPrompt: ...

355 +

356 + async def _tokenize_singleton_prompt_async(

357 + self,

358 + prompt: SingletonDictPrompt,

359 + params: TokenizeParams,

360 + ) -> SingletonTokPrompt:

361 + if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

362 + prompt = params.apply_pre_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

363 + prompt = await self._tokenize_prompt_async(prompt, params)

364 +

365 + if params.needs_detokenization and "prompt" not in prompt:

366 + if "prompt_token_ids" not in prompt:

367 + raise RuntimeError("Cannot run detokenization on embeddings")

368 +

369 + prompt = await self._detokenize_prompt_async(prompt) # type: ignore[arg-type]

370 +

371 + return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

372 +

373 def _tokenize_enc_dec_prompt(

374 self,

375 prompt: EncoderDecoderDictPrompt,

376 params: TokenizeParams,

377 ) -> EncoderDecoderTokPrompt:

378 enc_prompt, dec_prompt = (

293 - self.tokenize_prompt(prompt["encoder_prompt"], params),

379 + self._tokenize_singleton_prompt(prompt["encoder_prompt"], params),

380 (

381 None

382 if prompt["decoder_prompt"] is None

297 - else self.tokenize_prompt(prompt["decoder_prompt"], params)

383 + else self._tokenize_singleton_prompt(prompt["decoder_prompt"], params)

384 ),

385 )

386

@@ -309,11 +395,13 @@ class BaseRenderer(ABC, Generic[_T]):

395 params: TokenizeParams,

396 ) -> EncoderDecoderTokPrompt:

397 enc_prompt, dec_prompt = await asyncio.gather(

312 - self.tokenize_prompt_async(prompt["encoder_prompt"], params),

398 + self._tokenize_singleton_prompt_async(prompt["encoder_prompt"], params),

399 (

400 asyncio.sleep(0)

401 if prompt["decoder_prompt"] is None

316 - else self.tokenize_prompt_async(prompt["decoder_prompt"], params)

402 + else self._tokenize_singleton_prompt_async(

403 + prompt["decoder_prompt"], params

404 + )

405 ),

406 )

407

@@ -322,27 +410,6 @@ class BaseRenderer(ABC, Generic[_T]):

410 decoder_prompt=dec_prompt,

411 )

412

325 - @overload

326 - def tokenize_prompt(

327 - self,

328 - prompt: TextPrompt | TokensPrompt,

329 - params: TokenizeParams,

330 - ) -> TokensPrompt: ...

331 -

332 - @overload

333 - def tokenize_prompt( # type: ignore[misc]

334 - self,

335 - prompt: EmbedsPrompt,

336 - params: TokenizeParams,

337 - ) -> EmbedsPrompt: ...

338 -

339 - @overload

340 - def tokenize_prompt( # type: ignore[misc]

341 - self,

342 - prompt: EncoderDecoderDictPrompt,

343 - params: TokenizeParams,

344 - ) -> EncoderDecoderTokPrompt: ...

345 -

413 def tokenize_prompt(

414 self,

415 prompt: DictPrompt,

@@ -351,17 +418,7 @@ class BaseRenderer(ABC, Generic[_T]):

418 if "encoder_prompt" in prompt:

419 return self._tokenize_enc_dec_prompt(prompt, params) # type: ignore[arg-type]

420

354 - if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

355 - prompt = params.apply_pre_tokenization(self.tokenizer, prompt)

356 - prompt = self._tokenize_prompt(prompt, params)

357 -

358 - if params.needs_detokenization and "prompt" not in prompt:

359 - if "prompt_token_ids" not in prompt:

360 - raise RuntimeError("Cannot run detokenization on embeddings")

361 -

362 - prompt = self._detokenize_prompt(prompt) # type: ignore[arg-type]

363 -

364 - return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

421 + return self._tokenize_singleton_prompt(prompt, params)

422

423 def tokenize_prompts(

424 self,

@@ -370,27 +427,6 @@ class BaseRenderer(ABC, Generic[_T]):

427 ) -> list[TokPrompt]:

428 return [self.tokenize_prompt(prompt, params) for prompt in prompts]

429

373 - @overload

374 - async def tokenize_prompt_async(

375 - self,

376 - prompt: TextPrompt | TokensPrompt,

377 - params: TokenizeParams,

378 - ) -> TokensPrompt: ...

379 -

380 - @overload

381 - async def tokenize_prompt_async( # type: ignore[misc]

382 - self,

383 - prompt: EmbedsPrompt,

384 - params: TokenizeParams,

385 - ) -> EmbedsPrompt: ...

386 -

387 - @overload

388 - async def tokenize_prompt_async( # type: ignore[misc]

389 - self,

390 - prompt: EncoderDecoderDictPrompt,

391 - params: TokenizeParams,

392 - ) -> EncoderDecoderTokPrompt: ...

393 -

430 async def tokenize_prompt_async(

431 self,

432 prompt: DictPrompt,

@@ -399,17 +435,7 @@ class BaseRenderer(ABC, Generic[_T]):

435 if "encoder_prompt" in prompt:

436 return await self._tokenize_enc_dec_prompt_async(prompt, params) # type: ignore[arg-type]

437

402 - if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:

403 - prompt = params.apply_pre_tokenization(self.tokenizer, prompt)

404 - prompt = await self._tokenize_prompt_async(prompt, params)

405 -

406 - if params.needs_detokenization and "prompt" not in prompt:

407 - if "prompt_token_ids" not in prompt:

408 - raise RuntimeError("Cannot run detokenization on embeddings")

409 -

410 - prompt = await self._detokenize_prompt_async(prompt) # type: ignore[arg-type]

411 -

412 - return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]

438 + return await self._tokenize_singleton_prompt_async(prompt, params)

439

440 async def tokenize_prompts_async(

441 self,

@@ -423,7 +449,7 @@ class BaseRenderer(ABC, Generic[_T]):

449 # Step 3: Add extra keys to the prompts

450 def _apply_prompt_extras(

451 self,

426 - prompts: Sequence[DictPrompt | TokPrompt],

452 + prompts: Sequence[TokPrompt],

453 prompt_extras: dict[str, Any] | None,

454 ):

455 if not prompt_extras:

@@ -433,6 +459,200 @@ class BaseRenderer(ABC, Generic[_T]):

459 target_prompt = extract_target_prompt(self.model_config, prompt)

460 target_prompt.update(prompt_extras) # type: ignore[arg-type]

461

462 + # Step 4: Convert to engine inputs

463 + def _validate_mm_uuids(

464 + self,

465 + mm_data: "MultiModalDataDict",

466 + mm_items: "MultiModalDataItems",

467 + mm_uuids: "MultiModalUUIDDict | None",

468 + ) -> None:

469 + if mm_uuids is None:

470 + mm_uuids = {}

471 +

472 + # NOTE: Keys corresponding to `None` in `mm_data` don't appear in `mm_items`

473 + modalities = mm_data.keys() | mm_uuids.keys()

474 +

475 + for modality in modalities:

476 + data_items = mm_items.get(modality) or list[Any]()

477 +

478 + uuid_items = mm_uuids.get(modality) or list[str | None]()

479 + if isinstance(uuid_items, str):

480 + uuid_items = [uuid_items]

481 +

482 + if len(data_items) > 0:

483 + if len(uuid_items) > 0 and len(data_items) != len(uuid_items):

484 + raise ValueError(

485 + f"If given, multi_modal_uuids[{modality!r}] must have "

486 + f"same length as multi_modal_data[{modality!r}], but "

487 + f"got {len(uuid_items)} vs {len(data_items)}."

488 + )

489 +

490 + for i, item in enumerate(data_items):

491 + if item is None:

492 + if not uuid_items:

493 + raise ValueError(

494 + f"multi_modal_data[{modality!r}][{i}] is empty but "

495 + f"multi_modal_uuids[{modality!r}] is missing."

496 + )

497 +

498 + if uuid_items[i] is None:

499 + raise ValueError(

500 + f"multi_modal_data[{modality!r}][{i}] is empty but "

501 + f"multi_modal_uuids[{modality!r}][{i}] is missing."

502 + )

503 +

504 + def _process_mm_uuids(

505 + self,

506 + mm_data: "MultiModalDataDict",

507 + mm_items: "MultiModalDataItems",

508 + mm_uuids: "MultiModalUUIDDict | None",

509 + mm_req_id: str,

510 + ):

511 + model_config = self.model_config

512 +

513 + # NOTE: When users explicitly turn off BOTH prefix caching and input

514 + # processing caching, no multimodal features or embeddings will be

515 + # reused across requests, therefore identifying multimodal data items

516 + # by their content is no longer necessary, and we create uuids with

517 + # `<mm_req_id>-<modality>-<index>`, overriding even user-provided ones.

518 + if (

519 + model_config.multimodal_config

520 + and model_config.multimodal_config.mm_processor_cache_gb == 0

521 + and not self.config.cache_config.enable_prefix_caching

522 + ):

523 + mm_uuids = {

524 + modality: [f"{mm_req_id}-{modality}-{i}" for i in range(data_count)]

525 + for modality, data_count in mm_items.get_all_counts().items()

526 + }

527 +

528 + self._validate_mm_uuids(mm_data, mm_items, mm_uuids)

529 +

530 + return mm_uuids

531 +

532 + # TODO: Remove str and tokenization_kwargs after deprecating InputPreprocessor

533 + def _process_multimodal(

534 + self,

535 + prompt: list[int] | str,

536 + mm_data: "MultiModalDataDict",

537 + mm_processor_kwargs: Mapping[str, object] | None,

538 + tokenization_kwargs: dict[str, Any] | None,

539 + mm_uuids: "MultiModalUUIDDict | None",

540 + ) -> "MultiModalInputs":

541 + from vllm.multimodal.processing.context import set_request_id

542 +

543 + mm_req_id = f"renderer-mm-{self._mm_req_counter.inc(1)}"

544 +

545 + mm_processor = self.get_mm_processor()

546 +

547 + mm_items = mm_processor.info.parse_mm_data(mm_data)

548 + mm_uuids = self._process_mm_uuids(mm_data, mm_items, mm_uuids, mm_req_id)

549 +

550 + with set_request_id(mm_req_id), set_default_torch_num_threads():

551 + mm_inputs = mm_processor.apply(

552 + prompt,

553 + mm_items,

554 + hf_processor_mm_kwargs=mm_processor_kwargs or {},

555 + tokenization_kwargs=tokenization_kwargs,

556 + mm_uuids=mm_uuids,

557 + )

558 +

559 + self.update_mm_cache_stats()

560 +

561 + return mm_inputs

562 +

563 + def _process_tokens(

564 + self,

565 + prompt: TokensPrompt,

566 + ) -> "TokenInputs | MultiModalInputs":

567 + prompt_token_ids = prompt["prompt_token_ids"]

568 +

569 + inputs: TokenInputs | MultiModalInputs

570 + if multi_modal_data := prompt.get("multi_modal_data"):

571 + inputs = self._process_multimodal(

572 + prompt_token_ids,

573 + multi_modal_data,

574 + mm_processor_kwargs=prompt.get("mm_processor_kwargs"),

575 + tokenization_kwargs=None, # Tokenization already done in Step 2

576 + mm_uuids=prompt.get("multi_modal_uuids"),

577 + )

578 + else:

579 + inputs = token_inputs(prompt_token_ids)

580 +

581 + if prompt_text := prompt.get("prompt"):

582 + inputs["prompt"] = prompt_text

583 + if cache_salt := prompt.get("cache_salt"):

584 + inputs["cache_salt"] = cache_salt

585 +

586 + return inputs

587 +

588 + def _process_embeds(

589 + self,

590 + prompt: EmbedsPrompt,

591 + ) -> EmbedsInputs:

592 + if not self.model_config.enable_prompt_embeds:

593 + raise ValueError(

594 + "You must set `--enable-prompt-embeds` to input `prompt_embeds`."

595 + )

596 +

597 + prompt_embeds = prompt["prompt_embeds"]

598 +

599 + # prompt_embeds must be (seq_len, hidden_size), but if the user

600 + # passes in a batch of size 1, i.e. (1, seq_len, hidden_size),

601 + # we can unambiguously process the intent by squeezing the batch

602 + # dimension.

603 + if prompt_embeds.ndim == 3:

604 + prompt_embeds = prompt_embeds.squeeze(dim=0)

605 +

606 + if prompt_embeds.ndim != 2:

607 + raise ValueError("prompt_embeds must be of shape (seq_len, hidden_size).")

608 +

609 + # Tensors must be on CPU for serialization between processes

610 + # in the MsgpackEncoder. Casting to CPU here ensures that there is no

611 + # hidden device transfer in the critical path of generation.

612 + prompt_embeds = prompt_embeds.cpu()

613 +

614 + return embeds_inputs(

615 + prompt_embeds=prompt_embeds,

616 + cache_salt=prompt.get("cache_salt"),

617 + )

618 +

619 + def _process_singleton(

620 + self,

621 + prompt: SingletonTokPrompt,

622 + ) -> SingletonInputs:

623 + if "prompt_embeds" in prompt:

624 + return self._process_embeds(prompt) # type: ignore[arg-type]

625 +

626 + return self._process_tokens(prompt) # type: ignore[arg-type]

627 +

628 + def _process_enc_dec(

629 + self,

630 + prompt: EncoderDecoderTokPrompt,

631 + ) -> EncoderDecoderInputs:

632 + enc_prompt = prompt["encoder_prompt"]

633 + dec_prompt = prompt["decoder_prompt"]

634 +

635 + return build_enc_dec_inputs(

636 + encoder_inputs=self._process_singleton(enc_prompt),

637 + decoder_inputs=(

638 + None if dec_prompt is None else self._process_singleton(dec_prompt)

639 + ),

640 + decoder_start_token_id=self.get_dec_start_token_id(),

641 + )

642 +

643 + def process_for_engine(

644 + self, prompt: TokPrompt, arrival_time: float

645 + ) -> ProcessorInputs:

646 + engine_prompt: ProcessorInputs

647 + if "encoder_prompt" in prompt:

648 + engine_prompt = self._process_enc_dec(prompt) # type: ignore[arg-type]

649 + else:

650 + engine_prompt = self._process_singleton(prompt)

651 +

652 + engine_prompt["arrival_time"] = arrival_time

653 +

654 + return engine_prompt

655 +

656 # Top-level methods

657 def render_cmpl(

658 self,

@@ -441,6 +661,8 @@ class BaseRenderer(ABC, Generic[_T]):

661 *,

662 prompt_extras: dict[str, Any] | None = None,

663 ):

664 + arrival_time = time.time()

665 +

666 if tok_params is None:

667 tok_params = self.default_cmpl_tok_params

668

@@ -449,8 +671,7 @@ class BaseRenderer(ABC, Generic[_T]):

671

672 self._apply_prompt_extras(tok_prompts, prompt_extras)

673

452 - # TODO: Apply multi-modal processor

453 - return tok_prompts

674 + return [self.process_for_engine(prompt, arrival_time) for prompt in tok_prompts]

675

676 async def render_cmpl_async(

677 self,

@@ -459,6 +680,8 @@ class BaseRenderer(ABC, Generic[_T]):

680 *,

681 prompt_extras: dict[str, Any] | None = None,

682 ):

683 + arrival_time = time.time()

684 +

685 if tok_params is None:

686 tok_params = self.default_cmpl_tok_params

687

@@ -467,8 +690,7 @@ class BaseRenderer(ABC, Generic[_T]):

690

691 self._apply_prompt_extras(tok_prompts, prompt_extras)

692

470 - # TODO: Apply multi-modal processor

471 - return tok_prompts

693 + return [self.process_for_engine(prompt, arrival_time) for prompt in tok_prompts]

694

695 def render_chat(

696 self,

@@ -478,6 +700,8 @@ class BaseRenderer(ABC, Generic[_T]):

700 *,

701 prompt_extras: dict[str, Any] | None = None,

702 ):

703 + arrival_time = time.time()

704 +

705 if tok_params is None:

706 tok_params = self.default_chat_tok_params

707

@@ -496,8 +720,11 @@ class BaseRenderer(ABC, Generic[_T]):

720

721 self._apply_prompt_extras(tok_prompts, prompt_extras)

722

499 - # TODO: Apply multi-modal processor

500 - return out_conversations, tok_prompts

723 + eng_prompts = [

724 + self.process_for_engine(prompt, arrival_time) for prompt in tok_prompts

725 + ]

726 +

727 + return out_conversations, eng_prompts

728

729 async def render_chat_async(

730 self,

@@ -507,6 +734,8 @@ class BaseRenderer(ABC, Generic[_T]):

734 *,

735 prompt_extras: dict[str, Any] | None = None,

736 ):

737 + arrival_time = time.time()

738 +

739 if tok_params is None:

740 tok_params = self.default_chat_tok_params

741

@@ -525,5 +754,8 @@ class BaseRenderer(ABC, Generic[_T]):

754

755 self._apply_prompt_extras(tok_prompts, prompt_extras)

756

528 - # TODO: Apply multi-modal processor

529 - return out_conversations, tok_prompts

757 + eng_prompts = [

758 + self.process_for_engine(prompt, arrival_time) for prompt in tok_prompts

759 + ]

760 +

761 + return out_conversations, eng_prompts

vllm/renderers/inputs/preprocess.py

+13 -10

@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, NamedTuple, TypeAlias, TypedDict, overload

10 from vllm.inputs import (

11 EmbedsPrompt,

12 ExplicitEncoderDecoderPrompt,

13 + ProcessorInputs,

14 PromptType,

15 SingletonPrompt,

16 TextPrompt,

@@ -115,7 +116,7 @@ that has been standardized into a dictionary.

116 """

117

118

118 -def parse_dec_only_prompt(prompt: object) -> DecoderOnlyDictPrompt:

119 +def parse_dec_only_prompt(prompt: PromptType | object) -> DecoderOnlyDictPrompt:

120 """

121 Parse a prompt for a decoder-only model and normalize it to a dictionary.

122 """

@@ -144,7 +145,7 @@ def parse_dec_only_prompt(prompt: object) -> DecoderOnlyDictPrompt:

145 raise TypeError("Prompt should be a string, list of tokens, or dictionary")

146

147

147 -def _parse_enc_prompt(prompt: object) -> EncoderDictPrompt:

148 +def _parse_enc_prompt(prompt: PromptType | object) -> EncoderDictPrompt:

149 if isinstance(prompt, str):

150 return TextPrompt(prompt=prompt)

151

@@ -166,7 +167,7 @@ def _parse_enc_prompt(prompt: object) -> EncoderDictPrompt:

167 raise TypeError("Prompt should be a string, list of tokens, or dictionary")

168

169

169 -def _parse_dec_prompt(prompt: object) -> DecoderDictPrompt:

170 +def _parse_dec_prompt(prompt: PromptType | object) -> DecoderDictPrompt:

171 if isinstance(prompt, str):

172 return TextPrompt(prompt=prompt)

173

@@ -195,13 +196,13 @@ def _parse_dec_prompt(prompt: object) -> DecoderDictPrompt:

196 raise TypeError("Prompt should be a string, list of tokens, or dictionary")

197

198

198 -def parse_enc_dec_prompt(prompt: object) -> EncoderDecoderDictPrompt:

199 +def parse_enc_dec_prompt(prompt: PromptType | object) -> EncoderDecoderDictPrompt:

200 """

201 Parse a prompt for an encoder-decoder model and normalize it to a dictionary.

202 """

203 if isinstance(prompt, dict) and "encoder_prompt" in prompt:

203 - enc_prompt: object = prompt["encoder_prompt"] # type: ignore[typeddict-item]

204 - dec_prompt: object | None = prompt["decoder_prompt"] # type: ignore[typeddict-item]

204 + enc_prompt = prompt["encoder_prompt"] # type: ignore[typeddict-item]

205 + dec_prompt = prompt["decoder_prompt"] # type: ignore[typeddict-item]

206 else:

207 enc_prompt = prompt

208 dec_prompt = None

@@ -235,21 +236,23 @@ def extract_target_prompt(model_config: "ModelConfig", prompt: object):

236

237 def extract_prompt_components(

238 model_config: "ModelConfig",

238 - prompt: object,

239 + prompt: PromptType | ProcessorInputs,

240 ) -> PromptComponents:

241 target_prompt = extract_target_prompt(model_config, prompt)

242

243 return PromptComponents(

244 text=target_prompt.get("prompt"),

244 - token_ids=target_prompt.get("prompt_token_ids"), # type: ignore[arg-type]

245 + token_ids=target_prompt.get("prompt_token_ids"),

246 embeds=target_prompt.get("prompt_embeds"),

247 )

248

249

249 -def extract_prompt_len(model_config: "ModelConfig", prompt: object):

250 +def extract_prompt_len(

251 + model_config: "ModelConfig", prompt: PromptType | ProcessorInputs

252 +):

253 target_prompt = extract_target_prompt(model_config, prompt)

254

255 return length_from_prompt_token_ids_or_embeds(

253 - target_prompt.get("prompt_token_ids"), # type: ignore[arg-type]

256 + target_prompt.get("prompt_token_ids"),

257 target_prompt.get("prompt_embeds"),

258 )

vllm/utils/tqdm_utils.py new

+39

@@ -0,0 +1,39 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from collections.abc import Callable, Iterable, Sequence

4 +from typing import Any, TypeVar, overload

5 +

6 +from tqdm.auto import tqdm

7 +

8 +_T = TypeVar("_T", bound=Iterable)

9 +

10 +

11 +@overload

12 +def maybe_tqdm(

13 + it: Sequence[_T],

14 + *,

15 + use_tqdm: bool | Callable[..., tqdm],

16 + **tqdm_kwargs: Any,

17 +) -> Sequence[_T]: ...

18 +

19 +

20 +@overload

21 +def maybe_tqdm(

22 + it: Iterable[_T],

23 + *,

24 + use_tqdm: bool | Callable[..., tqdm],

25 + **tqdm_kwargs: Any,

26 +) -> Iterable[_T]: ...

27 +

28 +

29 +def maybe_tqdm(

30 + it: Iterable[_T],

31 + *,

32 + use_tqdm: bool | Callable[..., tqdm],

33 + **tqdm_kwargs: Any,

34 +) -> Iterable[_T]:

35 + if not use_tqdm:

36 + return it

37 +

38 + tqdm_func = use_tqdm if callable(use_tqdm) else tqdm

39 + return tqdm_func(it, **tqdm_kwargs)

vllm/v1/engine/async_llm.py

+15 -11

@@ -20,7 +20,7 @@ from vllm.distributed.weight_transfer.base import (

20 )

21 from vllm.engine.arg_utils import AsyncEngineArgs

22 from vllm.engine.protocol import EngineClient, StreamingInput

23 -from vllm.inputs import PromptType

23 +from vllm.inputs import ProcessorInputs, PromptType

24 from vllm.logger import init_logger

25 from vllm.lora.request import LoRARequest

26 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

@@ -28,7 +28,6 @@ from vllm.outputs import STREAM_FINISHED, PoolingRequestOutput, RequestOutput

28 from vllm.plugins.io_processors import get_io_processor

29 from vllm.pooling_params import PoolingParams

30 from vllm.renderers import merge_kwargs, renderer_from_config

31 -from vllm.renderers.inputs import DictPrompt, TokPrompt

31 from vllm.renderers.inputs.preprocess import extract_prompt_components

32 from vllm.sampling_params import RequestOutputKind, SamplingParams

33 from vllm.tasks import SupportedTask

@@ -290,8 +289,7 @@ class AsyncLLM(EngineClient):

289 request_id: str,

290 prompt: EngineCoreRequest

291 | PromptType

293 - | DictPrompt

294 - | TokPrompt

292 + | ProcessorInputs

293 | AsyncGenerator[StreamingInput, None],

294 params: SamplingParams | PoolingParams,

295 arrival_time: float | None = None,

@@ -301,6 +299,7 @@ class AsyncLLM(EngineClient):

299 priority: int = 0,

300 data_parallel_rank: int | None = None,

301 prompt_text: str | None = None,

302 + reasoning_ended: bool | None = None,

303 ) -> RequestOutputCollector:

304 """Add new request to the AsyncLLM."""

305

@@ -336,6 +335,9 @@ class AsyncLLM(EngineClient):

335 )

336

337 if isinstance(prompt, AsyncGenerator):

338 + if reasoning_ended is not None:

339 + raise NotImplementedError

340 +

341 # Streaming input case.

342 return await self._add_streaming_input_request(

343 request_id,

@@ -359,10 +361,6 @@ class AsyncLLM(EngineClient):

361 "latter will be used, and the former will be ignored."

362 )

363 else:

362 - if prompt_text is not None:

363 - raise ValueError(

364 - "should only provide prompt_text with EngineCoreRequest"

365 - )

364 request = self.input_processor.process_inputs(

365 request_id,

366 prompt,

@@ -377,6 +375,9 @@ class AsyncLLM(EngineClient):

375 )

376 prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)

377

378 + if reasoning_ended is not None:

379 + request.reasoning_ended = reasoning_ended

380 +

381 self.input_processor.assign_request_id(request)

382

383 # We start the output_handler on the first call to add_request() so

@@ -536,8 +537,7 @@ class AsyncLLM(EngineClient):

537 self,

538 prompt: EngineCoreRequest

539 | PromptType

539 - | DictPrompt

540 - | TokPrompt

540 + | ProcessorInputs

541 | AsyncGenerator[StreamingInput, None],

542 sampling_params: SamplingParams,

543 request_id: str,

@@ -548,6 +548,7 @@ class AsyncLLM(EngineClient):

548 trace_headers: Mapping[str, str] | None = None,

549 priority: int = 0,

550 data_parallel_rank: int | None = None,

551 + reasoning_ended: bool | None = None,

552 ) -> AsyncGenerator[RequestOutput, None]:

553 """

554 Main function called by the API server to kick off a request

@@ -576,6 +577,7 @@ class AsyncLLM(EngineClient):

577 priority=priority,

578 data_parallel_rank=data_parallel_rank,

579 prompt_text=prompt_text,

580 + reasoning_ended=reasoning_ended,

581 )

582

583 # The output_handler task pushes items into the queue.

@@ -770,13 +772,14 @@ class AsyncLLM(EngineClient):

772

773 async def encode(

774 self,

773 - prompt: PromptType | DictPrompt | TokPrompt,

775 + prompt: PromptType | ProcessorInputs,

776 pooling_params: PoolingParams,

777 request_id: str,

778 lora_request: LoRARequest | None = None,

779 trace_headers: Mapping[str, str] | None = None,

780 priority: int = 0,

781 tokenization_kwargs: dict[str, Any] | None = None,

782 + reasoning_ended: bool | None = None,

783 ) -> AsyncGenerator[PoolingRequestOutput, None]:

784 """

785 Main function called by the API server to kick off a request

@@ -802,6 +805,7 @@ class AsyncLLM(EngineClient):

805 tokenization_kwargs=tokenization_kwargs,

806 trace_headers=trace_headers,

807 priority=priority,

808 + reasoning_ended=reasoning_ended,

809 )

810

811 # The output_handler task pushes items into the queue.

vllm/v1/engine/input_processor.py

+10 -158

@@ -3,7 +3,7 @@

4 import time

5 from collections.abc import Mapping

6 -from typing import Any, Literal, cast

6 +from typing import Any, Literal

8 import vllm.envs as envs

9 from vllm.config import VllmConfig

@@ -11,7 +11,6 @@ from vllm.inputs.data import (

11 ProcessorInputs,

12 PromptType,

13 SingletonInputs,

14 - SingletonPrompt,

14 )

15 from vllm.inputs.parse import split_enc_dec_inputs

16 from vllm.inputs.preprocess import InputPreprocessor

@@ -20,22 +19,16 @@ from vllm.lora.request import LoRARequest

19 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

20 from vllm.multimodal.encoder_budget import MultiModalBudget

21 from vllm.multimodal.inputs import (

23 - MultiModalDataDict,

22 MultiModalFeatureSpec,

25 - MultiModalUUIDDict,

23 )

27 -from vllm.multimodal.parse import ModalityDataItems, MultiModalDataItems

28 -from vllm.multimodal.processing.context import set_request_id

24 from vllm.multimodal.utils import argsort_mm_positions

25 from vllm.pooling_params import PoolingParams

26 from vllm.renderers import BaseRenderer, renderer_from_config

32 -from vllm.renderers.inputs import DictPrompt, TokPrompt

27 from vllm.sampling_params import SamplingParams

28 from vllm.tasks import POOLING_TASKS, SupportedTask

29 from vllm.tokenizers import TokenizerLike

30 from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid

31 from vllm.utils.jsontree import json_iter_leaves

38 -from vllm.utils.torch_utils import set_default_torch_num_threads

32 from vllm.v1.engine import EngineCoreRequest

34 logger = init_logger(__name__)

@@ -133,81 +126,6 @@ class InputProcessor:

126 f"but got {type(params).__name__}"

127 )

128

136 - def _parse_mm_items(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:

137 - mm_processor = self.renderer.get_mm_processor()

138 - return mm_processor.info.parse_mm_data(mm_data)

139 -

140 - def _validate_singleton_mm_uuids(self, prompt: SingletonPrompt) -> None:

141 - if not isinstance(prompt, dict):

142 - return

143 -

144 - mm_data = cast(MultiModalDataDict, prompt.get("multi_modal_data") or {})

145 - mm_uuids = cast(MultiModalUUIDDict, prompt.get("multi_modal_uuids") or {})

146 - if not mm_data and not mm_uuids:

147 - return

148 -

149 - mm_data_parsed = self._parse_mm_items(

150 - {k: v for k, v in mm_data.items() if v is not None}

151 - )

152 - mm_uuids_parsed = {

153 - k: [v] if isinstance(v, str) else v

154 - for k, v in mm_uuids.items()

155 - if v is not None

156 - }

157 -

158 - # NOTE: Include the keys corresponding to `None`

159 - modalities = mm_data.keys() | mm_uuids.keys()

160 -

161 - for modality in modalities:

162 - data_items = cast(

163 - ModalityDataItems | list[Any], mm_data_parsed.get(modality, [])

164 - )

165 - uuid_items = cast(list[str | None], mm_uuids_parsed.get(modality, []))

166 -

167 - if len(data_items) > 0:

168 - if len(uuid_items) > 0 and len(data_items) != len(uuid_items):

169 - raise ValueError(

170 - f"If given, multi_modal_uuids[{modality!r}] must have "

171 - f"same length as multi_modal_data[{modality!r}], but "

172 - f"got {len(uuid_items)} vs {len(data_items)}."

173 - )

174 -

175 - for i, item in enumerate(data_items):

176 - if item is None:

177 - if not uuid_items:

178 - raise ValueError(

179 - f"multi_modal_data[{modality!r}][{i}] is empty but "

180 - f"multi_modal_uuids[{modality!r}] is missing."

181 - )

182 -

183 - if uuid_items[i] is None:

184 - raise ValueError(

185 - f"multi_modal_data[{modality!r}][{i}] is empty but "

186 - f"multi_modal_uuids[{modality!r}][{i}] is missing."

187 - )

188 - else:

189 - if len(uuid_items) == 0:

190 - raise ValueError(

191 - f"multi_modal_data[{modality!r}] is empty but "

192 - f"multi_modal_uuids[{modality!r}] is missing."

193 - )

194 -

195 - def _validate_mm_uuids(self, prompt: PromptType | DictPrompt | TokPrompt) -> None:

196 - """

197 - Validate that user-provided multi_modal_uuids align with

198 - multi_modal_data in the incoming request prompt(s).

199 - Only checks lengths; `None` entries are allowed and will be

200 - auto-hashed downstream.

201 - """

202 -

203 - if isinstance(prompt, dict) and "encoder_prompt" in prompt:

204 - self._validate_singleton_mm_uuids(prompt["encoder_prompt"]) # type: ignore[typeddict-item]

205 -

206 - if (dec_prompt := prompt["decoder_prompt"]) is not None: # type: ignore[typeddict-item]

207 - self._validate_singleton_mm_uuids(dec_prompt)

208 - else:

209 - self._validate_singleton_mm_uuids(prompt)

210 -

129 def _validate_lora(self, lora_request: LoRARequest | None) -> None:

130 if lora_request is None:

131 return

@@ -227,47 +145,6 @@ class InputProcessor:

145 "[lora_path]` to use the LoRA tokenizer."

146 )

147

230 - def _extract_singleton_mm_data(

231 - self, prompt: SingletonPrompt

232 - ) -> MultiModalDataDict | None:

233 - if not isinstance(prompt, dict):

234 - return None

235 -

236 - return prompt.get("multi_modal_data")

237 -

238 - def _extract_mm_data(

239 - self, prompt: PromptType | DictPrompt | TokPrompt

240 - ) -> MultiModalDataDict | None:

241 - if isinstance(prompt, dict) and "encoder_prompt" in prompt:

242 - return self._extract_singleton_mm_data(prompt["encoder_prompt"]) # type: ignore[typeddict-item]

243 - else:

244 - return self._extract_singleton_mm_data(prompt)

245 -

246 - def _maybe_build_mm_uuids(

247 - self,

248 - request_id: str,

249 - prompt: PromptType | DictPrompt | TokPrompt,

250 - ) -> MultiModalUUIDDict | None:

251 - """Build per-item multimodal hash overrides when enabled. In this case,

252 - multimodal data items are identified by their request id, modality and

253 - index rather than their content.

254 -

255 - Returns a dictionary of modality -> list[str] of overrides, or None if

256 - disabled or no multimodal data is present.

257 - """

258 - mm_data = self._extract_mm_data(prompt)

259 - if not mm_data:

260 - return None

261 -

262 - mm_items = self._parse_mm_items(

263 - {k: v for k, v in mm_data.items() if v is not None}

264 - )

265 -

266 - return {

267 - modality: [f"{request_id}-{modality}-{i}" for i in range(data_count)]

268 - for modality, data_count in mm_items.get_all_counts().items()

269 - }

270 -

148 def _get_mm_identifier(

149 self,

150 mm_hash: str,

@@ -309,7 +186,7 @@ class InputProcessor:

186 def process_inputs(

187 self,

188 request_id: str,

312 - prompt: PromptType | DictPrompt | TokPrompt,

189 + prompt: PromptType | ProcessorInputs,

190 params: SamplingParams | PoolingParams,

191 arrival_time: float | None = None,

192 lora_request: LoRARequest | None = None,

@@ -333,43 +210,18 @@ class InputProcessor:

210 f"is out of range [0, {num_ranks})."

211 )

212

336 - if arrival_time is None:

337 - arrival_time = time.time()

213 + if isinstance(prompt, dict) and "type" in prompt:

214 + if arrival_time is None:

215 + arrival_time = prompt.get("arrival_time", time.time()) # type: ignore[assignment]

216

339 - # Optionally generate multimodal hash overrides to avoid hashing

340 - # multimodal data items by their content as their identifiers.

341 -

342 - # NOTE: when users explicitly turn off BOTH prefix caching and input

343 - # processing caching, no multimodal features or embeddings will be

344 - # reused across requests, therefore identifying multimodal data items

345 - # by their content is no longer necessary, and we create uuids with

346 - # request id-modality-index as multimodal hash overrides.

347 - if (

348 - self.model_config.multimodal_config

349 - and self.model_config.multimodal_config.mm_processor_cache_gb == 0

350 - and not self.cache_config.enable_prefix_caching

351 - ):

352 - mm_uuids = self._maybe_build_mm_uuids(request_id, prompt)

217 + processed_inputs: ProcessorInputs = prompt # type: ignore[assignment]

218 else:

354 - # Otherwise, use user-provided uuids as multimodal hash overrides

355 - # if provided.

356 - self._validate_mm_uuids(prompt)

357 - if isinstance(prompt, dict):

358 - mm_uuids = cast(

359 - MultiModalUUIDDict | None, prompt.get("multi_modal_uuids")

360 - )

361 - else:

362 - mm_uuids = None

363 -

364 - # Process inputs, which includes:

365 - # 1. Tokenize text prompt, with LoRA request if one exists.

366 - # 2. For multimodal models with a merged preprocessor, preprocess

367 - # multimodal data and expand prompt token ids accordingly.

368 - with set_request_id(request_id), set_default_torch_num_threads():

369 - processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(

219 + if arrival_time is None:

220 + arrival_time = time.time()

221 +

222 + processed_inputs = self.input_preprocessor.preprocess(

223 prompt,

224 tokenization_kwargs=tokenization_kwargs,

372 - mm_uuids=mm_uuids,

225 )

226

227 from vllm.platforms import current_platform

vllm/v1/engine/llm_engine.py

+8 -6

@@ -14,7 +14,7 @@ from vllm.config import ParallelConfig, VllmConfig

14 from vllm.distributed import stateless_destroy_torch_distributed_process_group

15 from vllm.distributed.parallel_state import get_dp_group

16 from vllm.engine.arg_utils import EngineArgs

17 -from vllm.inputs import PromptType

17 +from vllm.inputs import ProcessorInputs, PromptType

18 from vllm.logger import init_logger

19 from vllm.lora.request import LoRARequest

20 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

@@ -22,7 +22,6 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput

22 from vllm.plugins.io_processors import get_io_processor

23 from vllm.pooling_params import PoolingParams

24 from vllm.renderers import renderer_from_config

25 -from vllm.renderers.inputs import DictPrompt, TokPrompt

25 from vllm.renderers.inputs.preprocess import extract_prompt_components

26 from vllm.sampling_params import SamplingParams

27 from vllm.tasks import SupportedTask

@@ -220,7 +219,7 @@ class LLMEngine:

219 def add_request(

220 self,

221 request_id: str,

223 - prompt: EngineCoreRequest | PromptType | DictPrompt | TokPrompt,

222 + prompt: EngineCoreRequest | PromptType | ProcessorInputs,

223 params: SamplingParams | PoolingParams,

224 arrival_time: float | None = None,

225 lora_request: LoRARequest | None = None,

@@ -228,7 +227,7 @@ class LLMEngine:

227 trace_headers: Mapping[str, str] | None = None,

228 priority: int = 0,

229 prompt_text: str | None = None,

231 - ) -> None:

230 + ) -> str:

231 # Validate the request_id type.

232 if not isinstance(request_id, str):

233 raise TypeError(f"request_id must be a string, got {type(request_id)}")

@@ -243,7 +242,6 @@ class LLMEngine:

242 "latter will be used, and the former will be ignored."

243 )

244 else:

246 - assert prompt_text is None

245 request = self.input_processor.process_inputs(

246 request_id,

247 prompt,

@@ -259,6 +257,8 @@ class LLMEngine:

257

258 self.input_processor.assign_request_id(request)

259

260 + req_id = request.request_id

261 +

262 # Use cloned params that may have been updated in process_inputs()

263 params = request.params

264

@@ -269,7 +269,7 @@ class LLMEngine:

269 self.output_processor.add_request(request, prompt_text, None, 0)

270 # Add the request to EngineCore.

271 self.engine_core.add_request(request)

272 - return

272 + return req_id

273

274 # Fan out child requests (for n>1).

275 parent_req = ParentRequest(request)

@@ -286,6 +286,8 @@ class LLMEngine:

286 # Add the request to EngineCore.

287 self.engine_core.add_request(child_request)

288

289 + return req_id

290 +

291 def step(self) -> list[RequestOutput | PoolingRequestOutput]:

292 if self.should_execute_dummy_batch:

293 self.should_execute_dummy_batch = False