@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 355f66348c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 17:15 UTC

[V1] Remove legacy input registry (#15673)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 28, 2025 at 14:34 UTC 355f66348c3ddb0a2c3217372f2ee47fb961d58f

17 files changed +132 -153

tests/models/multimodal/processing/test_h2ovl.py

+1 -6

@@ -10,7 +10,6 @@ from transformers import PretrainedConfig

10 from vllm.multimodal import MULTIMODAL_REGISTRY

11 from vllm.multimodal.image import rescale_image_size

12 from vllm.multimodal.processing import BaseMultiModalProcessor

13 -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

14 from ....conftest import _ImageAssets

15 from ...utils import build_model_context

@@ -156,11 +155,7 @@ def test_processor_override(

155 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

156 limit_mm_per_prompt={"image": len(size_factors)},

157 )

159 - tokenizer = cached_tokenizer_from_config(ctx.model_config)

160 - processor = MULTIMODAL_REGISTRY.create_processor(

161 - ctx.model_config,

162 - tokenizer=tokenizer,

163 - )

158 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

159 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

160

161 min_num = min_dynamic_patch if dynamic_image_size else 1

tests/models/multimodal/processing/test_idefics3.py

+1 -6

@@ -4,7 +4,6 @@ import pytest

4 from transformers import Idefics3Config

6 from vllm.multimodal import MULTIMODAL_REGISTRY

7 -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

8 from ....conftest import _ImageAssets

9 from ...utils import build_model_context

@@ -38,11 +37,7 @@ def test_processor_override(

37 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

38 limit_mm_per_prompt={"image": num_imgs},

39 )

41 - tokenizer = cached_tokenizer_from_config(ctx.model_config)

42 - processor = MULTIMODAL_REGISTRY.create_processor(

43 - ctx.model_config,

44 - tokenizer=tokenizer,

45 - )

40 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

41 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

43 # Build the image str / prompt based on the number of images we pass

tests/models/multimodal/processing/test_internvl.py

+1 -6

@@ -10,7 +10,6 @@ from transformers import PretrainedConfig

10 from vllm.multimodal import MULTIMODAL_REGISTRY

11 from vllm.multimodal.image import rescale_image_size

12 from vllm.multimodal.processing import BaseMultiModalProcessor

13 -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

14 from ....conftest import _ImageAssets

15 from ...utils import build_model_context

@@ -113,11 +112,7 @@ def test_processor_override(

112 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

113 limit_mm_per_prompt={"image": len(size_factors)},

114 )

116 - tokenizer = cached_tokenizer_from_config(ctx.model_config)

117 - processor = MULTIMODAL_REGISTRY.create_processor(

118 - ctx.model_config,

119 - tokenizer=tokenizer,

120 - )

115 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

116 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

117

118 min_num = min_dynamic_patch if dynamic_image_size else 1

tests/models/multimodal/processing/test_llava_next.py

+3 -13

@@ -10,7 +10,6 @@ from pqdm.threads import pqdm

10 from vllm.multimodal import MULTIMODAL_REGISTRY

11 from vllm.multimodal.parse import ImageSize

12 from vllm.multimodal.processing import BaseMultiModalProcessor

13 -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

14 from ...utils import build_model_context

@@ -40,10 +39,7 @@ def test_processor_max_tokens(model_id):

39 mm_processor_kwargs=None,

40 limit_mm_per_prompt={"image": 1},

41 )

43 - processor = MULTIMODAL_REGISTRY.create_processor(

44 - ctx.model_config,

45 - tokenizer=cached_tokenizer_from_config(ctx.model_config),

46 - )

42 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

43 info = processor.info

45 seen_aspect_ratios = set[float]()

@@ -139,10 +135,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):

135 mm_processor_kwargs=None,

136 limit_mm_per_prompt={"image": num_imgs},

137 )

142 - processor = MULTIMODAL_REGISTRY.create_processor(

143 - ctx.model_config,

144 - tokenizer=cached_tokenizer_from_config(ctx.model_config),

145 - )

138 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

139

140 image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),

141 (488, 183), (2560, 1669)]

@@ -168,10 +161,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):

161 mm_processor_kwargs=None,

162 limit_mm_per_prompt={"image": num_imgs},

163 )

171 - processor = MULTIMODAL_REGISTRY.create_processor(

172 - ctx.model_config,

173 - tokenizer=cached_tokenizer_from_config(ctx.model_config),

174 - )

164 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

165

166 seen_aspect_ratios = set[float]()

167 image_sizes = list[ImageSize]()

tests/models/multimodal/processing/test_llava_onevision.py

+3 -13

@@ -10,7 +10,6 @@ from pqdm.threads import pqdm

10 from vllm.multimodal import MULTIMODAL_REGISTRY

11 from vllm.multimodal.parse import ImageSize

12 from vllm.multimodal.processing import BaseMultiModalProcessor

13 -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

14 from ...utils import build_model_context

@@ -41,10 +40,7 @@ def test_processor_max_tokens(model_id):

40 mm_processor_kwargs=None,

41 limit_mm_per_prompt={"image": 1},

42 )

44 - processor = MULTIMODAL_REGISTRY.create_processor(

45 - ctx.model_config,

46 - tokenizer=cached_tokenizer_from_config(ctx.model_config),

47 - )

43 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

44 info = processor.info

46 seen_aspect_ratios = set[float]()

@@ -139,10 +135,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):

135 mm_processor_kwargs=None,

136 limit_mm_per_prompt={"image": num_imgs},

137 )

142 - processor = MULTIMODAL_REGISTRY.create_processor(

143 - ctx.model_config,

144 - tokenizer=cached_tokenizer_from_config(ctx.model_config),

145 - )

138 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

139

140 image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),

141 (488, 183), (2560, 1669)]

@@ -169,10 +162,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):

162 mm_processor_kwargs=None,

163 limit_mm_per_prompt={"image": num_imgs},

164 )

172 - processor = MULTIMODAL_REGISTRY.create_processor(

173 - ctx.model_config,

174 - tokenizer=cached_tokenizer_from_config(ctx.model_config),

175 - )

165 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

166

167 seen_aspect_ratios = set[float]()

168 image_sizes = list[ImageSize]()

tests/models/multimodal/processing/test_phi3v.py

+1 -6

@@ -3,7 +3,6 @@

3 import pytest

5 from vllm.multimodal import MULTIMODAL_REGISTRY

6 -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

7 from ....conftest import _ImageAssets

8 from ...utils import build_model_context

@@ -39,11 +38,7 @@ def test_processor_override(

38 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

39 limit_mm_per_prompt={"image": num_imgs},

40 )

42 - tokenizer = cached_tokenizer_from_config(ctx.model_config)

43 - processor = MULTIMODAL_REGISTRY.create_processor(

44 - ctx.model_config,

45 - tokenizer=tokenizer,

46 - )

41 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

42 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

44 # Build the image str / prompt based on the number of images we pass

tests/models/multimodal/processing/test_qwen2_vl.py

+2 -6

@@ -3,7 +3,6 @@

3 import pytest

5 from vllm.multimodal import MULTIMODAL_REGISTRY

6 -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

7 from ....conftest import _ImageAssets

8 from ...utils import build_model_context

@@ -34,11 +33,8 @@ def test_processor_override(

33 mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

34 limit_mm_per_prompt={"image": num_imgs},

35 )

37 - tokenizer = cached_tokenizer_from_config(ctx.model_config)

38 - processor = MULTIMODAL_REGISTRY.create_processor(

39 - ctx.model_config,

40 - tokenizer=tokenizer,

41 - )

36 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

37 + tokenizer = processor.info.get_tokenizer()

38 hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

40 # Build the image str / prompt based on the number of images we pass

tests/multimodal/test_processing.py

+4 -14

@@ -28,8 +28,7 @@ from vllm.multimodal.processing import (PlaceholderFeaturesInfo,

28 replace_token_matches)

29 # yapf: enable

30 from vllm.multimodal.profiling import MultiModalProfiler

31 -from vllm.transformers_utils.tokenizer import (AnyTokenizer,

32 - cached_tokenizer_from_config)

31 +from vllm.transformers_utils.tokenizer import AnyTokenizer

32 from vllm.utils import full_groupby

34 from .utils import random_image

@@ -955,10 +954,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):

954 limit_mm_per_prompt=limit_mm_per_prompt,

955 )

956

958 - processor = MULTIMODAL_REGISTRY.create_processor(

959 - model_config,

960 - tokenizer=cached_tokenizer_from_config(model_config),

961 - )

957 + processor = MULTIMODAL_REGISTRY.create_processor(model_config)

958 profiler = MultiModalProfiler(processor)

959

960 mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})

@@ -994,10 +990,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):

990 limit_mm_per_prompt=limit_mm_per_prompt,

991 )

992

997 - processor = MULTIMODAL_REGISTRY.create_processor(

998 - model_config,

999 - tokenizer=cached_tokenizer_from_config(model_config),

1000 - )

993 + processor = MULTIMODAL_REGISTRY.create_processor(model_config)

994

995 rng = np.random.RandomState(0)

996 image = random_image(rng, min_wh=128, max_wh=256)

@@ -1066,10 +1059,7 @@ def test_hf_processor_kwargs(model_id, call_kwargs, expected_kwargs):

1059 revision=None,

1060 )

1061

1069 - processor = MULTIMODAL_REGISTRY.create_processor(

1070 - model_config,

1071 - tokenizer=cached_tokenizer_from_config(model_config),

1072 - )

1062 + processor = MULTIMODAL_REGISTRY.create_processor(model_config)

1063 orig_get_hf_processor = processor.info.get_hf_processor

1064

1065 def get_hf_processor(self, **kwargs):

vllm/inputs/preprocess.py

+6 -6

@@ -261,13 +261,13 @@ class InputPreprocessor:

261 # initialized without a tokenizer while using also multi-modal

262 # input.

263 if not self.tokenizer:

264 - tokenizer = None

264 + tokenizer = object() # Dummy

265 else:

266 tokenizer_group = self.get_tokenizer_group()

267 tokenizer = tokenizer_group.get_lora_tokenizer(lora_request)

268

269 - mm_processor = self.mm_registry.create_processor(

270 - self.model_config, tokenizer)

269 + mm_processor = self.mm_registry.create_processor(self.model_config,

270 + tokenizer=tokenizer)

271

272 if mm_processor_kwargs is None:

273 mm_processor_kwargs = {}

@@ -288,14 +288,14 @@ class InputPreprocessor:

288 # initialized without a tokenizer while using also multi-modal

289 # input.

290 if not self.tokenizer:

291 - tokenizer = None

291 + tokenizer = object() # Dummy

292 else:

293 tokenizer_group = self.get_tokenizer_group()

294 tokenizer = await tokenizer_group.get_lora_tokenizer_async(

295 lora_request)

296

297 - mm_processor = self.mm_registry.create_processor(

298 - self.model_config, tokenizer)

297 + mm_processor = self.mm_registry.create_processor(self.model_config,

298 + tokenizer=tokenizer)

299 if mm_processor_kwargs is None:

300 mm_processor_kwargs = {}

301

vllm/inputs/registry.py

+17 -8

@@ -13,8 +13,7 @@ from typing_extensions import TypeVar, assert_never

14 from vllm.logger import init_logger

15 from vllm.transformers_utils.processor import cached_processor_from_config

16 -from vllm.transformers_utils.tokenizer import (AnyTokenizer,

17 - cached_tokenizer_from_config)

16 +from vllm.transformers_utils.tokenizer import AnyTokenizer

17 from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,

18 resolve_mm_processor_kwargs)

@@ -329,17 +328,27 @@ class InputRegistry:

328 from vllm.model_executor.model_loader import get_model_architecture

329 from vllm.multimodal import MultiModalKwargs

330 from vllm.multimodal.profiling import MultiModalProfiler

331 + from vllm.sequence import SequenceData

332

333 if mm_registry.has_processor(model_config):

334 - tokenizer = cached_tokenizer_from_config(model_config)

334 processor = mm_registry.create_processor(model_config,

336 - tokenizer,

335 disable_cache=True)

336 profiler = MultiModalProfiler(processor)

339 - dummy_data_factory = (profiler.get_encoder_dummy_data

340 - if is_encoder_data else

341 - profiler.get_decoder_dummy_data)

342 - dummy_data = dummy_data_factory(seq_len)

337 +

338 + dummy_data_v1 = (profiler.get_encoder_dummy_data(seq_len)

339 + if is_encoder_data else

340 + profiler.get_decoder_dummy_data(seq_len))

341 + _seq_data = SequenceData.from_seqs(

342 + dummy_data_v1.prompt_token_ids) # type: ignore[attr-defined]

343 +

344 + dummy_data = DummyData(

345 + seq_data=_seq_data,

346 + multi_modal_data=getattr(dummy_data_v1, "multi_modal_data",

347 + None),

348 + multi_modal_placeholders=getattr(dummy_data_v1,

349 + "multi_modal_placeholders",

350 + None),

351 + )

352 else:

353 model_cls, _ = get_model_architecture(model_config)

354 if is_encoder_data:

vllm/multimodal/profiling.py

+24 -31

@@ -3,18 +3,18 @@

3 from abc import ABC, abstractmethod

4 from collections.abc import Mapping

5 from dataclasses import dataclass, field

6 -from typing import Generic, TypeVar, cast

6 +from typing import Generic, NamedTuple, TypeVar, cast

8 import numpy as np

9 import numpy.typing as npt

10 from PIL import Image

12 import vllm.envs as envs

13 -from vllm.inputs import DummyData

13 from vllm.logger import init_logger

15 from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,

17 - MultiModalInputs)

16 + MultiModalInputs, MultiModalKwargs,

17 + MultiModalPlaceholderDict)

18 from .processing import BaseMultiModalProcessor, BaseProcessingInfo

20 logger = init_logger(__name__)

@@ -31,6 +31,20 @@ class ProcessorInputs:

31 hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)

34 +class DummyEncoderData(NamedTuple):

35 + """Dummy data used for profiling."""

36 +

37 + prompt_token_ids: list[int]

38 +

39 +

40 +class DummyDecoderData(NamedTuple):

41 + """Dummy data used for profiling."""

42 +

43 + prompt_token_ids: list[int]

44 + multi_modal_data: MultiModalKwargs

45 + multi_modal_placeholders: MultiModalPlaceholderDict

46 +

47 +

48 _I = TypeVar("_I", bound=BaseProcessingInfo)

@@ -179,13 +193,7 @@ class MultiModalProfiler(Generic[_I]):

193 "tokens.")

194 return mm_inputs, total_placeholders_by_modality

195

182 - def get_encoder_dummy_data(

183 - self,

184 - seq_len: int,

185 - ) -> DummyData:

186 - # Avoid circular import

187 - from vllm.sequence import SequenceData

188 -

196 + def get_encoder_dummy_data(self, seq_len: int) -> DummyEncoderData:

197 mm_inputs, _ = self.get_and_validate_mm_inputs(seq_len)

198 mm_inputs = cast(MultiModalEncDecInputs, mm_inputs)

199

@@ -197,19 +205,9 @@ class MultiModalProfiler(Generic[_I]):

205 num_tokens_to_pad = max(total_len, seq_len) - total_len

206 encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)

207

200 - return DummyData(

201 - seq_data=SequenceData.from_seqs(encoder_prompt_token_ids),

202 - multi_modal_data=None,

203 - multi_modal_placeholders=None,

204 - )

205 -

206 - def get_decoder_dummy_data(

207 - self,

208 - seq_len: int,

209 - ) -> DummyData:

210 - # Avoid circular import

211 - from vllm.sequence import SequenceData

208 + return DummyEncoderData(encoder_prompt_token_ids)

209

210 + def get_decoder_dummy_data(self, seq_len: int) -> DummyDecoderData:

211 (mm_inputs, total_placeholders_by_modality

212 ) = self.get_and_validate_mm_inputs(seq_len)

213

@@ -231,16 +229,11 @@ class MultiModalProfiler(Generic[_I]):

229 "and/or reduce `mm_counts`.", seq_len, total_len,

230 total_placeholders_by_modality)

231

234 - return DummyData(

235 - seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),

236 - multi_modal_data=None,

237 - multi_modal_placeholders=None,

238 - )

239 -

240 - prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))

232 + if total_len < seq_len:

233 + prompt_token_ids.extend([0] * (seq_len - total_len))

234

242 - return DummyData(

243 - seq_data=SequenceData.from_seqs(prompt_token_ids),

235 + return DummyDecoderData(

236 + prompt_token_ids=prompt_token_ids,

237 multi_modal_data=mm_inputs["mm_kwargs"],

238 multi_modal_placeholders=mm_inputs["mm_placeholders"],

239 )

vllm/multimodal/registry.py

+53 -10

@@ -21,7 +21,8 @@ from .image import ImagePlugin

21 from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors

22 from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,

23 ProcessingCache)

24 -from .profiling import BaseDummyInputsBuilder, MultiModalProfiler

24 +from .profiling import (BaseDummyInputsBuilder, DummyDecoderData,

25 + DummyEncoderData, MultiModalProfiler)

26 from .video import VideoPlugin

28 if TYPE_CHECKING:

@@ -256,10 +257,7 @@ class MultiModalRegistry:

257 on underlying model configuration.

258 """

259 if self.has_processor(model_config):

259 - tokenizer = cached_tokenizer_from_config(model_config)

260 - processor = self.create_processor(model_config,

261 - tokenizer,

262 - disable_cache=True)

260 + processor = self.create_processor(model_config, disable_cache=True)

261 seq_len = model_config.max_model_len

262 mm_limits = self.get_mm_limits_per_prompt(model_config)

263 return processor.info.get_mm_max_tokens_per_item(

@@ -373,10 +371,7 @@ class MultiModalRegistry:

371 This should be called after :meth:`init_mm_limits_per_prompt`.

372 """

373 if self.has_processor(model_config):

376 - tokenizer = cached_tokenizer_from_config(model_config)

377 - processor = self.create_processor(model_config,

378 - tokenizer,

379 - disable_cache=True)

374 + processor = self.create_processor(model_config, disable_cache=True)

375 profiler = MultiModalProfiler(processor)

376 return profiler.get_mm_limits()

377

@@ -436,8 +431,8 @@ class MultiModalRegistry:

431 def create_processor(

432 self,

433 model_config: "ModelConfig",

439 - tokenizer: AnyTokenizer,

434 *,

435 + tokenizer: Optional[AnyTokenizer] = None,

436 disable_cache: Optional[bool] = None,

437 ) -> BaseMultiModalProcessor[BaseProcessingInfo]:

438 """

@@ -446,6 +441,8 @@ class MultiModalRegistry:

441 See also:

442 :ref:`mm-processing`

443 """

444 + if tokenizer is None:

445 + tokenizer = cached_tokenizer_from_config(model_config)

446 if disable_cache is None:

447 disable_cache = model_config.disable_mm_preprocessor_cache

448

@@ -456,3 +453,49 @@ class MultiModalRegistry:

453 cache = None if disable_cache else self._processing_cache

454

455 return factories.build_processor(ctx, cache=cache)

456 +

457 + def get_decoder_dummy_data(

458 + self,

459 + model_config: "ModelConfig",

460 + seq_len: int,

461 + ) -> DummyDecoderData:

462 + """

463 + Create dummy data for profiling the memory usage of a model.

464 +

465 + The model is identified by ``model_config``.

466 + """

467 + processor = self.create_processor(model_config, disable_cache=True)

468 + profiler = MultiModalProfiler(processor)

469 + dummy_data = profiler.get_decoder_dummy_data(seq_len)

470 +

471 + # Having more tokens is over-conservative but otherwise fine

472 + token_ids = dummy_data.prompt_token_ids

473 + if len(token_ids) < seq_len:

474 + raise AssertionError(

475 + f"Expected at least {seq_len} dummy tokens for profiling, "

476 + f"but found {len(token_ids)} tokens instead.")

477 +

478 + return dummy_data

479 +

480 + def get_encoder_dummy_data(

481 + self,

482 + model_config: "ModelConfig",

483 + seq_len: int,

484 + ) -> DummyEncoderData:

485 + """

486 + Create dummy data for profiling the memory usage of a model.

487 +

488 + The model is identified by ``model_config``.

489 + """

490 + processor = self.create_processor(model_config, disable_cache=True)

491 + profiler = MultiModalProfiler(processor)

492 + dummy_data = profiler.get_encoder_dummy_data(seq_len)

493 +

494 + # Having more tokens is over-conservative but otherwise fine

495 + token_ids = dummy_data.prompt_token_ids

496 + if len(token_ids) < seq_len:

497 + logger.warning_once(

498 + f"Expected at least {seq_len} dummy encoder tokens for "

499 + f"profiling, but found {len(token_ids)} tokens instead.")

500 +

501 + return dummy_data

vllm/v1/engine/async_llm.py

+4 -3

@@ -14,10 +14,11 @@ from vllm.config import ModelConfig, VllmConfig

14 from vllm.engine.arg_utils import AsyncEngineArgs

15 from vllm.engine.protocol import EngineClient

16 from vllm.envs import VLLM_V1_OUTPUT_PROC_CHUNK_SIZE

17 -from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType

17 +from vllm.inputs import PromptType

18 from vllm.inputs.preprocess import InputPreprocessor

19 from vllm.logger import init_logger

20 from vllm.lora.request import LoRARequest

21 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

22 from vllm.outputs import RequestOutput

23 from vllm.pooling_params import PoolingParams

24 from vllm.prompt_adapter.request import PromptAdapterRequest

@@ -48,7 +49,7 @@ class AsyncLLM(EngineClient):

49 executor_class: type[Executor],

50 log_stats: bool,

51 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,

51 - input_registry: InputRegistry = INPUT_REGISTRY,

52 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

53 use_cached_outputs: bool = False,

54 log_requests: bool = True,

55 start_engine_loop: bool = True,

@@ -90,7 +91,7 @@ class AsyncLLM(EngineClient):

91 self.processor = Processor(

92 vllm_config=vllm_config,

93 tokenizer=self.tokenizer,

93 - input_registry=input_registry,

94 + mm_registry=mm_registry,

95 )

97 # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).

vllm/v1/engine/llm_engine.py

+1 -3

@@ -11,7 +11,7 @@ from vllm.config import ParallelConfig, VllmConfig

11 from vllm.distributed import stateless_destroy_torch_distributed_process_group

12 from vllm.engine.arg_utils import EngineArgs

13 from vllm.engine.metrics_types import StatLoggerBase

14 -from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType

14 +from vllm.inputs import PromptType

15 from vllm.logger import init_logger

16 from vllm.lora.request import LoRARequest

17 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

@@ -44,7 +44,6 @@ class LLMEngine:

44 log_stats: bool,

45 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,

46 stat_loggers: Optional[dict[str, StatLoggerBase]] = None,

47 - input_registry: InputRegistry = INPUT_REGISTRY,

47 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

48 use_cached_outputs: bool = False,

49 multiprocess_mode: bool = False,

@@ -80,7 +79,6 @@ class LLMEngine:

79 # Processor (convert Inputs --> EngineCoreRequests)

80 self.processor = Processor(vllm_config=vllm_config,

81 tokenizer=self.tokenizer,

83 - input_registry=input_registry,

82 mm_registry=mm_registry)

84 # OutputProcessor (convert EngineCoreOutputs --> RequestOutput).

vllm/v1/engine/processor.py

+10 -12

@@ -5,8 +5,7 @@ from collections.abc import Mapping

5 from typing import Optional, Union

7 from vllm.config import VllmConfig

8 -from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,

9 - PromptType, SingletonInputsAdapter)

8 +from vllm.inputs import ProcessorInputs, PromptType

9 from vllm.inputs.parse import split_enc_dec_inputs

10 from vllm.inputs.preprocess import InputPreprocessor

11 from vllm.lora.request import LoRARequest

@@ -31,7 +30,6 @@ class Processor:

30 self,

31 vllm_config: VllmConfig,

32 tokenizer: BaseTokenizerGroup,

34 - input_registry: InputRegistry = INPUT_REGISTRY,

33 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

34 ):

@@ -210,7 +208,6 @@ class Processor:

208 self._validate_model_inputs(processed_inputs, lora_request)

209

210 encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)

213 - decoder_inputs = SingletonInputsAdapter(decoder_inputs)

211

212 # TODO: Impl encoder-decoder

213 if encoder_inputs is not None:

@@ -221,8 +218,9 @@ class Processor:

218 sampling_params = params.clone()

219 # If unset max tokens, then generate up to the max_model_len.

220 if sampling_params.max_tokens is None:

224 - sampling_params.max_tokens = (self.model_config.max_model_len -

225 - len(decoder_inputs.prompt_token_ids))

221 + sampling_params.max_tokens = (

222 + self.model_config.max_model_len -

223 + len(decoder_inputs["prompt_token_ids"]))

224 sampling_params.update_from_generation_config(

225 self.generation_config_fields, eos_token_id)

226 sampling_params.update_from_tokenizer(

@@ -232,8 +230,8 @@ class Processor:

230 sorted_mm_inputs: Optional[list[MultiModalKwargs]] = None

231 sorted_mm_positions: Optional[list[PlaceholderRange]] = None

232 sorted_mm_hashes: Optional[list[str]] = None

235 - if (decoder_mm_inputs := decoder_inputs.multi_modal_data):

236 - assert isinstance(decoder_mm_inputs, MultiModalKwargs)

233 + if decoder_inputs["type"] == "multimodal":

234 + decoder_mm_inputs = decoder_inputs["mm_kwargs"]

235

236 # The output of merged multi-modal processor (`decoder_mm_inputs`)

237 # contains the kwargs for all items from all modalities.

@@ -254,8 +252,8 @@ class Processor:

252 sorted_mm_positions,

253 sorted_mm_hashes,

254 ) = merge_and_sort_multimodal_metadata(

257 - decoder_inputs.multi_modal_placeholders,

258 - decoder_inputs.multi_modal_hashes if self.use_hash else None,

255 + decoder_inputs["mm_placeholders"],

256 + decoder_inputs["mm_hashes"] if self.use_hash else None,

257 )

258

259 # NOTE: Sort multimodal inputs/kwargs ONLY IF there are multiple

@@ -281,8 +279,8 @@ class Processor:

279

280 return EngineCoreRequest(

281 request_id=request_id,

284 - prompt=decoder_inputs.prompt,

285 - prompt_token_ids=decoder_inputs.prompt_token_ids,

282 + prompt=decoder_inputs.get("prompt"),

283 + prompt_token_ids=decoder_inputs["prompt_token_ids"],

284 mm_inputs=sorted_mm_inputs,

285 mm_hashes=sorted_mm_hashes,

286 mm_placeholders=sorted_mm_positions,

vllm/v1/worker/gpu_model_runner.py

+1 -8

@@ -15,7 +15,6 @@ from vllm.attention.layer import Attention

15 from vllm.config import CompilationLevel, VllmConfig

16 from vllm.distributed.parallel_state import get_pp_group, graph_capture

17 from vllm.forward_context import set_forward_context

18 -from vllm.inputs import INPUT_REGISTRY

18 from vllm.logger import init_logger

19 from vllm.model_executor.layers.fused_moe import FusedMoE

20 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding

@@ -130,7 +129,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):

129 self.cascade_attn_enabled = not self.model_config.disable_cascade_attn

130

131 # Multi-modal data support

133 - self.input_registry = INPUT_REGISTRY

132 self.mm_registry = MULTIMODAL_REGISTRY

133 self.uses_mrope = model_config.uses_mrope

134

@@ -1473,16 +1471,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):

1471 encoder_budget, max_num_mm_items, dummy_data_modality)

1472

1473 # Create dummy batch of multimodal inputs.

1476 - dummy_request_data = self.input_registry.dummy_data_for_profiling(

1474 + dummy_request_data = self.mm_registry.get_decoder_dummy_data(

1475 model_config=self.model_config,

1476 seq_len=self.max_num_tokens,

1479 - mm_registry=self.mm_registry,

1477 )

1478 dummy_mm_data = dummy_request_data.multi_modal_data

1482 - if not isinstance(dummy_mm_data, MultiModalKwargs):

1483 - # TODO: Delete this check once input mapper is fully removed.

1484 - raise RuntimeError(

1485 - "Legacy input mapper is not supported in V1")

1479

1480 # Dummy data definition may contain multiple multimodal items

1481 # (e.g, multiple images) for a single request, therefore here we

vllm/v1/worker/tpu_model_runner.py

-2

@@ -17,7 +17,6 @@ from vllm.attention.backends.abstract import AttentionType

17 from vllm.attention.layer import Attention

18 from vllm.config import VllmConfig

19 from vllm.forward_context import set_forward_context

20 -from vllm.inputs import INPUT_REGISTRY

20 from vllm.logger import init_logger

21 from vllm.model_executor.model_loader import get_model

22 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs

@@ -102,7 +101,6 @@ class TPUModelRunner:

101 self.hidden_size = model_config.get_hidden_size()

102

103 # Multi-modal data support

105 - self.input_registry = INPUT_REGISTRY

104 self.mm_registry = MULTIMODAL_REGISTRY

105 self.uses_mrope = model_config.uses_mrope

106 # TODO: Support M-RoPE (e.g, Qwen2-VL)