58fab50d82 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/features/multimodal_inputs.md

+25 -4

@@ -359,13 +359,19 @@ Full example: [examples/offline_inference/audio_language.py](../../examples/offl

359 To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model,

360 pass a tensor of shape `(num_items, feature_size, hidden_size of LM)` to the corresponding field of the multi-modal dictionary.

361

362 +You must enable this feature via `enable_mm_embeds=True`.

363 +

364 +!!! warning

365 + The vLLM engine may crash if incorrect shape of embeddings is passed.

366 + Only enable this flag for trusted users!

367 +

368 ??? code

369

370 ```python

371 from vllm import LLM

372

373 # Inference with image embeddings as input

368 - llm = LLM(model="llava-hf/llava-1.5-7b-hf")

374 + llm = LLM(model="llava-hf/llava-1.5-7b-hf", enable_mm_embeds=True)

375

376 # Refer to the HuggingFace repo for the correct format to use

377 prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"

@@ -397,7 +403,11 @@ For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embedd

403 image_embeds = torch.load(...)

404

405 # Qwen2-VL

400 - llm = LLM("Qwen/Qwen2-VL-2B-Instruct", limit_mm_per_prompt={"image": 4})

406 + llm = LLM(

407 + "Qwen/Qwen2-VL-2B-Instruct",

408 + limit_mm_per_prompt={"image": 4},

409 + enable_mm_embeds=True,

410 + )

411 mm_data = {

412 "image": {

413 "image_embeds": image_embeds,

@@ -407,7 +417,12 @@ For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embedd

417 }

418

419 # MiniCPM-V

410 - llm = LLM("openbmb/MiniCPM-V-2_6", trust_remote_code=True, limit_mm_per_prompt={"image": 4})

420 + llm = LLM(

421 + "openbmb/MiniCPM-V-2_6",

422 + trust_remote_code=True,

423 + limit_mm_per_prompt={"image": 4},

424 + enable_mm_embeds=True,

425 + )

426 mm_data = {

427 "image": {

428 "image_embeds": image_embeds,

@@ -732,7 +747,13 @@ Full example: [examples/online_serving/openai_chat_completion_client_for_multimo

747 ### Embedding Inputs

748

749 To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model,

735 -pass a tensor of shape to the corresponding field of the multi-modal dictionary.

750 +pass a tensor of shape `(num_items, feature_size, hidden_size of LM)` to the corresponding field of the multi-modal dictionary.

751 +

752 +You must enable this feature via the `--enable-mm-embeds` flag in `vllm serve`.

753 +

754 +!!! warning

755 + The vLLM engine may crash if incorrect shape of embeddings is passed.

756 + Only enable this flag for trusted users!

757

758 #### Image Embedding Inputs

759

docs/features/prompt_embeds.md

+5 -1

@@ -20,12 +20,16 @@ You can pass prompt embeddings from Hugging Face Transformers models to the `'p

20

21 ## Online Serving

22

23 -Our OpenAI-compatible server accepts prompt embeddings inputs via the [Completions API](https://platform.openai.com/docs/api-reference/completions). Prompt embeddings inputs are added via a new `'prompt_embeds'` key in the JSON package.

23 +Our OpenAI-compatible server accepts prompt embeddings inputs via the [Completions API](https://platform.openai.com/docs/api-reference/completions). Prompt embeddings inputs are added via a new `'prompt_embeds'` key in the JSON package and are enabled by the `--enable-prompt-embeds` flag in `vllm serve`.

24

25 When a mixture of `'prompt_embeds'` and `'prompt'` inputs are provided in a single request, the prompt embeds are always returned first.

26

27 Prompt embeddings are passed in as base64 encoded torch tensors.

28

29 +!!! warning

30 + The vLLM engine may crash if incorrect shape of embeddings is passed.

31 + Only enable this flag for trusted users!

32 +

33 ### Transformers Inputs via OpenAI Client

34

35 First, launch the OpenAI-compatible server:

examples/offline_inference/prithvi_geospatial_mae.py

+1

@@ -49,6 +49,7 @@ class PrithviMAE:

49 dtype="float16",

50 enforce_eager=True,

51 model_impl="terratorch",

52 + enable_mm_embeds=True,

53 )

54

55 def run(self, input_data, location_coords):

examples/offline_inference/prithvi_geospatial_mae_io_processor.py

+1

@@ -38,6 +38,7 @@ def main():

38 max_num_seqs=32,

39 io_processor_plugin="prithvi_to_tiff",

40 model_impl="terratorch",

41 + enable_mm_embeds=True,

42 )

43

44 pooling_params = PoolingParams(task="token_classify", activation=False)

examples/online_serving/prithvi_geospatial_mae.py

+1

@@ -19,6 +19,7 @@ import requests

19 # --task embed --trust-remote-code

20 # --skip-tokenizer-init --enforce-eager

21 # --io-processor-plugin prithvi_to_tiff

22 +# --enable-mm-embeds

23

24

25 def main():

tests/entrypoints/llm/test_prompt_validation.py

+16 -1

@@ -2,6 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 import pytest

5 +import torch

6

7 from vllm import LLM

8

@@ -12,8 +13,22 @@ def test_empty_prompt():

13 llm.generate([""])

14

15

15 -@pytest.mark.skip_v1

16 def test_out_of_vocab_token():

17 llm = LLM(model="openai-community/gpt2", enforce_eager=True)

18 with pytest.raises(ValueError, match="out of vocabulary"):

19 llm.generate({"prompt_token_ids": [999999]})

20 +

21 +

22 +def test_require_mm_embeds():

23 + llm = LLM(

24 + model="llava-hf/llava-1.5-7b-hf",

25 + enforce_eager=True,

26 + enable_mm_embeds=False,

27 + )

28 + with pytest.raises(ValueError, match="--enable-mm-embeds"):

29 + llm.generate(

30 + {

31 + "prompt": "<image>",

32 + "multi_modal_data": {"image": torch.empty(1, 1, 1)},

33 + }

34 + )

tests/entrypoints/openai/test_completion_with_prompt_embeds.py

+13

@@ -292,3 +292,16 @@ async def test_prompt_logprobs_raises_error(

292 temperature=0.0,

293 extra_body={"prompt_embeds": encoded_embeds, "prompt_logprobs": True},

294 )

295 +

296 +

297 +@pytest.mark.asyncio

298 +async def test_empty_prompt_embeds(

299 + client_with_prompt_embeds: openai.AsyncOpenAI,

300 +) -> None:

301 + await client_with_prompt_embeds.completions.create(

302 + model=MODEL_NAME,

303 + prompt="Hello",

304 + max_tokens=5,

305 + temperature=0.0,

306 + extra_body={"prompt_embeds": []},

307 + )

tests/entrypoints/openai/test_prompt_validation.py

+27 -2

@@ -2,6 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 import io

5 +from unittest.mock import Mock

6

7 # imports for structured outputs tests

8 import openai

@@ -10,7 +11,8 @@ import pytest

11 import regex as re

12 import torch

13

13 -from vllm.entrypoints.renderer import BaseRenderer

14 +from vllm.config import ModelConfig

15 +from vllm.entrypoints.renderer import CompletionRenderer

16

17 from ...utils import RemoteOpenAIServer

18

@@ -59,6 +61,10 @@ async def test_out_of_vocab_token_ids():

61 def test_load_prompt_embeds(

62 dtype: torch.dtype, layout: torch.layout, seq_len: int, hidden_size: int

63 ):

64 + model_config = Mock(spec=ModelConfig)

65 + model_config.enable_prompt_embeds = True

66 + renderer = CompletionRenderer(model_config, tokenizer=None)

67 +

68 # construct arbitrary tensors of various dtypes, layouts, and sizes.

69 # We need to check against different layouts to make sure that if a user

70 # uses sparse tensors to reduce the transmission size of prompt embeddings,

@@ -83,7 +89,7 @@ def test_load_prompt_embeds(

89 buffer.seek(0)

90 encoded_tensor = pybase64.b64encode(buffer.getvalue())

91

86 - loaded_prompt_embeds = BaseRenderer.load_prompt_embeds(encoded_tensor)

92 + loaded_prompt_embeds = renderer.load_prompt_embeds(encoded_tensor)

93 assert len(loaded_prompt_embeds) == 1

94 loaded_tensor = loaded_prompt_embeds[0]["prompt_embeds"]

95 assert loaded_tensor.device.type == "cpu"

@@ -91,3 +97,22 @@ def test_load_prompt_embeds(

97 torch.testing.assert_close(

98 loaded_tensor, tensor.to("cpu").to_dense(), equal_nan=True

99 )

100 +

101 +

102 +@pytest.mark.parametrize("dtype", [torch.float32])

103 +@pytest.mark.parametrize("seq_len", [2])

104 +@pytest.mark.parametrize("hidden_size", [2])

105 +def test_disable_prompt_embeds(dtype: torch.dtype, seq_len: int, hidden_size: int):

106 + model_config = Mock(spec=ModelConfig)

107 + model_config.enable_prompt_embeds = False

108 + renderer = CompletionRenderer(model_config, tokenizer=None)

109 +

110 + tensor = torch.randn((seq_len, hidden_size), dtype=dtype)

111 +

112 + buffer = io.BytesIO()

113 + torch.save(tensor, buffer)

114 + buffer.seek(0)

115 + encoded_tensor = pybase64.b64encode(buffer.getvalue())

116 +

117 + with pytest.raises(ValueError, match="--enable-prompt-embeds"):

118 + renderer.load_prompt_embeds(encoded_tensor)

tests/entrypoints/openai/test_vision_embeds.py renamed

+29 -31

@@ -15,30 +15,7 @@ MODEL_NAME = "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"

15 DTYPE = "float16"

16

17

18 -@pytest.fixture(scope="module")

19 -def server():

20 - args = [

21 - "--runner",

22 - "pooling",

23 - # use half precision for speed and memory savings in CI environment

24 - "--dtype",

25 - DTYPE,

26 - "--enforce-eager",

27 - "--trust-remote-code",

28 - "--skip-tokenizer-init",

29 - "--max-num-seqs",

30 - "32",

31 - "--model-impl",

32 - "terratorch",

33 - ]

34 -

35 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

36 - yield remote_server

37 -

38 -

39 -@pytest.mark.asyncio

40 -@pytest.mark.parametrize("model_name", [MODEL_NAME])

41 -async def test_single_request(server: RemoteOpenAIServer, model_name: str):

18 +def _terratorch_dummy_inputs(model_name: str):

19 pixel_values = torch.full((6, 512, 512), 1.0, dtype=torch.float16)

20 location_coords = torch.full((1, 2), 1.0, dtype=torch.float16)

21

@@ -54,7 +31,7 @@ async def test_single_request(server: RemoteOpenAIServer, model_name: str):

31 binary_data = buffer_coord.read()

32 base64_coord_embedding = base64.b64encode(binary_data).decode("utf-8")

33

57 - prompt = {

34 + return {

35 "model": model_name,

36 "additional_data": {"prompt_token_ids": [1]},

37 "encoding_format": "base64",

@@ -74,12 +51,33 @@ async def test_single_request(server: RemoteOpenAIServer, model_name: str):

51 ],

52 }

53

77 - # test single pooling

78 - response = requests.post(server.url_for("pooling"), json=prompt)

79 - response.raise_for_status()

54

81 - output = response.json()["data"][0]["data"]

55 +@pytest.mark.parametrize("model_name", [MODEL_NAME])

56 +async def test_single_request(model_name: str):

57 + args = [

58 + "--runner",

59 + "pooling",

60 + # use half precision for speed and memory savings in CI environment

61 + "--dtype",

62 + DTYPE,

63 + "--enforce-eager",

64 + "--trust-remote-code",

65 + "--max-num-seqs",

66 + "32",

67 + "--model-impl",

68 + "terratorch",

69 + "--skip-tokenizer-init",

70 + "--enable-mm-embeds",

71 + ]

72 +

73 + with RemoteOpenAIServer(MODEL_NAME, args) as server:

74 + prompt = _terratorch_dummy_inputs(model_name)

75 +

76 + # test single pooling

77 + response = requests.post(server.url_for("pooling"), json=prompt)

78 + response.raise_for_status()

79

83 - np_response = np.frombuffer(base64.b64decode(output), dtype=np.float32)

80 + output = response.json()["data"][0]["data"]

81

85 - assert len(np_response) == 524288

82 + np_response = np.frombuffer(base64.b64decode(output), dtype=np.float32)

83 + assert len(np_response) == 524288

tests/entrypoints/test_chat_utils.py

+17 -4

@@ -73,6 +73,19 @@ def phi3v_model_config_mm_interleaved():

73 )

74

75

76 +@pytest.fixture(scope="function")

77 +def phi3v_model_config_image_embeds():

78 + return ModelConfig(

79 + PHI3V_MODEL_ID,

80 + runner="generate",

81 + trust_remote_code=True,

82 + limit_mm_per_prompt={

83 + "image": 2,

84 + },

85 + enable_mm_embeds=True,

86 + )

87 +

88 +

89 @pytest.fixture(scope="module")

90 def phi3v_tokenizer():

91 return get_tokenizer(PHI3V_MODEL_ID)

@@ -799,7 +812,7 @@ def test_parse_chat_messages_empty_pil_image_with_uuid(

812

813

814 def test_parse_chat_messages_empty_image_embeds_with_uuid(

802 - phi3v_model_config,

815 + phi3v_model_config_image_embeds,

816 phi3v_tokenizer,

817 ):

818 uuid = "abcd"

@@ -813,7 +826,7 @@ def test_parse_chat_messages_empty_image_embeds_with_uuid(

826 ],

827 }

828 ],

816 - phi3v_model_config,

829 + phi3v_model_config_image_embeds,

830 phi3v_tokenizer,

831 content_format="string",

832 )

@@ -832,7 +845,7 @@ def test_parse_chat_messages_empty_image_embeds_with_uuid(

845

846 @pytest.mark.asyncio

847 async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(

835 - phi3v_model_config,

848 + phi3v_model_config_image_embeds,

849 phi3v_tokenizer,

850 ):

851 uuid = "abcd"

@@ -846,7 +859,7 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(

859 ],

860 }

861 ],

849 - phi3v_model_config,

862 + phi3v_model_config_image_embeds,

863 phi3v_tokenizer,

864 content_format="string",

865 )

tests/entrypoints/test_renderer.py

+1

@@ -17,6 +17,7 @@ from vllm.inputs.data import is_embeds_prompt

17 class MockModelConfig:

18 max_model_len: int = 100

19 encoder_config: dict | None = None

20 + enable_prompt_embeds: bool = True

21

22

23 class MockTokenizerResult:

tests/models/multimodal/generation/test_common.py

+1 -2

@@ -109,8 +109,7 @@ VLM_TEST_SETTINGS = {

109 limit_mm_per_prompt={"image": 4},

110 )

111 ],

112 - # TODO: Revert to "auto" when CPU backend can use torch > 2.6

113 - dtype="bfloat16" if current_platform.is_cpu() else "auto",

112 + vllm_runner_kwargs={"enable_mm_embeds": True},

113 marks=[pytest.mark.core_model, pytest.mark.cpu_model],

114 ),

115 "qwen2_5_vl": VLMTestInfo(

tests/models/multimodal/generation/test_qwen2_vl.py

+1

@@ -292,6 +292,7 @@ def run_embedding_input_test(

292 tensor_parallel_size=tensor_parallel_size,

293 distributed_executor_backend=distributed_executor_backend,

294 default_torch_num_threads=1,

295 + enable_mm_embeds=True,

296 ) as vllm_model:

297 outputs_per_case_for_original_input = [

298 vllm_model.generate_greedy_logprobs(

tests/models/multimodal/pooling/test_prithvi_mae.py

+1

@@ -34,6 +34,7 @@ def _run_test(

34 dtype="half",

35 enforce_eager=True,

36 skip_tokenizer_init=True,

37 + enable_mm_embeds=True,

38 # Limit the maximum number of sequences to avoid the

39 # test going OOM during the warmup run

40 max_num_seqs=32,

tests/models/test_initialization.py

+6

@@ -104,6 +104,11 @@ def can_initialize(

104 m.setenv("VLLM_ATTENTION_BACKEND", "TRITON_ATTN")

105 if model_arch == "WhisperForConditionalGeneration":

106 m.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")

107 +

108 + extra_args = {}

109 + if model_arch in ("PrithviGeoSpatialMAE", "Terratorch"):

110 + extra_args["enable_mm_embeds"] = True

111 +

112 LLM(

113 model_info.default,

114 tokenizer=model_info.tokenizer,

@@ -128,6 +133,7 @@ def can_initialize(

133 else "vllm",

134 hf_overrides=hf_overrides_fn,

135 max_num_seqs=model_info.max_num_seqs,

136 + **extra_args,

137 )

138

139

tests/models/test_terratorch.py

+1

@@ -32,6 +32,7 @@ def test_inference(

32 dtype="half",

33 enforce_eager=True,

34 skip_tokenizer_init=True,

35 + enable_mm_embeds=True,

36 # Limit the maximum number of sequences to avoid the

37 # test going OOM during the warmup run

38 max_num_seqs=32,

tests/plugins_tests/test_io_processor_plugins.py

+1

@@ -38,6 +38,7 @@ def server():

38 "prithvi_to_tiff",

39 "--model-impl",

40 "terratorch",

41 + "--enable-mm-embeds",

42 ]

43

44 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

tests/v1/entrypoints/openai/test_completion.py

-15

@@ -6,7 +6,6 @@ import openai # use the official client for correctness check

6 import pytest

7 import pytest_asyncio

8 import regex as re

9 -import requests

9 from openai import BadRequestError

10

11 from tests.utils import RemoteOpenAIServer

@@ -686,17 +685,3 @@ async def test_invalid_grammar(client: openai.AsyncOpenAI, model_name: str):

685 "structured_outputs": {"grammar": invalid_simplified_sql_grammar}

686 },

687 )

689 -

690 -

691 -@pytest.mark.asyncio

692 -async def test_completion_with_empty_prompt_embeds(client: openai.AsyncOpenAI) -> None:

693 - """Test completion with empty prompt embeds."""

694 - payload: dict[str, object] = {"prompt": "Hello", "prompt_embeds": []}

695 - headers: dict[str, str] = {"Content-Type": "application/json"}

696 - # base_url = http://localhost:8000/v1/completions

697 - response = requests.post(

698 - f"{client.base_url}completions", headers=headers, json=payload

699 - )

700 - assert response.status_code == 200, (

701 - f"Expected status code 200, got {response.status_code}. "

702 - )

tests/v1/entrypoints/openai/test_completion_with_image_embeds.py

+1

@@ -32,6 +32,7 @@ def default_image_embeds_server_args() -> list[str]:

32 "--enforce-eager",

33 "--limit-mm-per-prompt",

34 json.dumps({"image": MAXIMUM_IMAGES}),

35 + "--enable-mm-embeds",

36 ]

37

38

vllm/config/model.py

+7 -2

@@ -232,8 +232,10 @@ class ModelConfig:

232 output will contain token ids."""

233 enable_prompt_embeds: bool = False

234 """If `True`, enables passing text embeddings as inputs via the

235 - `prompt_embeds` key. Note that enabling this will double the time required

236 - for graph compilation."""

235 + `prompt_embeds` key.

236 +

237 + WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed.

238 + Only enable this flag for trusted users!"""

239 served_model_name: str | list[str] | None = None

240 """The model name(s) used in the API. If multiple names are provided, the

241 server will respond to any of the provided names. The model name in the

@@ -303,6 +305,7 @@ class ModelConfig:

305 """Configuration for multimodal model. If `None`, this will be inferred

306 from the architecture of `self.model`."""

307 limit_mm_per_prompt: InitVar[dict[str, int | dict[str, int]] | None] = None

308 + enable_mm_embeds: InitVar[bool | None] = None

309 media_io_kwargs: InitVar[dict[str, dict[str, Any]] | None] = None

310 mm_processor_kwargs: InitVar[dict[str, Any] | None] = None

311 mm_processor_cache_gb: InitVar[float | None] = None

@@ -421,6 +424,7 @@ class ModelConfig:

424 self,

425 # Multimodal config init vars

426 limit_mm_per_prompt: dict[str, int] | None,

427 + enable_mm_embeds: bool | None,

428 media_io_kwargs: dict[str, dict[str, Any]] | None,

429 mm_processor_kwargs: dict[str, Any] | None,

430 mm_processor_cache_gb: float | None,

@@ -731,6 +735,7 @@ class ModelConfig:

735

736 mm_config_kwargs = dict(

737 limit_per_prompt=limit_mm_per_prompt,

738 + enable_mm_embeds=enable_mm_embeds,

739 media_io_kwargs=media_io_kwargs,

740 mm_processor_kwargs=mm_processor_kwargs,

741 mm_processor_cache_gb=mm_processor_cache_gb,

vllm/config/multimodal.py

+8

@@ -75,6 +75,14 @@ class MultiModalConfig:

75 {"image": 16, "video": {"count": 1, "num_frames": 32, "width": 512,

76 "height": 512}}

77 """

78 + enable_mm_embeds: bool = False

79 + """If `True`, enables passing multimodal embeddings:

80 + for `LLM` class, this refers to tensor inputs under `multi_modal_data`;

81 + for the OpenAI-compatible server, this refers to chat messages with content

82 + `"type": "*_embeds"`.

83 +

84 + WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed.

85 + Only enable this flag for trusted users!"""

86 media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)

87 """Additional args passed to process media inputs, keyed by modalities.

88 For example, to set num_frames for video, set

vllm/engine/arg_utils.py

+5

@@ -438,6 +438,7 @@ class EngineArgs:

438 limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field(

439 MultiModalConfig, "limit_per_prompt"

440 )

441 + enable_mm_embeds: bool = MultiModalConfig.enable_mm_embeds

442 interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings

443 media_io_kwargs: dict[str, dict[str, Any]] = get_field(

444 MultiModalConfig, "media_io_kwargs"

@@ -896,6 +897,9 @@ class EngineArgs:

897 multimodal_group.add_argument(

898 "--limit-mm-per-prompt", **multimodal_kwargs["limit_per_prompt"]

899 )

900 + multimodal_group.add_argument(

901 + "--enable-mm-embeds", **multimodal_kwargs["enable_mm_embeds"]

902 + )

903 multimodal_group.add_argument(

904 "--media-io-kwargs", **multimodal_kwargs["media_io_kwargs"]

905 )

@@ -1159,6 +1163,7 @@ class EngineArgs:

1163 enable_prompt_embeds=self.enable_prompt_embeds,

1164 served_model_name=self.served_model_name,

1165 limit_mm_per_prompt=self.limit_mm_per_prompt,

1166 + enable_mm_embeds=self.enable_mm_embeds,

1167 interleave_mm_strings=self.interleave_mm_strings,

1168 media_io_kwargs=self.media_io_kwargs,

1169 skip_mm_profiling=self.skip_mm_profiling,

vllm/entrypoints/chat_utils.py

+20

@@ -811,6 +811,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):

811 allowed_media_domains=tracker.allowed_media_domains,

812 )

813

814 + @property

815 + def model_config(self) -> ModelConfig:

816 + return self._tracker.model_config

817 +

818 def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:

819 image = self._connector.fetch_image(image_url) if image_url else None

820

@@ -822,6 +826,12 @@ class MultiModalContentParser(BaseMultiModalContentParser):

826 image_embeds: str | dict[str, str] | None,

827 uuid: str | None = None,

828 ) -> None:

829 + mm_config = self.model_config.get_multimodal_config()

830 + if not mm_config.enable_mm_embeds:

831 + raise ValueError(

832 + "You must set `--enable-mm-embeds` to input `image_embeds`"

833 + )

834 +

835 if isinstance(image_embeds, dict):

836 embeds = {

837 k: self._connector.fetch_image_embedding(v)

@@ -886,6 +896,10 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

896 allowed_media_domains=tracker.allowed_media_domains,

897 )

898

899 + @property

900 + def model_config(self) -> ModelConfig:

901 + return self._tracker.model_config

902 +

903 def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:

904 image_coro = self._connector.fetch_image_async(image_url) if image_url else None

905

@@ -897,6 +911,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

911 image_embeds: str | dict[str, str] | None,

912 uuid: str | None = None,

913 ) -> None:

914 + mm_config = self.model_config.get_multimodal_config()

915 + if not mm_config.enable_mm_embeds:

916 + raise ValueError(

917 + "You must set `--enable-mm-embeds` to input `image_embeds`"

918 + )

919 +

920 future: asyncio.Future[str | dict[str, str] | None] = asyncio.Future()

921

922 if isinstance(image_embeds, dict):

vllm/entrypoints/renderer.py

+5 -2

@@ -156,14 +156,17 @@ class BaseRenderer(ABC):

156 """

157 raise NotImplementedError

158

159 - @classmethod

159 def load_prompt_embeds(

161 - cls,

160 + self,

161 prompt_embeds: bytes | list[bytes],

162 truncate_prompt_tokens: Annotated[int, Field(ge=0)] | None = None,

163 cache_salt: str | None = None,

164 ) -> list[EngineEmbedsPrompt]:

165 """Load and validate base64-encoded embeddings into prompt objects."""

166 + if not self.model_config.enable_prompt_embeds:

167 + raise ValueError(

168 + "You must set `--enable-prompt-embeds` to input `prompt_embeds`."

169 + )

170

171 def _load_and_validate_embed(embed: bytes) -> EngineEmbedsPrompt:

172 tensor = torch.load(

vllm/multimodal/processing.py

+10

@@ -1308,6 +1308,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1308 [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].

1309 """

1310 mm_items = self.data_parser.parse_mm_data(mm_data)

1311 +

1312 + mm_config = self.info.ctx.model_config.get_multimodal_config()

1313 + if not mm_config.enable_mm_embeds:

1314 + for modality, items in mm_items.items():

1315 + if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):

1316 + raise ValueError(

1317 + f"You must set `--enable-mm-embeds` to input "

1318 + f"`{modality}_embeds`"

1319 + )

1320 +

1321 for modality, items in mm_items.items():

1322 self.validate_num_items(modality, len(items))

1323

[Frontend] Require flag for loading text and image embeds (#27204)