@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 80b18230e0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 01:00 UTC

[Frontend] Add multimodal support to /inference/v1/generate endpoint (#38405)

Signed-off-by: Nithin Chalapathi <nithin.ch10@gmail.com> Signed-off-by: Nithin Chalapathi <nithinc@berkeley.edu> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Nithin Chalapathi committed Apr 17, 2026 at 20:31 UTC 80b18230e0ffe94f8198862675a0dc5e86c10997
9 files changed +542 -25
examples/online_serving/disaggregated_serving/example_mm_serve.py new
+117
@@ -0,0 +1,117 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Disaggregated multimodal serving: render → generate round-trip.
4 +
5 +Demonstrates the two-phase disaggregated flow:
6 + 1. /v1/chat/completions/render – preprocesses a multimodal chat request
7 + into token IDs and serialized tensor features.
8 + 2. /inference/v1/generate – runs inference on the preprocessed tokens.
9 +
10 +The render response is passed *directly* to generate with only
11 +``sampling_params`` added, showing that the two endpoints compose with
12 +zero client-side transformation.
13 +
14 +Launch the server first:
15 +
16 + vllm serve Qwen/Qwen3-VL-2B-Instruct \
17 + --dtype bfloat16 --max-model-len 4096 --enforce-eager
18 +
19 +Then run this script:
20 +
21 + python example_mm_serve.py
22 +"""
23 +
24 +import io
25 +
26 +import pybase64 as base64
27 +import requests
28 +from PIL import Image
29 +from transformers import AutoTokenizer
30 +
31 +BASE_URL = "http://localhost:8000"
32 +MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
33 +
34 +
35 +def make_data_url(image: Image.Image) -> str:
36 + """Encode a PIL image as a base64 data URL."""
37 + buf = io.BytesIO()
38 + image.save(buf, format="PNG")
39 + b64 = base64.b64encode(buf.getvalue()).decode()
40 + return f"data:image/png;base64,{b64}"
41 +
42 +
43 +def main():
44 + # -- Step 1: Create a test image (solid red) -------------------------
45 + image = Image.new("RGB", (224, 224), color=(255, 0, 0))
46 + data_url = make_data_url(image)
47 + print("Created 224x224 red test image")
48 +
49 + # -- Step 2: Render (preprocess) -------------------------------------
50 + render_payload = {
51 + "model": MODEL_NAME,
52 + "messages": [
53 + {
54 + "role": "user",
55 + "content": [
56 + {"type": "image_url", "image_url": {"url": data_url}},
57 + {
58 + "type": "text",
59 + "text": "What color is this image? Answer in one word.",
60 + },
61 + ],
62 + }
63 + ],
64 + }
65 +
66 + print("\n--- Render ---")
67 + render_resp = requests.post(
68 + f"{BASE_URL}/v1/chat/completions/render", json=render_payload
69 + )
70 + render_resp.raise_for_status()
71 + render_data = render_resp.json()
72 +
73 + print(f"Response keys: {list(render_data.keys())}")
74 + print(f"Number of token_ids: {len(render_data['token_ids'])}")
75 +
76 + features = render_data.get("features")
77 + if features and features.get("kwargs_data"):
78 + print(f"kwargs_data modalities: {list(features['kwargs_data'].keys())}")
79 + for modality, items in features["kwargs_data"].items():
80 + print(
81 + f" {modality}: {len(items)} item(s), "
82 + f"first item type: {type(items[0])} length: {len(items[0])}"
83 + if items
84 + else "First item: (empty)"
85 + )
86 + else:
87 + print("WARNING: no kwargs_data in render response")
88 +
89 + # -- Step 3: Generate (inference) ------------------------------------
90 + # Pass the render output directly — only add sampling_params.
91 + generate_payload = render_data
92 + generate_payload["sampling_params"] = {
93 + "max_tokens": 20,
94 + "temperature": 0.0,
95 + }
96 +
97 + print("\n--- Generate ---")
98 + gen_resp = requests.post(f"{BASE_URL}/inference/v1/generate", json=generate_payload)
99 + gen_resp.raise_for_status()
100 + gen_data = gen_resp.json()
101 +
102 + # -- Step 4: Decode & print ------------------------------------------
103 + output_ids = gen_data["choices"][0]["token_ids"]
104 + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
105 + text = tokenizer.decode(output_ids, skip_special_tokens=True)
106 +
107 + print(f"Output token count: {len(output_ids)}")
108 + print(f"Generated text: {text!r}")
109 +
110 + if "red" in text.lower():
111 + print("\nModel correctly identified the red image.")
112 + else:
113 + print(f"\nWARNING: Expected 'red' in output, got: {text!r}")
114 +
115 +
116 +if __name__ == "__main__":
117 + main()
tests/entrypoints/openai/test_mm_serde.py new
+111
@@ -0,0 +1,111 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Roundtrip tests for multimodal serde used by the disagg generate endpoint."""
4 +
5 +import torch
6 +
7 +from vllm.entrypoints.serve.disagg.mm_serde import (
8 + decode_mm_kwargs_item,
9 + encode_mm_kwargs_item,
10 +)
11 +from vllm.entrypoints.serve.disagg.protocol import (
12 + MultiModalFeatures,
13 + PlaceholderRangeInfo,
14 +)
15 +from vllm.multimodal.inputs import (
16 + MultiModalBatchedField,
17 + MultiModalFieldElem,
18 + MultiModalFlatField,
19 + MultiModalKwargsItem,
20 + MultiModalSharedField,
21 +)
22 +
23 +
24 +def test_mm_kwargs_item_roundtrip():
25 + """Full roundtrip test with all three field types and multiple dtypes."""
26 + e1 = MultiModalFieldElem(
27 + data=torch.zeros(1000, dtype=torch.bfloat16),
28 + field=MultiModalBatchedField(),
29 + )
30 + e2 = MultiModalFieldElem(
31 + data=torch.ones(100, dtype=torch.int32),
32 + field=MultiModalSharedField(batch_size=4),
33 + )
34 + e3 = MultiModalFieldElem(
35 + data=torch.randn(20, dtype=torch.float32),
36 + field=MultiModalFlatField(slices=[slice(0, 10), slice(10, 20)], dim=0),
37 + )
38 +
39 + item = MultiModalKwargsItem({"pixel_values": e1, "grid_thw": e2, "embeds": e3})
40 + encoded = encode_mm_kwargs_item(item)
41 +
42 + # Encoded result is a base64 string
43 + assert isinstance(encoded, str)
44 +
45 + decoded = decode_mm_kwargs_item(encoded)
46 +
47 + assert set(decoded.keys()) == {"pixel_values", "grid_thw", "embeds"}
48 + assert torch.equal(item["pixel_values"].data, decoded["pixel_values"].data)
49 + assert torch.equal(item["grid_thw"].data, decoded["grid_thw"].data)
50 + assert torch.equal(item["embeds"].data, decoded["embeds"].data)
51 + assert isinstance(decoded["pixel_values"].field, MultiModalBatchedField)
52 + assert isinstance(decoded["grid_thw"].field, MultiModalSharedField)
53 + assert isinstance(decoded["embeds"].field, MultiModalFlatField)
54 +
55 +
56 +def test_mm_kwargs_item_none_data():
57 + """Roundtrip with None data field."""
58 + elem = MultiModalFieldElem(
59 + data=None,
60 + field=MultiModalSharedField(batch_size=2),
61 + )
62 + item = MultiModalKwargsItem({"empty": elem})
63 + encoded = encode_mm_kwargs_item(item)
64 + decoded = decode_mm_kwargs_item(encoded)
65 +
66 + assert decoded["empty"].data is None
67 + assert isinstance(decoded["empty"].field, MultiModalSharedField)
68 +
69 +
70 +def test_mm_kwargs_item_nested_tensors():
71 + """Roundtrip with nested tensor data."""
72 + nested = [torch.randn(3, 4), torch.randn(5, 4)]
73 + elem = MultiModalFieldElem(
74 + data=nested,
75 + field=MultiModalBatchedField(),
76 + )
77 + item = MultiModalKwargsItem({"nested": elem})
78 + encoded = encode_mm_kwargs_item(item)
79 + decoded = decode_mm_kwargs_item(encoded)
80 +
81 + decoded_data = decoded["nested"].data
82 + assert len(decoded_data) == 2
83 + assert torch.equal(nested[0], decoded_data[0])
84 + assert torch.equal(nested[1], decoded_data[1])
85 +
86 +
87 +def test_mm_features_with_kwargs_data():
88 + """Test that MultiModalFeatures can carry serialized tensor data."""
89 + elem = MultiModalFieldElem(
90 + data=torch.randn(5, 3, dtype=torch.float32),
91 + field=MultiModalBatchedField(),
92 + )
93 + item = MultiModalKwargsItem({"pixel_values": elem})
94 + encoded = encode_mm_kwargs_item(item)
95 +
96 + features = MultiModalFeatures(
97 + mm_hashes={"image": ["abc123"]},
98 + mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=10)]},
99 + kwargs_data={"image": [encoded]},
100 + )
101 +
102 + # JSON roundtrip
103 + json_str = features.model_dump_json()
104 + features2 = MultiModalFeatures.model_validate_json(json_str)
105 +
106 + assert features2.mm_hashes == {"image": ["abc123"]}
107 + assert features2.kwargs_data is not None
108 + assert len(features2.kwargs_data["image"]) == 1
109 +
110 + decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0])
111 + assert torch.equal(elem.data, decoded["pixel_values"].data)
tests/entrypoints/serve/disagg/test_serving_multimodal_tokens.py new
+158
@@ -0,0 +1,158 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for multimodal features through the /inference/v1/generate endpoint.
4 +
5 +Mirrors test_serving_tokens.py but exercises the multimodal piping
6 +using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render ->
7 +/generate -> /detokenize path. Intentionally avoids running the HF
8 +processor in the pytest parent process to keep os.fork() in sibling
9 +tests (e.g. test_weight_transfer_llm.py) deadlock-free.
10 +"""
11 +
12 +import os
13 +
14 +import httpx
15 +import pytest
16 +import pytest_asyncio
17 +from PIL import Image
18 +
19 +from tests.utils import RemoteOpenAIServer
20 +from vllm.multimodal.utils import encode_image_url
21 +
22 +MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
23 +GEN_ENDPOINT = "/inference/v1/generate"
24 +RENDER_ENDPOINT = "/v1/chat/completions/render"
25 +DETOKENIZE_ENDPOINT = "/detokenize"
26 +
27 +
28 +@pytest.fixture(scope="module")
29 +def test_image():
30 + return Image.new("RGB", (224, 224), color=(255, 0, 0))
31 +
32 +
33 +@pytest.fixture(scope="module")
34 +def server():
35 + args = [
36 + "--dtype",
37 + "bfloat16",
38 + "--max-model-len",
39 + "4096",
40 + "--enforce-eager",
41 + "--no-enable-prefix-caching",
42 + ]
43 +
44 + envs = os.environ.copy()
45 + envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
46 +
47 + with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
48 + yield remote_server
49 +
50 +
51 +@pytest_asyncio.fixture
52 +async def client(server: RemoteOpenAIServer):
53 + transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
54 + headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
55 + async with httpx.AsyncClient(
56 + transport=transport,
57 + base_url=server.url_root,
58 + timeout=600,
59 + headers=headers,
60 + ) as c:
61 + yield c
62 +
63 +
64 +@pytest.mark.asyncio
65 +async def test_render_to_generate_roundtrip(client, test_image):
66 + """End-to-end: render a multimodal chat -> feed into generate -> decode.
67 +
68 + All preprocessing and detokenization happens in the server subprocess;
69 + the pytest parent never imports transformers or touches torch tensors.
70 + """
71 + data_url = encode_image_url(test_image, format="PNG")
72 +
73 + render_payload = {
74 + "model": MODEL_NAME,
75 + "messages": [
76 + {
77 + "role": "user",
78 + "content": [
79 + {"type": "image_url", "image_url": {"url": data_url}},
80 + {
81 + "type": "text",
82 + "text": "What color is this image? Answer in one word.",
83 + },
84 + ],
85 + }
86 + ],
87 + }
88 +
89 + render_resp = await client.post(RENDER_ENDPOINT, json=render_payload)
90 + render_resp.raise_for_status()
91 + render_data = render_resp.json()
92 +
93 + # Validate render output structure: keys exist and values are non-empty
94 + # and well-typed.
95 + assert "token_ids" in render_data
96 + assert isinstance(render_data["token_ids"], list)
97 + assert len(render_data["token_ids"]) > 0
98 + assert all(isinstance(t, int) for t in render_data["token_ids"])
99 +
100 + assert "features" in render_data
101 + features = render_data["features"]
102 + assert features is not None
103 + assert isinstance(features, dict)
104 +
105 + assert "mm_hashes" in features
106 + assert "image" in features["mm_hashes"]
107 + image_hashes = features["mm_hashes"]["image"]
108 + assert isinstance(image_hashes, list)
109 + assert len(image_hashes) > 0
110 + assert all(isinstance(h, str) and h for h in image_hashes)
111 +
112 + assert "mm_placeholders" in features
113 + assert "image" in features["mm_placeholders"]
114 + image_placeholders = features["mm_placeholders"]["image"]
115 + assert isinstance(image_placeholders, list)
116 + assert len(image_placeholders) > 0
117 + for p in image_placeholders:
118 + assert isinstance(p.get("offset"), int)
119 + assert isinstance(p.get("length"), int)
120 + assert p["length"] > 0
121 +
122 + assert "kwargs_data" in features
123 + assert "image" in features["kwargs_data"]
124 + assert len(features["kwargs_data"]["image"]) > 0
125 +
126 + # Build generate request from render output
127 + generate_payload = render_data
128 + generate_payload["sampling_params"] = {
129 + "max_tokens": 10,
130 + "temperature": 0.0,
131 + }
132 +
133 + gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload)
134 + gen_resp.raise_for_status()
135 + gen_data = gen_resp.json()
136 +
137 + assert "choices" in gen_data
138 + assert isinstance(gen_data["choices"], list)
139 + assert len(gen_data["choices"]) >= 1
140 + choice = gen_data["choices"][0]
141 + assert "token_ids" in choice
142 + assert isinstance(choice["token_ids"], list)
143 + assert len(choice["token_ids"]) > 0
144 + assert all(isinstance(t, int) for t in choice["token_ids"])
145 +
146 + detok_resp = await client.post(
147 + DETOKENIZE_ENDPOINT,
148 + json={"model": MODEL_NAME, "tokens": choice["token_ids"]},
149 + )
150 + detok_resp.raise_for_status()
151 + detok_data = detok_resp.json()
152 + assert "prompt" in detok_data
153 + text = detok_data["prompt"]
154 + assert isinstance(text, str)
155 + assert len(text) > 0
156 + assert "red" in text.lower(), (
157 + f"Expected model to identify the red image, got: {text!r}"
158 + )
tests/utils_/test_serial_utils.py
+41 -2
@@ -7,17 +7,39 @@ from tests.models.utils import check_embeddings_close
7 from vllm.utils.serial_utils import (
8 EMBED_DTYPES,
9 ENDIANNESS,
10 + MM_METADATA_DTYPES,
11 EmbedDType,
12 Endianness,
13 + MmMetadataDType,
14 binary2tensor,
15 tensor2binary,
16 )
17
18 +FLOAT_EMBED_DTYPES = tuple(EMBED_DTYPES.keys())
19 +INTEGER_EMBED_DTYPES = tuple(MM_METADATA_DTYPES.keys())
20 +
21 +
22 +def _build_integer_tensor(
23 + embed_dtype: MmMetadataDType, shape: tuple[int, ...]
24 +) -> torch.Tensor:
25 + torch_dtype = MM_METADATA_DTYPES[embed_dtype].torch_dtype
26 +
27 + if torch_dtype is torch.bool:
28 + return torch.randint(0, 2, shape, dtype=torch.int32).to(torch.bool)
29 + if torch_dtype is torch.uint8:
30 + return torch.randint(0, 256, shape, dtype=torch.uint8)
31 + if torch_dtype is torch.int32:
32 + return torch.randint(-(2**20), 2**20, shape, dtype=torch.int32)
33 + if torch_dtype is torch.int64:
34 + return torch.randint(-(2**62), 2**62, shape, dtype=torch.int64)
35 +
36 + raise AssertionError(f"Unsupported non-floating embed dtype: {embed_dtype}")
37 +
38
39 @pytest.mark.parametrize("endianness", ENDIANNESS)
18 -@pytest.mark.parametrize("embed_dtype", EMBED_DTYPES.keys())
40 +@pytest.mark.parametrize("embed_dtype", FLOAT_EMBED_DTYPES)
41 @torch.inference_mode()
20 -def test_encode_and_decode(embed_dtype: EmbedDType, endianness: Endianness):
42 +def test_encode_and_decode_floats(embed_dtype: EmbedDType, endianness: Endianness):
43 for i in range(10):
44 tensor = torch.rand(2, 3, 5, 7, 11, 13, device="cpu", dtype=torch.float32)
45 shape = tensor.shape
@@ -40,3 +62,20 @@ def test_encode_and_decode(embed_dtype: EmbedDType, endianness: Endianness):
62 name_1="new",
63 tol=1e-2,
64 )
65 +
66 +
67 +@pytest.mark.parametrize("endianness", ENDIANNESS)
68 +@pytest.mark.parametrize("embed_dtype", INTEGER_EMBED_DTYPES)
69 +@torch.inference_mode()
70 +def test_encode_and_decode_integers(
71 + embed_dtype: MmMetadataDType, endianness: Endianness
72 +):
73 + shape = (2, 3, 5, 7, 11, 13)
74 +
75 + for i in range(10):
76 + tensor = _build_integer_tensor(embed_dtype, shape)
77 + binary = tensor2binary(tensor, embed_dtype, endianness)
78 + new_tensor = binary2tensor(binary, shape, embed_dtype, endianness)
79 +
80 + assert new_tensor.dtype == MM_METADATA_DTYPES[embed_dtype].torch_dtype
81 + torch.testing.assert_close(tensor, new_tensor, atol=0, rtol=0)
vllm/entrypoints/serve/disagg/mm_serde.py new
+27
@@ -0,0 +1,27 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Encode/decode utilities for multimodal tensors and field metadata
4 +over JSON/HTTP, used by the disaggregated generate endpoint."""
5 +
6 +from __future__ import annotations
7 +
8 +import pybase64
9 +
10 +from vllm.multimodal.inputs import MultiModalKwargsItem
11 +from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
12 +
13 +_encoder = MsgpackEncoder(size_threshold=2**62) # force all tensors inline
14 +_decoder = MsgpackDecoder(t=MultiModalKwargsItem)
15 +
16 +
17 +def encode_mm_kwargs_item(item: MultiModalKwargsItem) -> str:
18 + """Serialize a MultiModalKwargsItem to a base64 string."""
19 + bufs = _encoder.encode(item)
20 + assert len(bufs) == 1, "All tensors should be inline"
21 + return pybase64.b64encode(bufs[0]).decode("ascii")
22 +
23 +
24 +def decode_mm_kwargs_item(data: str) -> MultiModalKwargsItem:
25 + """Deserialize a base64 string back to a MultiModalKwargsItem."""
26 + raw = pybase64.b64decode(data)
27 + return _decoder.decode(raw)
vllm/entrypoints/serve/disagg/protocol.py
+9 -8
@@ -35,14 +35,6 @@ class MultiModalFeatures(BaseModel):
35 Carries hashes (for cache lookup / identification) and placeholder
36 positions so the downstream `/generate` service knows *where* in
37 the token sequence each multimodal item lives.
38 -
39 - Note:
40 - Phase 1 — metadata only.
41 - Phase 2 should add `mm_kwargs` (processed tensor data) using a
42 - binary transport so the ``/generate` side can skip re-processing.
43 - The `/generate` endpoint must also be updated to inject these
44 - features into `EngineInput` before passing to
45 - `InputProcessor.process_inputs`.
38 """
39
40 mm_hashes: dict[str, list[str]]
@@ -51,6 +43,15 @@ class MultiModalFeatures(BaseModel):
43 mm_placeholders: dict[str, list[PlaceholderRangeInfo]]
44 """Per-modality placeholder ranges in the token sequence."""
45
46 + kwargs_data: dict[str, list[str | None]] | None = None
47 + """Per-modality serialized tensor data.
48 +
49 + Each value is a list parallel to ``mm_hashes[modality]``. A ``str``
50 + entry is a base64-encoded ``MultiModalKwargsItem``; ``None`` means
51 + the item should be resolved from cache. The entire field is
52 + ``None`` for metadata-only (cache-hit) responses.
53 + """
54 +
55
56 class GenerateRequest(BaseModel):
57 request_id: str = Field(
vllm/entrypoints/serve/disagg/serving.py
+43 -5
@@ -25,6 +25,7 @@ from vllm.entrypoints.openai.engine.protocol import (
25 )
26 from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
27 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
28 +from vllm.entrypoints.serve.disagg.mm_serde import decode_mm_kwargs_item
29 from vllm.entrypoints.serve.disagg.protocol import (
30 GenerateRequest,
31 GenerateResponse,
@@ -34,8 +35,14 @@ from vllm.entrypoints.serve.disagg.protocol import (
35 )
36 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
37 from vllm.entrypoints.utils import should_include_usage
38 +from vllm.inputs import EngineInput, mm_input
39 from vllm.logger import init_logger
40 from vllm.logprobs import Logprob
41 +from vllm.multimodal.inputs import (
42 + MultiModalKwargsItem,
43 + MultiModalKwargsItems,
44 + PlaceholderRange,
45 +)
46 from vllm.outputs import RequestOutput
47 from vllm.sampling_params import RequestOutputKind, SamplingParams
48 from vllm.utils.collection_utils import as_list
@@ -103,11 +110,42 @@ class ServingTokens(OpenAIServing):
110 if raw_request:
111 raw_request.state.request_metadata = request_metadata
112
106 - (engine_input,) = await self.openai_serving_render.preprocess_completion(
107 - request,
108 - prompt_input=request.token_ids,
109 - prompt_embeds=None,
110 - )
113 + engine_input: EngineInput
114 + if features := request.features:
115 + # Convert PlaceholderRangeInfo → PlaceholderRange per modality.
116 + mm_placeholders: dict[str, list[PlaceholderRange]] = {
117 + modality: [
118 + PlaceholderRange(offset=p.offset, length=p.length) for p in ranges
119 + ]
120 + for modality, ranges in features.mm_placeholders.items()
121 + }
122 +
123 + # Deserialize tensor data when present; None → cache hit.
124 + mm_kwargs: dict[str, list[MultiModalKwargsItem | None]] = {}
125 + if features.kwargs_data is not None:
126 + for modality, items in features.kwargs_data.items():
127 + mm_kwargs[modality] = [
128 + decode_mm_kwargs_item(item) if item is not None else None
129 + for item in items
130 + ]
131 + else:
132 + for modality, hashes in features.mm_hashes.items():
133 + mm_kwargs[modality] = [None] * len(hashes)
134 +
135 + engine_input = mm_input(
136 + prompt_token_ids=request.token_ids,
137 + mm_kwargs=MultiModalKwargsItems(mm_kwargs),
138 + mm_hashes=features.mm_hashes,
139 + mm_placeholders=mm_placeholders,
140 + cache_salt=request.cache_salt,
141 + )
142 + else:
143 + (engine_input,) = await self.openai_serving_render.preprocess_completion(
144 + request,
145 + prompt_input=request.token_ids,
146 + prompt_embeds=None,
147 + skip_mm_cache=True,
148 + )
149
150 # Schedule the request and get the result generator.
151 result_generator: AsyncGenerator[RequestOutput, None] | None = None
vllm/entrypoints/serve/render/serving.py
+20 -4
@@ -2,7 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from collections.abc import Sequence
4 from http import HTTPStatus
5 -from typing import Any
5 +from typing import Any, cast
6
7 from openai_harmony import Message as OpenAIMessage
8
@@ -25,6 +25,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
25 render_for_completion,
26 )
27 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
28 +from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
29 from vllm.entrypoints.serve.disagg.protocol import (
30 GenerateRequest,
31 MultiModalFeatures,
@@ -37,6 +38,7 @@ from vllm.entrypoints.utils import (
38 from vllm.inputs import (
39 EngineInput,
40 MultiModalHashes,
41 + MultiModalInput,
42 MultiModalPlaceholders,
43 PromptType,
44 SingletonPrompt,
@@ -251,6 +253,7 @@ class OpenAIServingRender:
253 default_template_kwargs=self.default_chat_template_kwargs,
254 tool_dicts=tool_dicts,
255 tool_parser=tool_parser,
256 + skip_mm_cache=True,
257 reasoning_parser=self.reasoning_parser,
258 )
259 else:
@@ -342,6 +345,7 @@ class OpenAIServingRender:
345 request,
346 prompt_input=request.prompt,
347 prompt_embeds=request.prompt_embeds,
348 + skip_mm_cache=True,
349 )
350
351 return engine_inputs
@@ -357,9 +361,10 @@ class OpenAIServingRender:
361 if engine_input.get("type") != "multimodal":
362 return None
363
360 - # At this point engine_input is a MultiModalInputs TypedDict.
361 - mm_hashes: MultiModalHashes = engine_input["mm_hashes"] # type: ignore[typeddict-item]
362 - raw_placeholders: MultiModalPlaceholders = engine_input["mm_placeholders"] # type: ignore[typeddict-item]
364 + # At this point engine_input is a MultiModalInput TypedDict.
365 + mm_engine_input = cast(MultiModalInput, engine_input)
366 + mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
367 + raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
368
369 mm_placeholders = {
370 modality: [
@@ -368,9 +373,20 @@ class OpenAIServingRender:
373 for modality, ranges in raw_placeholders.items()
374 }
375
376 + # Serialize tensor data per modality.
377 + kwargs_data: dict[str, list[str | None]] | None = None
378 + if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
379 + kwargs_data = {}
380 + for modality, items in raw_mm_kwargs.items():
381 + kwargs_data[modality] = [
382 + encode_mm_kwargs_item(item) if item is not None else None
383 + for item in items
384 + ]
385 +
386 return MultiModalFeatures(
387 mm_hashes=mm_hashes,
388 mm_placeholders=mm_placeholders,
389 + kwargs_data=kwargs_data,
390 )
391
392 def _make_request_with_harmony(
vllm/utils/serial_utils.py
+16 -6
@@ -27,6 +27,7 @@ class DTypeInfo:
27
28
29 EmbedDType = Literal["float32", "float16", "bfloat16", "fp8_e4m3", "fp8_e5m2"]
30 +MmMetadataDType = Literal["int32", "int64", "uint8", "bool"]
31 Endianness = Literal["native", "big", "little"]
32 EncodingFormat = Literal["float", "base64", "bytes", "bytes_only"]
33
@@ -42,6 +43,15 @@ EMBED_DTYPES: Mapping[EmbedDType, DTypeInfo] = {
43 "fp8_e4m3": DTypeInfo(torch.float8_e4m3fn, torch.uint8, np.uint8),
44 "fp8_e5m2": DTypeInfo(torch.float8_e5m2, torch.uint8, np.uint8),
45 }
46 +MM_METADATA_DTYPES: Mapping[MmMetadataDType, DTypeInfo] = {
47 + "int32": DTypeInfo(torch.int32, torch.int32, np.int32),
48 + "int64": DTypeInfo(torch.int64, torch.int64, np.int64),
49 + "uint8": DTypeInfo(torch.uint8, torch.uint8, np.uint8),
50 + "bool": DTypeInfo(torch.bool, torch.uint8, np.uint8),
51 +}
52 +_ALL_SERIAL_DTYPES: Mapping[str, DTypeInfo] = {
53 + k: v for d in (EMBED_DTYPES, MM_METADATA_DTYPES) for k, v in d.items()
54 +}
55 ENDIANNESS: tuple[Endianness, ...] = get_args(Endianness)
56
57
@@ -56,14 +66,14 @@ def tensor2base64(x: torch.Tensor) -> str:
66
67 def tensor2binary(
68 tensor: torch.Tensor,
59 - embed_dtype: EmbedDType,
69 + embed_dtype: "EmbedDType | MmMetadataDType",
70 endianness: Endianness,
71 ) -> bytes:
72 assert isinstance(tensor, torch.Tensor)
63 - assert embed_dtype in EMBED_DTYPES
73 + assert embed_dtype in _ALL_SERIAL_DTYPES
74 assert endianness in ENDIANNESS
75
66 - dtype_info = EMBED_DTYPES[embed_dtype]
76 + dtype_info = _ALL_SERIAL_DTYPES[embed_dtype]
77
78 np_array = (
79 tensor.to(dtype_info.torch_dtype)
@@ -82,13 +92,13 @@ def tensor2binary(
92 def binary2tensor(
93 binary: bytes,
94 shape: tuple[int, ...],
85 - embed_dtype: EmbedDType,
95 + embed_dtype: "EmbedDType | MmMetadataDType",
96 endianness: Endianness,
97 ) -> torch.Tensor:
88 - assert embed_dtype in EMBED_DTYPES
98 + assert embed_dtype in _ALL_SERIAL_DTYPES
99 assert endianness in ENDIANNESS
100
91 - dtype_info = EMBED_DTYPES[embed_dtype]
101 + dtype_info = _ALL_SERIAL_DTYPES[embed_dtype]
102
103 np_array = np.frombuffer(binary, dtype=dtype_info.numpy_view_dtype).reshape(shape)
104