@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8e75d88554
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 12:45 UTC

add io_process_plugin for sparse embedding (#34214)

Signed-off-by: augusto.yjh <augusto.yjh@antgroup.com> Signed-off-by: Augusto Yao <augusto.yjh@antgroup.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Augusto Yao committed Feb 28, 2026 at 17:16 UTC 8e75d885544c9d7602344e9db2c7e3cff9b73c11
14 files changed +441 -10
.buildkite/test-amd.yaml
+9 -1
@@ -1390,6 +1390,10 @@ steps:
1390 - pip install -e ./plugins/prithvi_io_processor_plugin
1391 - pytest -v -s plugins_tests/test_io_processor_plugins.py
1392 - pip uninstall prithvi_io_processor_plugin -y
1393 + # test bge_m3_sparse io_processor plugin
1394 + - pip install -e ./plugins/bge_m3_sparse_plugin
1395 + - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
1396 + - pip uninstall bge_m3_sparse_plugin -y
1397 # end io_processor plugins test
1398 # begin stat_logger plugins test
1399 - pip install -e ./plugins/vllm_add_dummy_stat_logger
@@ -2967,6 +2971,10 @@ steps:
2971 - pip install -e ./plugins/prithvi_io_processor_plugin
2972 - pytest -v -s plugins_tests/test_io_processor_plugins.py
2973 - pip uninstall prithvi_io_processor_plugin -y
2974 + # test bge_m3_sparse io_processor plugin
2975 + - pip install -e ./plugins/bge_m3_sparse_plugin
2976 + - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
2977 + - pip uninstall bge_m3_sparse_plugin -y
2978 # end io_processor plugins test
2979 # begin stat_logger plugins test
2980 - pip install -e ./plugins/vllm_add_dummy_stat_logger
@@ -3248,4 +3256,4 @@ steps:
3256 num_gpus: 4
3257 working_dir: "/vllm-workspace"
3258 commands:
3251 - - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
\ No newline at end of file
3259 + - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
.buildkite/test_areas/plugins.yaml
+4
@@ -19,6 +19,10 @@ steps:
19 - pip install -e ./plugins/prithvi_io_processor_plugin
20 - pytest -v -s plugins_tests/test_io_processor_plugins.py
21 - pip uninstall prithvi_io_processor_plugin -y
22 + # test bge_m3_sparse io_processor plugin
23 + - pip install -e ./plugins/bge_m3_sparse_plugin
24 + - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
25 + - pip uninstall bge_m3_sparse_plugin -y
26 # end io_processor plugins test
27 # begin stat_logger plugins test
28 - pip install -e ./plugins/vllm_add_dummy_stat_logger
docs/design/io_processor_plugins.md
+4 -3
@@ -13,12 +13,13 @@ IOProcessorInput = TypeVar("IOProcessorInput")
13 IOProcessorOutput = TypeVar("IOProcessorOutput")
14
15 class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):
16 - def __init__(self, vllm_config: VllmConfig):
16 + """Abstract interface for pre/post-processing of engine I/O."""
17 +
18 + def __init__(self, vllm_config: VllmConfig, renderer: BaseRenderer):
19 super().__init__()
20
21 self.vllm_config = vllm_config
22
21 - @abstractmethod
23 def parse_data(self, data: object) -> IOProcessorInput:
24 raise NotImplementedError
25
@@ -32,7 +33,7 @@ class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):
33 self,
34 params: PoolingParams | None = None,
35 ) -> PoolingParams:
35 - return params or PoolingParams()
36 + return params or PoolingParams(task="plugin")
37
38 @abstractmethod
39 def pre_process(
tests/plugins/bge_m3_sparse_plugin/bge_m3_sparse_processor/__init__.py new
+6
@@ -0,0 +1,6 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +
5 +def register_bge_m3_sparse_embeddings_processor():
6 + return "bge_m3_sparse_processor.sparse_embeddings_processor.BgeM3SparseEmbeddingsProcessor" # noqa: E501
tests/plugins/bge_m3_sparse_plugin/bge_m3_sparse_processor/sparse_embeddings_processor.py new
+135
@@ -0,0 +1,135 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Sequence
5 +
6 +from vllm.config import VllmConfig
7 +from vllm.entrypoints.openai.engine.protocol import UsageInfo
8 +from vllm.inputs.data import PromptType
9 +from vllm.logger import init_logger
10 +from vllm.outputs import PoolingRequestOutput
11 +from vllm.plugins.io_processors.interface import (
12 + IOProcessor,
13 +)
14 +from vllm.pooling_params import PoolingParams
15 +from vllm.renderers import BaseRenderer
16 +from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
17 +
18 +from .types import (
19 + SparseEmbeddingCompletionRequestMixin,
20 + SparseEmbeddingResponse,
21 + SparseEmbeddingResponseData,
22 + SparseEmbeddingTokenWeight,
23 +)
24 +
25 +logger = init_logger(__name__)
26 +
27 +
28 +class BgeM3SparseEmbeddingsProcessor(
29 + IOProcessor[SparseEmbeddingCompletionRequestMixin, SparseEmbeddingResponse]
30 +):
31 + def __init__(self, vllm_config: VllmConfig, renderer: BaseRenderer):
32 + super().__init__(vllm_config, renderer)
33 + self.offline_requests: list[SparseEmbeddingCompletionRequestMixin] = []
34 + self.online_requests: dict[str, SparseEmbeddingCompletionRequestMixin] = {}
35 + self.renderer: BaseRenderer = renderer
36 +
37 + def merge_pooling_params(
38 + self,
39 + params: PoolingParams | None = None,
40 + ) -> PoolingParams:
41 + if params is None:
42 + params = PoolingParams()
43 + # refer to PoolingCompletionRequest.to_pooling_params
44 + params.task = "token_classify"
45 + return params
46 +
47 + def parse_request(
48 + self, request_data: object
49 + ) -> SparseEmbeddingCompletionRequestMixin:
50 + # for vllm.entrypoints.llm.LLM, offline mode, calls `encode` directly.
51 + if isinstance(request_data, dict):
52 + return SparseEmbeddingCompletionRequestMixin(**request_data)
53 + raise TypeError("request_data should be a dictionary")
54 +
55 + def pre_process(
56 + self,
57 + prompt: SparseEmbeddingCompletionRequestMixin,
58 + request_id: str | None = None,
59 + **kwargs,
60 + ) -> PromptType | Sequence[PromptType]:
61 + if request_id is not None:
62 + assert request_id not in self.online_requests, "request_id duplicated"
63 + self.online_requests[request_id] = prompt
64 + else:
65 + self.offline_requests.append(prompt)
66 + return prompt.input
67 +
68 + def _get_sparse_embedding_request(self, request_id: str | None = None):
69 + if request_id:
70 + return self.online_requests.pop(request_id, None)
71 + return self.offline_requests.pop()
72 +
73 + def _build_sparse_embedding_token_weights(
74 + self,
75 + sparse_embedding: dict[int, float],
76 + return_tokens: bool = False,
77 + ) -> list[SparseEmbeddingTokenWeight]:
78 + token_ids = sparse_embedding.keys()
79 + token_weights = sparse_embedding.values()
80 + tokens = [None] * len(token_ids)
81 +
82 + if return_tokens and self.renderer is not None:
83 + tokens = convert_ids_list_to_tokens(
84 + self.renderer.get_tokenizer(), token_ids
85 + )
86 + sparse_embedding_output: list[SparseEmbeddingTokenWeight] = []
87 + for token_id, weight, token in zip(token_ids, token_weights, tokens):
88 + sparse_embedding_output.append(
89 + SparseEmbeddingTokenWeight(
90 + token_id=token_id, weight=weight, token=token
91 + )
92 + )
93 + return sparse_embedding_output
94 +
95 + def post_process(
96 + self,
97 + model_output: Sequence[PoolingRequestOutput],
98 + request_id: str | None = None,
99 + **kwargs,
100 + ) -> SparseEmbeddingResponse:
101 + num_prompt_tokens = 0
102 + response_data = []
103 + return_tokens = self._get_sparse_embedding_request(request_id).return_tokens
104 + for idx in range(len(model_output)):
105 + mo = model_output[idx]
106 + sparse_embedding: dict[int, float] = {}
107 + num_prompt_tokens += len(mo.prompt_token_ids)
108 + if len(mo.prompt_token_ids) != len(mo.outputs.data):
109 + # this is the case that add_special_tokens is True,
110 + # which means first token and last token are special tokens
111 + mo.prompt_token_ids = mo.prompt_token_ids[1:]
112 + for token_id, weight in zip(mo.prompt_token_ids, mo.outputs.data.tolist()):
113 + sparse_embedding[token_id] = max(
114 + weight, sparse_embedding.get(token_id, 0.0)
115 + )
116 + response_data.append(
117 + SparseEmbeddingResponseData(
118 + index=idx,
119 + sparse_embedding=self._build_sparse_embedding_token_weights(
120 + sparse_embedding,
121 + return_tokens,
122 + ),
123 + )
124 + )
125 +
126 + usage = UsageInfo(
127 + prompt_tokens=num_prompt_tokens,
128 + total_tokens=num_prompt_tokens,
129 + )
130 + resp = SparseEmbeddingResponse(
131 + data=response_data,
132 + usage=usage,
133 + )
134 +
135 + return resp
tests/plugins/bge_m3_sparse_plugin/bge_m3_sparse_processor/types.py new
+32
@@ -0,0 +1,32 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from pydantic import BaseModel, Field
5 +
6 +from vllm.entrypoints.openai.engine.protocol import UsageInfo
7 +from vllm.entrypoints.pooling.base.protocol import CompletionRequestMixin
8 +
9 +
10 +class SparseEmbeddingCompletionRequestMixin(CompletionRequestMixin):
11 + return_tokens: bool | None = Field(
12 + default=None,
13 + description="Whether to return dict shows the mapping of token_id to text."
14 + "`None` or False means not return.",
15 + )
16 +
17 +
18 +class SparseEmbeddingTokenWeight(BaseModel):
19 + token_id: int
20 + weight: float
21 + token: str | None
22 +
23 +
24 +class SparseEmbeddingResponseData(BaseModel):
25 + index: int
26 + object: str = "sparse-embedding"
27 + sparse_embedding: list[SparseEmbeddingTokenWeight]
28 +
29 +
30 +class SparseEmbeddingResponse(BaseModel):
31 + data: list[SparseEmbeddingResponseData]
32 + usage: UsageInfo
tests/plugins/bge_m3_sparse_plugin/setup.py new
+15
@@ -0,0 +1,15 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from setuptools import setup
5 +
6 +setup(
7 + name="bge-m3-sparse-plugin",
8 + version="0.1",
9 + packages=["bge_m3_sparse_processor"],
10 + entry_points={
11 + "vllm.io_processor_plugins": [
12 + "bge_m3_sparse_plugin = bge_m3_sparse_processor:register_bge_m3_sparse_embeddings_processor", # noqa: E501
13 + ]
14 + },
15 +)
tests/plugins/prithvi_io_processor_plugin/prithvi_io_processor/prithvi_processor.py
+3 -2
@@ -22,6 +22,7 @@ from vllm.inputs.data import PromptType
22 from vllm.logger import init_logger
23 from vllm.outputs import PoolingRequestOutput
24 from vllm.plugins.io_processors.interface import IOProcessor
25 +from vllm.renderers import BaseRenderer
26
27 from .types import DataModuleConfig, ImagePrompt, ImageRequestOutput
28
@@ -218,8 +219,8 @@ def load_image(
219 class PrithviMultimodalDataProcessor(IOProcessor[ImagePrompt, ImageRequestOutput]):
220 indices = [0, 1, 2, 3, 4, 5]
221
221 - def __init__(self, vllm_config: VllmConfig):
222 - super().__init__(vllm_config)
222 + def __init__(self, vllm_config: VllmConfig, renderer: BaseRenderer):
223 + super().__init__(vllm_config, renderer)
224
225 self.datamodule = Sen1Floods11NonGeoDataModule(
226 data_root=datamodule_config["data_root"],
tests/plugins_tests/test_bge_m3_sparse_io_processor_plugins.py new
+212
@@ -0,0 +1,212 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +
6 +import pytest
7 +import requests
8 +
9 +# Test configuration for BGE-M3 sparse plugin
10 +from tests.utils import RemoteOpenAIServer
11 +from vllm.entrypoints.pooling.pooling.protocol import IOProcessorResponse
12 +
13 +model_config = {
14 + "model_name": "BAAI/bge-m3",
15 + "plugin": "bge_m3_sparse_plugin",
16 + "test_input": "What is the capital of France?",
17 + "hf_overrides": json.dumps(
18 + {"architectures": ["BgeM3EmbeddingModel"], "head_dtype": "float16"}
19 + ),
20 +}
21 +
22 +
23 +def _float_close(expected: object, result: object):
24 + assert isinstance(expected, float) and isinstance(result, float), (
25 + f"{expected=} or {result=} is not float"
26 + )
27 + return (expected - result) < 1e-3 or abs(expected / result - 1) < 1e-3
28 +
29 +
30 +def _get_attr_or_val(obj: object | dict, key: str):
31 + if isinstance(obj, dict) and key in obj:
32 + return obj[key]
33 + return getattr(obj, key, None)
34 +
35 +
36 +def _check_sparse_embedding(data, check_tokens=False):
37 + expected_weights = [
38 + {"token_id": 32, "weight": 0.0552978515625, "token": "?"},
39 + {"token_id": 70, "weight": 0.09808349609375, "token": "the"},
40 + {"token_id": 83, "weight": 0.08154296875, "token": "is"},
41 + {"token_id": 111, "weight": 0.11810302734375, "token": "of"},
42 + {"token_id": 4865, "weight": 0.1171875, "token": "What"},
43 + {"token_id": 9942, "weight": 0.292236328125, "token": "France"},
44 + {"token_id": 10323, "weight": 0.2802734375, "token": "capital"},
45 + ]
46 + expected_embed = {x["token_id"]: x for x in expected_weights}
47 +
48 + assert len(data) == len(expected_embed)
49 + for entry in data:
50 + expected_val = expected_embed[_get_attr_or_val(entry, "token_id")]
51 + assert _float_close(
52 + expected_val["weight"], _get_attr_or_val(entry, "weight")
53 + ), f"actual embed {entry} not equal to {expected_val}"
54 + if check_tokens:
55 + assert expected_val["token"] == _get_attr_or_val(entry, "token"), (
56 + f"actual embed {entry} not equal to {expected_val}"
57 + )
58 + else:
59 + assert _get_attr_or_val(entry, "token") is None, (
60 + f"{entry} should not return token"
61 + )
62 +
63 +
64 +@pytest.fixture(scope="function")
65 +def server():
66 + args = [
67 + "--runner",
68 + "pooling",
69 + "--enforce-eager",
70 + "--max-num-seqs",
71 + "32",
72 + "--hf_overrides",
73 + model_config["hf_overrides"],
74 + "--io-processor-plugin",
75 + model_config["plugin"],
76 + ]
77 +
78 + with RemoteOpenAIServer(model_config["model_name"], args) as remote_server:
79 + yield remote_server
80 +
81 +
82 +@pytest.mark.asyncio
83 +@pytest.mark.parametrize(
84 + "return_tokens",
85 + [True, False],
86 +)
87 +async def test_bge_m3_sparse_plugin_online(
88 + server: RemoteOpenAIServer, return_tokens: bool
89 +):
90 + """Test BGE-M3 sparse plugin in online mode via API."""
91 + request_payload = {
92 + "model": model_config["model_name"],
93 + "task": "token_classify",
94 + "data": {"input": model_config["test_input"], "return_tokens": return_tokens},
95 + }
96 +
97 + ret = requests.post(
98 + server.url_for("pooling"),
99 + json=request_payload,
100 + )
101 +
102 + response = ret.json()
103 +
104 + # Verify the request response is in the correct format
105 + assert (parsed_response := IOProcessorResponse(**response).data)
106 +
107 + # Verify the output is formatted as expected for this plugin
108 + assert _get_attr_or_val(parsed_response, "data")
109 + assert len(_get_attr_or_val(parsed_response, "data")) > 0
110 +
111 + data_entry = _get_attr_or_val(parsed_response, "data")[0]
112 + assert _get_attr_or_val(data_entry, "object") == "sparse-embedding"
113 + assert _get_attr_or_val(data_entry, "sparse_embedding")
114 +
115 + # Verify sparse embedding format
116 + sparse_embedding = _get_attr_or_val(data_entry, "sparse_embedding")
117 + assert isinstance(sparse_embedding, list)
118 + _check_sparse_embedding(sparse_embedding, return_tokens)
119 +
120 + # Verify usage information
121 + usage = _get_attr_or_val(parsed_response, "usage")
122 + assert usage, f"usage not found for {parsed_response}"
123 + assert _get_attr_or_val(usage, "prompt_tokens") > 0
124 + assert _get_attr_or_val(usage, "total_tokens") == _get_attr_or_val(
125 + usage, "prompt_tokens"
126 + )
127 +
128 +
129 +@pytest.mark.parametrize(
130 + "return_tokens",
131 + [True, False],
132 +)
133 +def test_bge_m3_sparse_plugin_offline(vllm_runner, return_tokens: bool):
134 + """Test BGE-M3 sparse plugin in offline mode."""
135 + prompt = {
136 + "data": {
137 + "input": model_config["test_input"],
138 + "return_tokens": return_tokens,
139 + }
140 + }
141 +
142 + with vllm_runner(
143 + model_config["model_name"],
144 + runner="pooling",
145 + enforce_eager=True,
146 + max_num_seqs=32,
147 + io_processor_plugin=model_config["plugin"],
148 + hf_overrides=json.loads(model_config["hf_overrides"]),
149 + default_torch_num_threads=1,
150 + ) as llm_runner:
151 + llm = llm_runner.get_llm()
152 + pooler_output = llm.encode(prompt, pooling_task="token_classify")
153 +
154 + outputs = pooler_output[0]
155 +
156 + # Verify output structure
157 + assert hasattr(outputs, "outputs")
158 + response = outputs.outputs
159 + assert hasattr(response, "data")
160 + assert len(response.data) == 1
161 + # Verify response data
162 + for i, output in enumerate(response.data):
163 + # Each output should have sparse embeddings
164 + sparse_embedding = output.sparse_embedding
165 + assert isinstance(sparse_embedding, list)
166 + _check_sparse_embedding(sparse_embedding, return_tokens)
167 +
168 + # Verify usage
169 + assert response.usage.prompt_tokens > 0
170 + assert response.usage.total_tokens == response.usage.prompt_tokens
171 +
172 +
173 +def test_bge_m3_sparse_plugin_offline_multiple_inputs(vllm_runner):
174 + """Test BGE-M3 sparse plugin with multiple inputs in offline mode."""
175 + prompts = {
176 + "data": {
177 + "input": [
178 + "What is the capital of France?",
179 + "What is the capital of Germany?",
180 + "What is the capital of Spain?",
181 + ],
182 + "return_tokens": True,
183 + }
184 + }
185 +
186 + with vllm_runner(
187 + model_config["model_name"],
188 + runner="pooling",
189 + enforce_eager=True,
190 + max_num_seqs=32,
191 + io_processor_plugin=model_config["plugin"],
192 + hf_overrides=json.loads(model_config["hf_overrides"]),
193 + default_torch_num_threads=1,
194 + ) as llm_runner:
195 + llm = llm_runner.get_llm()
196 + pooler_output = llm.encode(prompts, pooling_task="token_classify")
197 +
198 + outputs = pooler_output[0]
199 +
200 + # Verify output structure
201 + assert hasattr(outputs, "outputs")
202 + response = outputs.outputs
203 + assert hasattr(response, "data")
204 + assert len(response.data) == 3
205 + for i, output in enumerate(response.data):
206 + # Each output should have sparse embeddings
207 + sparse_embedding = output.sparse_embedding
208 + assert isinstance(sparse_embedding, list)
209 +
210 + # Verify usage
211 + assert response.usage.prompt_tokens > 0
212 + assert response.usage.total_tokens == response.usage.prompt_tokens
tests/plugins_tests/test_io_processor_plugins.py
+1 -1
@@ -39,7 +39,7 @@ def _compute_image_hash(base64_data: str) -> str:
39 def test_loading_missing_plugin():
40 vllm_config = VllmConfig()
41 with pytest.raises(ValueError):
42 - get_io_processor(vllm_config, "wrong_plugin")
42 + get_io_processor(vllm_config, None, "wrong_plugin")
43
44
45 @pytest.fixture(scope="function")
vllm/plugins/io_processors/__init__.py
+16 -2
@@ -1,18 +1,22 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +import inspect
5 import logging
6
7 from vllm.config import VllmConfig
8 from vllm.plugins import IO_PROCESSOR_PLUGINS_GROUP, load_plugins_by_group
9 from vllm.plugins.io_processors.interface import IOProcessor
10 +from vllm.renderers import BaseRenderer
11 from vllm.utils.import_utils import resolve_obj_by_qualname
12
13 logger = logging.getLogger(__name__)
14
15
16 def get_io_processor(
15 - vllm_config: VllmConfig, plugin_from_init: str | None = None
17 + vllm_config: VllmConfig,
18 + renderer: BaseRenderer,
19 + plugin_from_init: str | None = None,
20 ) -> IOProcessor | None:
21 # Input.Output processors are loaded as plugins under the
22 # 'vllm.io_processor_plugins' group. Similar to platform
@@ -65,4 +69,14 @@ def get_io_processor(
69
70 activated_plugin_cls = loadable_plugins[model_plugin]
71
68 - return resolve_obj_by_qualname(activated_plugin_cls)(vllm_config)
72 + activated_plugin_typ = resolve_obj_by_qualname(activated_plugin_cls)
73 +
74 + # for backward compatibility, the plugin does not have a renderer argument
75 + if "renderer" not in inspect.signature(activated_plugin_typ.__init__).parameters:
76 + logger.warning(
77 + "The renderer argument will be required in v0.18, "
78 + "please update your IOProcessor plugin: %s",
79 + activated_plugin_cls,
80 + )
81 + return activated_plugin_typ(vllm_config)
82 + return activated_plugin_typ(vllm_config, renderer)
vllm/plugins/io_processors/interface.py
+2 -1
@@ -9,6 +9,7 @@ from vllm.config import VllmConfig
9 from vllm.inputs.data import PromptType
10 from vllm.outputs import PoolingRequestOutput
11 from vllm.pooling_params import PoolingParams
12 +from vllm.renderers import BaseRenderer
13 from vllm.sampling_params import SamplingParams
14
15 IOProcessorInput = TypeVar("IOProcessorInput")
@@ -18,7 +19,7 @@ IOProcessorOutput = TypeVar("IOProcessorOutput")
19 class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):
20 """Abstract interface for pre/post-processing of engine I/O."""
21
21 - def __init__(self, vllm_config: VllmConfig):
22 + def __init__(self, vllm_config: VllmConfig, renderer: BaseRenderer):
23 super().__init__()
24
25 self.vllm_config = vllm_config
vllm/v1/engine/async_llm.py
+1
@@ -135,6 +135,7 @@ class AsyncLLM(EngineClient):
135 self.renderer = renderer = renderer_from_config(self.vllm_config)
136 self.io_processor = get_io_processor(
137 self.vllm_config,
138 + self.renderer,
139 self.model_config.io_processor_plugin,
140 )
141
vllm/v1/engine/llm_engine.py
+1
@@ -92,6 +92,7 @@ class LLMEngine:
92 self.renderer = renderer = renderer_from_config(self.vllm_config)
93 self.io_processor = get_io_processor(
94 self.vllm_config,
95 + self.renderer,
96 self.model_config.io_processor_plugin,
97 )
98