A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 05:00 UTC
(bugfix): Fixed encode in LLM entrypoint for IOProcessr plugin prompts (#34618)
Signed-off-by: Christian Pinto <christian.pinto@ibm.com>
Christian Pinto committed
Feb 16, 2026 at 15:33 UTC
6930becd453ab81dce074505521a41a397d0c727
3 files changed
+17
-5
examples/pooling/plugin/prithvi_geospatial_mae_io_processor.py
+4
-2
@@ -20,13 +20,15 @@ def main():
20
torch.set_default_dtype(torch.float16)
21
image_url = "https://huggingface.co/christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM/resolve/main/valencia_example_2024-10-26.tiff" # noqa: E501
22
23
- img_prompt = dict(
23
+ img_data = dict(
24
data=image_url,
25
data_format="url",
26
image_format="tiff",
27
out_data_format="b64_json",
28
)
29
30
+ prompt = dict(data=img_data)
31
+
32
llm = LLM(
33
model="ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
34
skip_tokenizer_init=True,
@@ -41,7 +43,7 @@ def main():
43
enable_mm_embeds=True,
44
)
45
44
- pooler_output = llm.encode(img_prompt, pooling_task="plugin")
46
+ pooler_output = llm.encode(prompt, pooling_task="plugin")
47
output = pooler_output[0].outputs
48
49
print(output)
tests/plugins_tests/test_io_processor_plugins.py
+4
-2
@@ -120,13 +120,15 @@ async def test_prithvi_mae_plugin_online(
120
def test_prithvi_mae_plugin_offline(
121
vllm_runner, model_name: str, image_url: str | dict, plugin: str, expected_hash: str
122
):
123
- img_prompt = dict(
123
+ img_data = dict(
124
data=image_url,
125
data_format="url",
126
image_format="tiff",
127
out_data_format="b64_json",
128
)
129
130
+ prompt = dict(data=img_data)
131
+
132
with vllm_runner(
133
model_name,
134
runner="pooling",
@@ -139,7 +141,7 @@ def test_prithvi_mae_plugin_offline(
141
io_processor_plugin=plugin,
142
default_torch_num_threads=1,
143
) as llm_runner:
142
- pooler_output = llm_runner.get_llm().encode(img_prompt, pooling_task="plugin")
144
+ pooler_output = llm_runner.get_llm().encode(prompt, pooling_task="plugin")
145
output = pooler_output[0].outputs
146
147
# verify the output is formatted as expected for this plugin
vllm/entrypoints/llm.py
+9
-1
@@ -1135,7 +1135,15 @@ class LLM:
1135
)
1136
1137
# Validate the request data is valid for the loaded plugin
1138
- validated_prompt = self.io_processor.parse_data(prompts)
1138
+ prompt_data = prompts.get("data")
1139
+ if prompt_data is None:
1140
+ raise ValueError(
1141
+ "The 'data' field of the prompt is expected to contain "
1142
+ "the prompt data and it cannot be None. "
1143
+ "Refer to the documentation of the IOProcessor "
1144
+ "in use for more details."
1145
+ )
1146
+ validated_prompt = self.io_processor.parse_data(prompt_data)
1147
1148
# obtain the actual model prompts from the pre-processor
1149
prompts = self.io_processor.pre_process(prompt=validated_prompt)