A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[docs][misc] IOProcessor plugins fixes (#24046)
Signed-off-by: Christian Pinto <christian.pinto@ibm.com>
Christian Pinto committed
Sep 1, 2025 at 17:17 UTC
cf91a89dd2b89f92e5877d0d7b4c32c70da6f3c1
6 files changed
+14
-23
docs/design/io_processor_plugins.md
+2
-2
@@ -64,9 +64,9 @@ The `parse_request` method is used for validating the user prompt and converting
64
The `pre_process*` methods take the validated plugin input to generate vLLM's model prompts for regular inference.
65
The `post_process*` methods take `PoolingRequestOutput` objects as input and generate a custom plugin output.
66
67
-The `output_to_response` method is used only for online serving and converts the plugin output to the `IOProcessorResponse` type that is then returned by the API Server. The implementation of the `/io_processor_pooling` serving endpoint is [here](../../vllm/entrypoints/openai/serving_pooling_with_io_plugin.py).
67
+The `output_to_response` method is used only for online serving and converts the plugin output to the `IOProcessorResponse` type that is then returned by the API Server. The implementation of the `/io_processor_pooling` serving endpoint is available here <gh-file:vllm/entrypoints/openai/serving_pooling_with_io_plugin.py>.
68
69
-An example implementation of a plugin that enables generating geotiff images with the PrithviGeospatialMAE model is available [here](https://github.com/christian-pinto/prithvi_io_processor_plugin). Please, also refer to our [online](../../examples/online_serving/prithvi_geospatial_mae.py) and [offline](../../examples/offline_inference/prithvi_geospatial_mae_io_processor.py) inference examples.
69
+An example implementation of a plugin that enables generating geotiff images with the PrithviGeospatialMAE model is available [here](https://github.com/christian-pinto/prithvi_io_processor_plugin). Please, also refer to our online (<gh-file:examples/online_serving/prithvi_geospatial_mae.py>) and offline (<gh-file:examples/offline_inference/prithvi_geospatial_mae_io_processor.py>) inference examples.
70
71
## Using an IO Processor plugin
72
examples/online_serving/prithvi_geospatial_mae.py
+1
@@ -33,6 +33,7 @@ def main():
33
},
34
"priority": 0,
35
"model": "christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM",
36
+ "softmax": False,
37
}
38
39
ret = requests.post(server_endpoint, json=request_payload_url)
tests/plugins/prithvi_io_processor_plugin/prithvi_io_processor/prithvi_processor.py
+1
-18
@@ -8,7 +8,7 @@ import datetime
8
import os
9
import tempfile
10
import urllib.request
11
-from collections.abc import AsyncGenerator, Sequence
11
+from collections.abc import Sequence
12
from typing import Any, Optional, Union
13
14
import albumentations
@@ -359,14 +359,6 @@ class PrithviMultimodalDataProcessor(IOProcessor):
359
360
return prompts
361
362
- async def pre_process_async(
363
- self,
364
- prompt: IOProcessorInput,
365
- request_id: Optional[str] = None,
366
- **kwargs,
367
- ) -> Union[PromptType, Sequence[PromptType]]:
368
- return self.pre_process(prompt, request_id, **kwargs)
369
-
362
def post_process(
363
self,
364
model_output: Sequence[PoolingRequestOutput],
@@ -421,15 +413,6 @@ class PrithviMultimodalDataProcessor(IOProcessor):
413
data=out_data,
414
request_id=request_id)
415
424
- async def post_process_async(
425
- self,
426
- model_output: AsyncGenerator[tuple[int, PoolingRequestOutput]],
427
- request_id: Optional[str] = None,
428
- **kwargs,
429
- ) -> IOProcessorOutput:
430
- collected_output = [item async for i, item in model_output]
431
- return self.post_process(collected_output, request_id, **kwargs)
432
-
416
417
class PrithviMultimodalDataProcessorIndia(PrithviMultimodalDataProcessor):
418
tests/plugins_tests/test_io_processor_plugins.py
+1
@@ -113,6 +113,7 @@ async def test_prithvi_mae_plugin_online(
113
},
114
"priority": 0,
115
"model": model_name,
116
+ "softmax": False
117
}
118
119
ret = requests.post(
vllm/entrypoints/openai/protocol.py
+2
-1
@@ -1424,9 +1424,10 @@ class IOProcessorRequest(OpenAIBaseModel, Generic[T]):
1424
When using plugins IOProcessor plugins, the actual input is processed
1425
by the plugin itself. Hence, we use a generic type for the request data
1426
"""
1427
+ softmax: bool = True
1428
1429
def to_pooling_params(self):
1429
- return PoolingParams(task="encode")
1430
+ return PoolingParams(task="encode", softmax=self.softmax)
1431
1432
1433
class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
vllm/plugins/io_processors/interface.py
+7
-2
@@ -49,7 +49,12 @@ class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):
49
request_id: Optional[str] = None,
50
**kwargs,
51
) -> IOProcessorOutput:
52
- collected_output = [item async for i, item in model_output]
52
+ # We cannot guarantee outputs are returned in the same order they were
53
+ # fed to vLLM.
54
+ # Let's sort them by id before post_processing
55
+ sorted_output = sorted([(i, item) async for i, item in model_output],
56
+ key=lambda output: output[0])
57
+ collected_output = [output[1] for output in sorted_output]
58
return self.post_process(collected_output, request_id, **kwargs)
59
60
@abstractmethod
@@ -59,4 +64,4 @@ class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):
64
@abstractmethod
65
def output_to_response(
66
self, plugin_output: IOProcessorOutput) -> IOProcessorResponse:
62
- raise NotImplementedError
\ No newline at end of file
67
+ raise NotImplementedError