A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Frontend][Misc] Enforce Pixel Values as Input Type for VLMs in API Server (#5374)
Roger Wang committed
Jun 10, 2024 at 02:13 UTC
68bc81703e2c41f38b1696269ce912cacb384e60
1 file changed
+10
vllm/entrypoints/openai/api_server.py
+10
@@ -183,6 +183,16 @@ if __name__ == "__main__":
183
served_model_names = [args.model]
184
185
engine_args = AsyncEngineArgs.from_cli_args(args)
186
+
187
+ # Enforce pixel values as image input type for vision language models
188
+ # when serving with API server
189
+ if engine_args.image_input_type is not None and \
190
+ engine_args.image_input_type.upper() != "PIXEL_VALUES":
191
+ raise ValueError(
192
+ f"Invalid image_input_type: {engine_args.image_input_type}. "
193
+ "Only --image-input-type 'pixel_values' is supported for serving "
194
+ "vision language models with the vLLM API server.")
195
+
196
engine = AsyncLLMEngine.from_engine_args(
197
engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
198