@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 68bc81703e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Frontend][Misc] Enforce Pixel Values as Input Type for VLMs in API Server (#5374)

Roger Wang committed Jun 10, 2024 at 02:13 UTC 68bc81703e2c41f38b1696269ce912cacb384e60
1 file changed +10
vllm/entrypoints/openai/api_server.py
+10
@@ -183,6 +183,16 @@ if __name__ == "__main__":
183 served_model_names = [args.model]
184
185 engine_args = AsyncEngineArgs.from_cli_args(args)
186 +
187 + # Enforce pixel values as image input type for vision language models
188 + # when serving with API server
189 + if engine_args.image_input_type is not None and \
190 + engine_args.image_input_type.upper() != "PIXEL_VALUES":
191 + raise ValueError(
192 + f"Invalid image_input_type: {engine_args.image_input_type}. "
193 + "Only --image-input-type 'pixel_values' is supported for serving "
194 + "vision language models with the vLLM API server.")
195 +
196 engine = AsyncLLMEngine.from_engine_args(
197 engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
198