A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Benchmark] Support Infinity API (#26641)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 12, 2025 at 01:45 UTC
5be7ca1b99c91751c515371298d2e9c3ff21941c
2 files changed
+96
-29
vllm/benchmarks/datasets.py
+1
-1
@@ -1584,7 +1584,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:
1584
1585
if dataset_class.IS_MULTIMODAL and not (
1586
args.backend in ("openai-chat", "openai-audio")
1587
- or "openai-embeddings-" in args.backend
1587
+ or "embeddings-" in args.backend
1588
):
1589
# multi-modal benchmark is only available on OpenAI Chat
1590
# endpoint-type.
vllm/benchmarks/lib/endpoint_request_func.py
+95
-28
@@ -581,29 +581,6 @@ async def async_request_openai_embeddings_chat(
581
)
582
583
584
-async def async_request_openai_embeddings_clip(
585
- request_func_input: RequestFuncInput,
586
- session: aiohttp.ClientSession,
587
- pbar: Optional[tqdm] = None,
588
-) -> RequestFuncOutput:
589
- if request_func_input.multi_modal_content:
590
- # Image input
591
- request_func_input.prompt = ""
592
-
593
- # max_model_len=77 is too short for most datasets,
594
- # so by default we truncate the prompt to max_model_len
595
- if request_func_input.extra_body is None:
596
- request_func_input.extra_body = {}
597
- if "truncate_prompt_tokens" not in request_func_input.extra_body:
598
- request_func_input.extra_body["truncate_prompt_tokens"] = -1
599
-
600
- return await async_request_openai_embeddings_chat(
601
- request_func_input,
602
- session,
603
- pbar=pbar,
604
- )
605
-
606
-
584
def _try_extract_request_idx(request_func_input: RequestFuncInput):
585
if request_func_input.request_id:
586
match = re.search(r"(\d+)$", request_func_input.request_id)
@@ -616,11 +593,20 @@ def _try_extract_request_idx(request_func_input: RequestFuncInput):
593
return None
594
595
619
-async def async_request_openai_embeddings_vlm2vec(
620
- request_func_input: RequestFuncInput,
621
- session: aiohttp.ClientSession,
622
- pbar: Optional[tqdm] = None,
623
-) -> RequestFuncOutput:
596
+def _preprocess_clip(request_func_input: RequestFuncInput):
597
+ if request_func_input.multi_modal_content:
598
+ # Image input
599
+ request_func_input.prompt = ""
600
+
601
+ # max_model_len=77 is too short for most datasets,
602
+ # so by default we truncate the prompt to max_model_len
603
+ if request_func_input.extra_body is None:
604
+ request_func_input.extra_body = {}
605
+ if "truncate_prompt_tokens" not in request_func_input.extra_body:
606
+ request_func_input.extra_body["truncate_prompt_tokens"] = -1
607
+
608
+
609
+def _preprocess_vlm2vec(request_func_input: RequestFuncInput):
610
if request_func_input.multi_modal_content:
611
request_idx = _try_extract_request_idx(request_func_input)
612
@@ -637,6 +623,28 @@ async def async_request_openai_embeddings_vlm2vec(
623
f"{request_func_input.prompt}"
624
)
625
626
+
627
+async def async_request_openai_embeddings_clip(
628
+ request_func_input: RequestFuncInput,
629
+ session: aiohttp.ClientSession,
630
+ pbar: Optional[tqdm] = None,
631
+) -> RequestFuncOutput:
632
+ _preprocess_clip(request_func_input)
633
+
634
+ return await async_request_openai_embeddings_chat(
635
+ request_func_input,
636
+ session,
637
+ pbar=pbar,
638
+ )
639
+
640
+
641
+async def async_request_openai_embeddings_vlm2vec(
642
+ request_func_input: RequestFuncInput,
643
+ session: aiohttp.ClientSession,
644
+ pbar: Optional[tqdm] = None,
645
+) -> RequestFuncOutput:
646
+ _preprocess_vlm2vec(request_func_input)
647
+
648
return await async_request_openai_embeddings_chat(
649
request_func_input,
650
session,
@@ -645,6 +653,61 @@ async def async_request_openai_embeddings_vlm2vec(
653
)
654
655
656
+async def async_request_infinity_embeddings(
657
+ request_func_input: RequestFuncInput,
658
+ session: aiohttp.ClientSession,
659
+ pbar: Optional[tqdm] = None,
660
+) -> RequestFuncOutput:
661
+ api_url = request_func_input.api_url
662
+ _validate_api_url(api_url, "Infinity Embeddings API", "embeddings")
663
+
664
+ payload = {
665
+ "model": request_func_input.model_name
666
+ if request_func_input.model_name
667
+ else request_func_input.model,
668
+ }
669
+
670
+ if request_func_input.prompt:
671
+ payload["input"] = request_func_input.prompt
672
+ else:
673
+ mm_content = request_func_input.multi_modal_content
674
+ assert isinstance(mm_content, dict)
675
+
676
+ mm_type = mm_content["type"]
677
+ payload["input"] = mm_content[mm_type]["url"]
678
+ payload["modality"] = mm_type.split("_", 1)[0]
679
+
680
+ _update_payload_common(payload, request_func_input)
681
+
682
+ headers = {
683
+ "Content-Type": "application/json",
684
+ "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
685
+ }
686
+ _update_headers_common(headers, request_func_input)
687
+
688
+ return await _run_openai_embeddings(
689
+ session,
690
+ api_url,
691
+ payload=payload,
692
+ headers=headers,
693
+ pbar=pbar,
694
+ )
695
+
696
+
697
+async def async_request_infinity_embeddings_clip(
698
+ request_func_input: RequestFuncInput,
699
+ session: aiohttp.ClientSession,
700
+ pbar: Optional[tqdm] = None,
701
+) -> RequestFuncOutput:
702
+ _preprocess_clip(request_func_input)
703
+
704
+ return await async_request_infinity_embeddings(
705
+ request_func_input,
706
+ session,
707
+ pbar=pbar,
708
+ )
709
+
710
+
711
# TODO: Add more request functions for different API protocols.
712
ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = {
713
"vllm": async_request_openai_completions,
@@ -655,6 +718,10 @@ ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = {
718
"openai-embeddings-chat": async_request_openai_embeddings_chat,
719
"openai-embeddings-clip": async_request_openai_embeddings_clip,
720
"openai-embeddings-vlm2vec": async_request_openai_embeddings_vlm2vec,
721
+ # Infinity embedding server: https://github.com/michaelfeil/infinity
722
+ "infinity-embeddings": async_request_infinity_embeddings,
723
+ "infinity-embeddings-clip": async_request_infinity_embeddings_clip,
724
+ # (Infinity embedding server does not support vlm2vec)
725
}
726
727
OPENAI_COMPATIBLE_BACKENDS = [