A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 16:30 UTC
[Doc] Fix run_batch docs (#34056)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 7, 2026 at 22:18 UTC
b956cdf818dbb05f4dc327cc76b46c18a1e37cf8
1 file changed
+5
-2
vllm/entrypoints/openai/run_batch.py
+5
-2
@@ -565,12 +565,15 @@ async def run_request(
565
return batch_output
566
567
568
+WrapperFn: TypeAlias = Callable[[Callable], Callable]
569
+
570
+
571
def handle_endpoint_request(
572
request: BatchRequestInput,
573
tracker: BatchProgressTracker,
574
url_matcher: Callable[[str], bool],
575
handler_getter: Callable[[], Callable | None],
573
- wrapper_fn: Callable[[Callable], Callable] | None = None,
576
+ wrapper_fn: WrapperFn | None = None,
577
) -> Awaitable[BatchRequestOutput] | None:
578
"""
579
Generic handler for endpoint requests.
@@ -602,7 +605,7 @@ def handle_endpoint_request(
605
return run_request(handler_fn, request, tracker)
606
607
605
-def make_transcription_wrapper(is_translation: bool):
608
+def make_transcription_wrapper(is_translation: bool) -> WrapperFn:
609
"""
610
Factory function to create a wrapper for transcription/translation handlers.
611
The wrapper converts BatchTranscriptionRequest or BatchTranslationRequest