@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bd429f2b75
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] Priority-based scheduling in async engine (#8850)

Sebastian Schoennenbeck committed Sep 28, 2024 at 00:07 UTC bd429f2b75f3622fabaf9c9470ca2e921f6f56ca
2 files changed +24 -3
vllm/engine/async_llm_engine.py
+23 -2
@@ -420,6 +420,7 @@ class _AsyncLLMEngine(LLMEngine):
420 lora_request: Optional[LoRARequest] = None,
421 trace_headers: Optional[Mapping[str, str]] = None,
422 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
423 + priority: int = 0,
424 ) -> None:
425 ...
426
@@ -433,6 +434,7 @@ class _AsyncLLMEngine(LLMEngine):
434 lora_request: Optional[LoRARequest] = None,
435 trace_headers: Optional[Mapping[str, str]] = None,
436 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
437 + priority: int = 0,
438 ) -> None:
439 ...
440
@@ -449,6 +451,7 @@ class _AsyncLLMEngine(LLMEngine):
451 lora_request: Optional[LoRARequest] = None,
452 trace_headers: Optional[Mapping[str, str]] = None,
453 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
454 + priority: int = 0,
455 *,
456 inputs: Optional[PromptType] = None, # DEPRECATED
457 ) -> None:
@@ -460,6 +463,9 @@ class _AsyncLLMEngine(LLMEngine):
463 if lora_request is not None and not self.lora_config:
464 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
465 "not enabled!")
466 + if priority != 0 and not self.scheduler_config.policy == "priority":
467 + raise ValueError(f"Got priority {priority} but "
468 + "Priority scheduling is not enabled.")
469 if arrival_time is None:
470 arrival_time = time.time()
471
@@ -479,6 +485,7 @@ class _AsyncLLMEngine(LLMEngine):
485 lora_request=lora_request,
486 prompt_adapter_request=prompt_adapter_request,
487 trace_headers=trace_headers,
488 + priority=priority,
489 )
490
491 async def check_health_async(self) -> None:
@@ -829,6 +836,7 @@ class AsyncLLMEngine:
836 lora_request: Optional[LoRARequest] = None,
837 trace_headers: Optional[Mapping[str, str]] = None,
838 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
839 + priority: int = 0,
840 ) -> Coroutine[None, None, AsyncGenerator[Union[
841 RequestOutput, EmbeddingRequestOutput], None]]:
842 ...
@@ -843,6 +851,7 @@ class AsyncLLMEngine:
851 lora_request: Optional[LoRARequest] = None,
852 trace_headers: Optional[Mapping[str, str]] = None,
853 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
854 + priority: int = 0,
855 ) -> Coroutine[None, None, AsyncGenerator[Union[
856 RequestOutput, EmbeddingRequestOutput], None]]:
857 ...
@@ -860,6 +869,7 @@ class AsyncLLMEngine:
869 lora_request: Optional[LoRARequest] = None,
870 trace_headers: Optional[Mapping[str, str]] = None,
871 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
872 + priority: int = 0,
873 *,
874 inputs: Optional[PromptType] = None, # DEPRECATED
875 ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
@@ -877,6 +887,11 @@ class AsyncLLMEngine:
887 "error that caused the background loop to stop "
888 "(AsyncEngineDeadError).")
889
890 + if (priority != 0
891 + and not self.engine.scheduler_config.policy == "priority"):
892 + raise ValueError(f"Got priority {priority} but "
893 + "Priority scheduling is not enabled.")
894 +
895 stream = self._request_tracker.add_request(
896 request_id,
897 verbose=self.log_requests,
@@ -885,7 +900,9 @@ class AsyncLLMEngine:
900 arrival_time=arrival_time or time.time(),
901 lora_request=lora_request,
902 trace_headers=trace_headers,
888 - prompt_adapter_request=prompt_adapter_request)
903 + prompt_adapter_request=prompt_adapter_request,
904 + priority=priority,
905 + )
906
907 return stream.generator()
908
@@ -896,7 +913,8 @@ class AsyncLLMEngine:
913 request_id: str,
914 lora_request: Optional[LoRARequest] = None,
915 trace_headers: Optional[Mapping[str, str]] = None,
899 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
916 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
917 + priority: int = 0,
918 ) -> AsyncGenerator[RequestOutput, None]:
919 """Generate outputs for a request.
920
@@ -913,6 +931,8 @@ class AsyncLLMEngine:
931 trace_headers: OpenTelemetry trace headers.
932 prompt_adapter_request: Prompt Adapter request to use
933 for generation, if any.
934 + priority: The priority of the request.
935 + Only applicable with priority scheduling.
936
937 Yields:
938 The output `RequestOutput` objects from the LLMEngine
@@ -968,6 +988,7 @@ class AsyncLLMEngine:
988 lora_request=lora_request,
989 trace_headers=trace_headers,
990 prompt_adapter_request=prompt_adapter_request,
991 + priority=priority,
992 ):
993 yield LLMEngine.validate_output(output, RequestOutput)
994
vllm/engine/llm_engine.py
+1 -1
@@ -796,7 +796,7 @@ class LLMEngine:
796 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
797 "not enabled!")
798
799 - if priority > 0 and not self.scheduler_config.policy == "priority":
799 + if priority != 0 and not self.scheduler_config.policy == "priority":
800 raise ValueError(f"Got priority {priority} but "
801 "Priority scheduling is not enabled.")
802