A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] Priority-based scheduling in async engine (#8850)
Sebastian Schoennenbeck committed
Sep 28, 2024 at 00:07 UTC
bd429f2b75f3622fabaf9c9470ca2e921f6f56ca
2 files changed
+24
-3
vllm/engine/async_llm_engine.py
+23
-2
@@ -420,6 +420,7 @@ class _AsyncLLMEngine(LLMEngine):
420
lora_request: Optional[LoRARequest] = None,
421
trace_headers: Optional[Mapping[str, str]] = None,
422
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
423
+ priority: int = 0,
424
) -> None:
425
...
426
@@ -433,6 +434,7 @@ class _AsyncLLMEngine(LLMEngine):
434
lora_request: Optional[LoRARequest] = None,
435
trace_headers: Optional[Mapping[str, str]] = None,
436
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
437
+ priority: int = 0,
438
) -> None:
439
...
440
@@ -449,6 +451,7 @@ class _AsyncLLMEngine(LLMEngine):
451
lora_request: Optional[LoRARequest] = None,
452
trace_headers: Optional[Mapping[str, str]] = None,
453
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
454
+ priority: int = 0,
455
*,
456
inputs: Optional[PromptType] = None, # DEPRECATED
457
) -> None:
@@ -460,6 +463,9 @@ class _AsyncLLMEngine(LLMEngine):
463
if lora_request is not None and not self.lora_config:
464
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
465
"not enabled!")
466
+ if priority != 0 and not self.scheduler_config.policy == "priority":
467
+ raise ValueError(f"Got priority {priority} but "
468
+ "Priority scheduling is not enabled.")
469
if arrival_time is None:
470
arrival_time = time.time()
471
@@ -479,6 +485,7 @@ class _AsyncLLMEngine(LLMEngine):
485
lora_request=lora_request,
486
prompt_adapter_request=prompt_adapter_request,
487
trace_headers=trace_headers,
488
+ priority=priority,
489
)
490
491
async def check_health_async(self) -> None:
@@ -829,6 +836,7 @@ class AsyncLLMEngine:
836
lora_request: Optional[LoRARequest] = None,
837
trace_headers: Optional[Mapping[str, str]] = None,
838
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
839
+ priority: int = 0,
840
) -> Coroutine[None, None, AsyncGenerator[Union[
841
RequestOutput, EmbeddingRequestOutput], None]]:
842
...
@@ -843,6 +851,7 @@ class AsyncLLMEngine:
851
lora_request: Optional[LoRARequest] = None,
852
trace_headers: Optional[Mapping[str, str]] = None,
853
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
854
+ priority: int = 0,
855
) -> Coroutine[None, None, AsyncGenerator[Union[
856
RequestOutput, EmbeddingRequestOutput], None]]:
857
...
@@ -860,6 +869,7 @@ class AsyncLLMEngine:
869
lora_request: Optional[LoRARequest] = None,
870
trace_headers: Optional[Mapping[str, str]] = None,
871
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
872
+ priority: int = 0,
873
*,
874
inputs: Optional[PromptType] = None, # DEPRECATED
875
) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
@@ -877,6 +887,11 @@ class AsyncLLMEngine:
887
"error that caused the background loop to stop "
888
"(AsyncEngineDeadError).")
889
890
+ if (priority != 0
891
+ and not self.engine.scheduler_config.policy == "priority"):
892
+ raise ValueError(f"Got priority {priority} but "
893
+ "Priority scheduling is not enabled.")
894
+
895
stream = self._request_tracker.add_request(
896
request_id,
897
verbose=self.log_requests,
@@ -885,7 +900,9 @@ class AsyncLLMEngine:
900
arrival_time=arrival_time or time.time(),
901
lora_request=lora_request,
902
trace_headers=trace_headers,
888
- prompt_adapter_request=prompt_adapter_request)
903
+ prompt_adapter_request=prompt_adapter_request,
904
+ priority=priority,
905
+ )
906
907
return stream.generator()
908
@@ -896,7 +913,8 @@ class AsyncLLMEngine:
913
request_id: str,
914
lora_request: Optional[LoRARequest] = None,
915
trace_headers: Optional[Mapping[str, str]] = None,
899
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
916
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
917
+ priority: int = 0,
918
) -> AsyncGenerator[RequestOutput, None]:
919
"""Generate outputs for a request.
920
@@ -913,6 +931,8 @@ class AsyncLLMEngine:
931
trace_headers: OpenTelemetry trace headers.
932
prompt_adapter_request: Prompt Adapter request to use
933
for generation, if any.
934
+ priority: The priority of the request.
935
+ Only applicable with priority scheduling.
936
937
Yields:
938
The output `RequestOutput` objects from the LLMEngine
@@ -968,6 +988,7 @@ class AsyncLLMEngine:
988
lora_request=lora_request,
989
trace_headers=trace_headers,
990
prompt_adapter_request=prompt_adapter_request,
991
+ priority=priority,
992
):
993
yield LLMEngine.validate_output(output, RequestOutput)
994
vllm/engine/llm_engine.py
+1
-1
@@ -796,7 +796,7 @@ class LLMEngine:
796
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
797
"not enabled!")
798
799
- if priority > 0 and not self.scheduler_config.policy == "priority":
799
+ if priority != 0 and not self.scheduler_config.policy == "priority":
800
raise ValueError(f"Got priority {priority} but "
801
"Priority scheduling is not enabled.")
802