A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Core] [Frontend] Priority scheduling for embeddings and in the OpenAI-API (#8965)
Sebastian Schoennenbeck committed
Oct 1, 2024 at 11:58 UTC
35bd2151684ffb20cdad825abe33e0e6f0cc005a
8 files changed
+53
-5
vllm/engine/async_llm_engine.py
+4
@@ -1043,6 +1043,7 @@ class AsyncLLMEngine:
1043
request_id: str,
1044
lora_request: Optional[LoRARequest] = None,
1045
trace_headers: Optional[Mapping[str, str]] = None,
1046
+ priority: int = 0,
1047
) -> AsyncGenerator[EmbeddingRequestOutput, None]:
1048
"""Generate outputs for a request from an embedding model.
1049
@@ -1057,6 +1058,8 @@ class AsyncLLMEngine:
1058
request_id: The unique id of the request.
1059
lora_request: LoRA request to use for generation, if any.
1060
trace_headers: OpenTelemetry trace headers.
1061
+ priority: The priority of the request.
1062
+ Only applicable with priority scheduling.
1063
1064
Yields:
1065
The output `EmbeddingRequestOutput` objects from the LLMEngine
@@ -1109,6 +1112,7 @@ class AsyncLLMEngine:
1112
pooling_params,
1113
lora_request=lora_request,
1114
trace_headers=trace_headers,
1115
+ priority=priority,
1116
):
1117
yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
1118
vllm/engine/multiprocessing/__init__.py
+5
@@ -30,6 +30,7 @@ class RPCProcessRequest:
30
lora_request: Optional[LoRARequest] = None
31
trace_headers: Optional[Mapping[str, str]] = None
32
prompt_adapter_request: Optional[PromptAdapterRequest] = None
33
+ priority: int = 0
34
35
@overload # DEPRECATED
36
def __init__(
@@ -41,6 +42,7 @@ class RPCProcessRequest:
42
lora_request: Optional[LoRARequest] = None,
43
trace_headers: Optional[Mapping[str, str]] = None,
44
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
45
+ priority: int = 0,
46
) -> None:
47
...
48
@@ -53,6 +55,7 @@ class RPCProcessRequest:
55
lora_request: Optional[LoRARequest] = None,
56
trace_headers: Optional[Mapping[str, str]] = None,
57
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
58
+ priority: int = 0,
59
) -> None:
60
...
61
@@ -68,6 +71,7 @@ class RPCProcessRequest:
71
lora_request: Optional[LoRARequest] = None,
72
trace_headers: Optional[Mapping[str, str]] = None,
73
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
74
+ priority: int = 0,
75
*,
76
inputs: Optional[PromptType] = None, # DEPRECATED
77
) -> None:
@@ -84,6 +88,7 @@ class RPCProcessRequest:
88
self.lora_request = lora_request
89
self.trace_headers = trace_headers
90
self.prompt_adapter_request = prompt_adapter_request
91
+ self.priority = priority
92
93
94
@dataclass
vllm/engine/multiprocessing/client.py
+16
-4
@@ -380,6 +380,7 @@ class MQLLMEngineClient:
380
lora_request: Optional[LoRARequest] = None,
381
trace_headers: Optional[Mapping[str, str]] = None,
382
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
383
+ priority: int = 0,
384
) -> AsyncGenerator[RequestOutput, None]:
385
...
386
@@ -392,6 +393,7 @@ class MQLLMEngineClient:
393
lora_request: Optional[LoRARequest] = None,
394
trace_headers: Optional[Mapping[str, str]] = None,
395
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
396
+ priority: int = 0,
397
) -> AsyncGenerator[RequestOutput, None]:
398
...
399
@@ -407,6 +409,7 @@ class MQLLMEngineClient:
409
lora_request: Optional[LoRARequest] = None,
410
trace_headers: Optional[Mapping[str, str]] = None,
411
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
412
+ priority: int = 0,
413
*,
414
inputs: Optional[PromptType] = None # DEPRECATED
415
) -> AsyncGenerator[RequestOutput, None]:
@@ -425,6 +428,9 @@ class MQLLMEngineClient:
428
trace_headers: OpenTelemetry trace headers.
429
prompt_adapter_request: Prompt Adapter request to use
430
for generation, if any.
431
+ priority: Priority of the request (lower means earlier handling).
432
+ Any priority other than 0 will lead to an error if the
433
+ scheduling policy is not "priority".
434
"""
435
if inputs is not None:
436
prompt = inputs
@@ -433,7 +439,7 @@ class MQLLMEngineClient:
439
440
return self._process_request(prompt, sampling_params, request_id,
441
lora_request, trace_headers,
436
- prompt_adapter_request)
442
+ prompt_adapter_request, priority)
443
444
@overload # DEPRECATED
445
def encode(
@@ -444,6 +450,7 @@ class MQLLMEngineClient:
450
request_id: str,
451
lora_request: Optional[LoRARequest] = None,
452
trace_headers: Optional[Mapping[str, str]] = None,
453
+ priority: int = 0,
454
) -> AsyncGenerator[EmbeddingRequestOutput, None]:
455
...
456
@@ -455,6 +462,7 @@ class MQLLMEngineClient:
462
request_id: str,
463
lora_request: Optional[LoRARequest] = None,
464
trace_headers: Optional[Mapping[str, str]] = None,
465
+ priority: int = 0,
466
) -> AsyncGenerator[EmbeddingRequestOutput, None]:
467
...
468
@@ -469,6 +477,7 @@ class MQLLMEngineClient:
477
request_id: Optional[str] = None,
478
lora_request: Optional[LoRARequest] = None,
479
trace_headers: Optional[Mapping[str, str]] = None,
480
+ priority: int = 0,
481
*,
482
inputs: Optional[PromptType] = None # DEPRECATED
483
) -> AsyncGenerator[EmbeddingRequestOutput, None]:
@@ -496,7 +505,7 @@ class MQLLMEngineClient:
505
and request_id is not None)
506
507
return self._process_request(prompt, pooling_params, request_id,
499
- lora_request, trace_headers)
508
+ lora_request, trace_headers, priority)
509
510
async def _process_request(
511
self,
@@ -505,7 +514,8 @@ class MQLLMEngineClient:
514
request_id: str,
515
lora_request: Optional[LoRARequest] = None,
516
trace_headers: Optional[Mapping[str, str]] = None,
508
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
517
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
518
+ priority: int = 0,
519
) -> Union[AsyncGenerator[RequestOutput, None], AsyncGenerator[
520
EmbeddingRequestOutput, None]]:
521
"""Send an RPCGenerateRequest to the RPCServer and stream responses."""
@@ -550,7 +560,9 @@ class MQLLMEngineClient:
560
request_id=request_id,
561
lora_request=lora_request,
562
trace_headers=trace_headers,
553
- prompt_adapter_request=prompt_adapter_request))
563
+ prompt_adapter_request=prompt_adapter_request,
564
+ priority=priority,
565
+ ))
566
567
# 3) Send the RPCGenerateRequest to the MQLLMEngine.
568
parts = (request_bytes,
vllm/engine/protocol.py
+3
-1
@@ -40,7 +40,8 @@ class EngineClient(Protocol):
40
request_id: str,
41
lora_request: Optional[LoRARequest] = None,
42
trace_headers: Optional[Mapping[str, str]] = None,
43
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
43
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
44
+ priority: int = 0,
45
) -> AsyncGenerator[RequestOutput, None]:
46
"""Generate outputs for a request."""
47
...
@@ -52,6 +53,7 @@ class EngineClient(Protocol):
53
request_id: str,
54
lora_request: Optional[LoRARequest] = None,
55
trace_headers: Optional[Mapping[str, str]] = None,
56
+ priority: int = 0,
57
) -> AsyncGenerator[EmbeddingRequestOutput, None]:
58
"""Generate outputs for a request from an embedding model."""
59
...
vllm/entrypoints/openai/protocol.py
+22
@@ -279,6 +279,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
279
description=(
280
"If specified, will override the default whitespace pattern "
281
"for guided json decoding."))
282
+ priority: int = Field(
283
+ default=0,
284
+ description=(
285
+ "The priority of the request (lower means earlier handling; "
286
+ "default: 0). Any priority other than 0 will raise an error "
287
+ "if the served model does not use priority scheduling."))
288
289
# doc: end-chat-completion-extra-params
290
@@ -552,6 +558,12 @@ class CompletionRequest(OpenAIBaseModel):
558
description=(
559
"If specified, will override the default whitespace pattern "
560
"for guided json decoding."))
561
+ priority: int = Field(
562
+ default=0,
563
+ description=(
564
+ "The priority of the request (lower means earlier handling; "
565
+ "default: 0). Any priority other than 0 will raise an error "
566
+ "if the served model does not use priority scheduling."))
567
568
# doc: end-completion-extra-params
569
@@ -665,6 +677,16 @@ class EmbeddingRequest(OpenAIBaseModel):
677
678
# doc: end-embedding-pooling-params
679
680
+ # doc: begin-embedding-extra-params
681
+ priority: int = Field(
682
+ default=0,
683
+ description=(
684
+ "The priority of the request (lower means earlier handling; "
685
+ "default: 0). Any priority other than 0 will raise an error "
686
+ "if the served model does not use priority scheduling."))
687
+
688
+ # doc: end-embedding-extra-params
689
+
690
def to_pooling_params(self):
691
return PoolingParams(additional_data=self.additional_data)
692
vllm/entrypoints/openai/serving_chat.py
+1
@@ -235,6 +235,7 @@ class OpenAIServingChat(OpenAIServing):
235
lora_request=lora_request,
236
trace_headers=trace_headers,
237
prompt_adapter_request=prompt_adapter_request,
238
+ priority=request.priority,
239
)
240
except ValueError as e:
241
# TODO: Use a vllm-specific Validation Error
vllm/entrypoints/openai/serving_completion.py
+1
@@ -148,6 +148,7 @@ class OpenAIServingCompletion(OpenAIServing):
148
lora_request=lora_request,
149
prompt_adapter_request=prompt_adapter_request,
150
trace_headers=trace_headers,
151
+ priority=request.priority,
152
)
153
154
generators.append(generator)
vllm/entrypoints/openai/serving_embedding.py
+1
@@ -148,6 +148,7 @@ class OpenAIServingEmbedding(OpenAIServing):
148
pooling_params,
149
request_id_item,
150
lora_request=lora_request,
151
+ priority=request.priority,
152
)
153
154
generators.append(generator)