@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 35bd215168
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Core] [Frontend] Priority scheduling for embeddings and in the OpenAI-API (#8965)

Sebastian Schoennenbeck committed Oct 1, 2024 at 11:58 UTC 35bd2151684ffb20cdad825abe33e0e6f0cc005a
8 files changed +53 -5
vllm/engine/async_llm_engine.py
+4
@@ -1043,6 +1043,7 @@ class AsyncLLMEngine:
1043 request_id: str,
1044 lora_request: Optional[LoRARequest] = None,
1045 trace_headers: Optional[Mapping[str, str]] = None,
1046 + priority: int = 0,
1047 ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
1048 """Generate outputs for a request from an embedding model.
1049
@@ -1057,6 +1058,8 @@ class AsyncLLMEngine:
1058 request_id: The unique id of the request.
1059 lora_request: LoRA request to use for generation, if any.
1060 trace_headers: OpenTelemetry trace headers.
1061 + priority: The priority of the request.
1062 + Only applicable with priority scheduling.
1063
1064 Yields:
1065 The output `EmbeddingRequestOutput` objects from the LLMEngine
@@ -1109,6 +1112,7 @@ class AsyncLLMEngine:
1112 pooling_params,
1113 lora_request=lora_request,
1114 trace_headers=trace_headers,
1115 + priority=priority,
1116 ):
1117 yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
1118
vllm/engine/multiprocessing/__init__.py
+5
@@ -30,6 +30,7 @@ class RPCProcessRequest:
30 lora_request: Optional[LoRARequest] = None
31 trace_headers: Optional[Mapping[str, str]] = None
32 prompt_adapter_request: Optional[PromptAdapterRequest] = None
33 + priority: int = 0
34
35 @overload # DEPRECATED
36 def __init__(
@@ -41,6 +42,7 @@ class RPCProcessRequest:
42 lora_request: Optional[LoRARequest] = None,
43 trace_headers: Optional[Mapping[str, str]] = None,
44 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
45 + priority: int = 0,
46 ) -> None:
47 ...
48
@@ -53,6 +55,7 @@ class RPCProcessRequest:
55 lora_request: Optional[LoRARequest] = None,
56 trace_headers: Optional[Mapping[str, str]] = None,
57 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
58 + priority: int = 0,
59 ) -> None:
60 ...
61
@@ -68,6 +71,7 @@ class RPCProcessRequest:
71 lora_request: Optional[LoRARequest] = None,
72 trace_headers: Optional[Mapping[str, str]] = None,
73 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
74 + priority: int = 0,
75 *,
76 inputs: Optional[PromptType] = None, # DEPRECATED
77 ) -> None:
@@ -84,6 +88,7 @@ class RPCProcessRequest:
88 self.lora_request = lora_request
89 self.trace_headers = trace_headers
90 self.prompt_adapter_request = prompt_adapter_request
91 + self.priority = priority
92
93
94 @dataclass
vllm/engine/multiprocessing/client.py
+16 -4
@@ -380,6 +380,7 @@ class MQLLMEngineClient:
380 lora_request: Optional[LoRARequest] = None,
381 trace_headers: Optional[Mapping[str, str]] = None,
382 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
383 + priority: int = 0,
384 ) -> AsyncGenerator[RequestOutput, None]:
385 ...
386
@@ -392,6 +393,7 @@ class MQLLMEngineClient:
393 lora_request: Optional[LoRARequest] = None,
394 trace_headers: Optional[Mapping[str, str]] = None,
395 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
396 + priority: int = 0,
397 ) -> AsyncGenerator[RequestOutput, None]:
398 ...
399
@@ -407,6 +409,7 @@ class MQLLMEngineClient:
409 lora_request: Optional[LoRARequest] = None,
410 trace_headers: Optional[Mapping[str, str]] = None,
411 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
412 + priority: int = 0,
413 *,
414 inputs: Optional[PromptType] = None # DEPRECATED
415 ) -> AsyncGenerator[RequestOutput, None]:
@@ -425,6 +428,9 @@ class MQLLMEngineClient:
428 trace_headers: OpenTelemetry trace headers.
429 prompt_adapter_request: Prompt Adapter request to use
430 for generation, if any.
431 + priority: Priority of the request (lower means earlier handling).
432 + Any priority other than 0 will lead to an error if the
433 + scheduling policy is not "priority".
434 """
435 if inputs is not None:
436 prompt = inputs
@@ -433,7 +439,7 @@ class MQLLMEngineClient:
439
440 return self._process_request(prompt, sampling_params, request_id,
441 lora_request, trace_headers,
436 - prompt_adapter_request)
442 + prompt_adapter_request, priority)
443
444 @overload # DEPRECATED
445 def encode(
@@ -444,6 +450,7 @@ class MQLLMEngineClient:
450 request_id: str,
451 lora_request: Optional[LoRARequest] = None,
452 trace_headers: Optional[Mapping[str, str]] = None,
453 + priority: int = 0,
454 ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
455 ...
456
@@ -455,6 +462,7 @@ class MQLLMEngineClient:
462 request_id: str,
463 lora_request: Optional[LoRARequest] = None,
464 trace_headers: Optional[Mapping[str, str]] = None,
465 + priority: int = 0,
466 ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
467 ...
468
@@ -469,6 +477,7 @@ class MQLLMEngineClient:
477 request_id: Optional[str] = None,
478 lora_request: Optional[LoRARequest] = None,
479 trace_headers: Optional[Mapping[str, str]] = None,
480 + priority: int = 0,
481 *,
482 inputs: Optional[PromptType] = None # DEPRECATED
483 ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
@@ -496,7 +505,7 @@ class MQLLMEngineClient:
505 and request_id is not None)
506
507 return self._process_request(prompt, pooling_params, request_id,
499 - lora_request, trace_headers)
508 + lora_request, trace_headers, priority)
509
510 async def _process_request(
511 self,
@@ -505,7 +514,8 @@ class MQLLMEngineClient:
514 request_id: str,
515 lora_request: Optional[LoRARequest] = None,
516 trace_headers: Optional[Mapping[str, str]] = None,
508 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
517 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
518 + priority: int = 0,
519 ) -> Union[AsyncGenerator[RequestOutput, None], AsyncGenerator[
520 EmbeddingRequestOutput, None]]:
521 """Send an RPCGenerateRequest to the RPCServer and stream responses."""
@@ -550,7 +560,9 @@ class MQLLMEngineClient:
560 request_id=request_id,
561 lora_request=lora_request,
562 trace_headers=trace_headers,
553 - prompt_adapter_request=prompt_adapter_request))
563 + prompt_adapter_request=prompt_adapter_request,
564 + priority=priority,
565 + ))
566
567 # 3) Send the RPCGenerateRequest to the MQLLMEngine.
568 parts = (request_bytes,
vllm/engine/protocol.py
+3 -1
@@ -40,7 +40,8 @@ class EngineClient(Protocol):
40 request_id: str,
41 lora_request: Optional[LoRARequest] = None,
42 trace_headers: Optional[Mapping[str, str]] = None,
43 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
43 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
44 + priority: int = 0,
45 ) -> AsyncGenerator[RequestOutput, None]:
46 """Generate outputs for a request."""
47 ...
@@ -52,6 +53,7 @@ class EngineClient(Protocol):
53 request_id: str,
54 lora_request: Optional[LoRARequest] = None,
55 trace_headers: Optional[Mapping[str, str]] = None,
56 + priority: int = 0,
57 ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
58 """Generate outputs for a request from an embedding model."""
59 ...
vllm/entrypoints/openai/protocol.py
+22
@@ -279,6 +279,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
279 description=(
280 "If specified, will override the default whitespace pattern "
281 "for guided json decoding."))
282 + priority: int = Field(
283 + default=0,
284 + description=(
285 + "The priority of the request (lower means earlier handling; "
286 + "default: 0). Any priority other than 0 will raise an error "
287 + "if the served model does not use priority scheduling."))
288
289 # doc: end-chat-completion-extra-params
290
@@ -552,6 +558,12 @@ class CompletionRequest(OpenAIBaseModel):
558 description=(
559 "If specified, will override the default whitespace pattern "
560 "for guided json decoding."))
561 + priority: int = Field(
562 + default=0,
563 + description=(
564 + "The priority of the request (lower means earlier handling; "
565 + "default: 0). Any priority other than 0 will raise an error "
566 + "if the served model does not use priority scheduling."))
567
568 # doc: end-completion-extra-params
569
@@ -665,6 +677,16 @@ class EmbeddingRequest(OpenAIBaseModel):
677
678 # doc: end-embedding-pooling-params
679
680 + # doc: begin-embedding-extra-params
681 + priority: int = Field(
682 + default=0,
683 + description=(
684 + "The priority of the request (lower means earlier handling; "
685 + "default: 0). Any priority other than 0 will raise an error "
686 + "if the served model does not use priority scheduling."))
687 +
688 + # doc: end-embedding-extra-params
689 +
690 def to_pooling_params(self):
691 return PoolingParams(additional_data=self.additional_data)
692
vllm/entrypoints/openai/serving_chat.py
+1
@@ -235,6 +235,7 @@ class OpenAIServingChat(OpenAIServing):
235 lora_request=lora_request,
236 trace_headers=trace_headers,
237 prompt_adapter_request=prompt_adapter_request,
238 + priority=request.priority,
239 )
240 except ValueError as e:
241 # TODO: Use a vllm-specific Validation Error
vllm/entrypoints/openai/serving_completion.py
+1
@@ -148,6 +148,7 @@ class OpenAIServingCompletion(OpenAIServing):
148 lora_request=lora_request,
149 prompt_adapter_request=prompt_adapter_request,
150 trace_headers=trace_headers,
151 + priority=request.priority,
152 )
153
154 generators.append(generator)
vllm/entrypoints/openai/serving_embedding.py
+1
@@ -148,6 +148,7 @@ class OpenAIServingEmbedding(OpenAIServing):
148 pooling_params,
149 request_id_item,
150 lora_request=lora_request,
151 + priority=request.priority,
152 )
153
154 generators.append(generator)