A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Misc] Minimum requirements for SageMaker compatibility (#11576)
Nathan Azrak committed
Jan 3, 2025 at 10:59 UTC
68d37809b9b52f4d012fa0dfbb187f0fe978bdbc
3 files changed
+95
-3
Dockerfile
+11
-2
@@ -234,8 +234,8 @@ RUN mv vllm test_docs/
234
#################### TEST IMAGE ####################
235
236
#################### OPENAI API SERVER ####################
237
-# openai api server alternative
238
-FROM vllm-base AS vllm-openai
237
+# base openai image with additional requirements, for any subsequent openai-style images
238
+FROM vllm-base AS vllm-openai-base
239
240
# install additional dependencies for openai api server
241
RUN --mount=type=cache,target=/root/.cache/pip \
@@ -247,5 +247,14 @@ RUN --mount=type=cache,target=/root/.cache/pip \
247
248
ENV VLLM_USAGE_SOURCE production-docker-image
249
250
+# define sagemaker first, so it is not default from `docker build`
251
+FROM vllm-openai-base AS vllm-sagemaker
252
+
253
+COPY examples/sagemaker-entrypoint.sh .
254
+RUN chmod +x sagemaker-entrypoint.sh
255
+ENTRYPOINT ["./sagemaker-entrypoint.sh"]
256
+
257
+FROM vllm-openai-base AS vllm-openai
258
+
259
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
260
#################### OPENAI API SERVER ####################
examples/sagemaker-entrypoint.sh
new
+24
@@ -0,0 +1,24 @@
1
+#!/bin/bash
2
+
3
+# Define the prefix for environment variables to look for
4
+PREFIX="SM_VLLM_"
5
+ARG_PREFIX="--"
6
+
7
+# Initialize an array for storing the arguments
8
+# port 8080 required by sagemaker, https://docs.aws.amazon.com/sagemaker/latest/dg/your-algorithms-inference-code.html#your-algorithms-inference-code-container-response
9
+ARGS=(--port 8080)
10
+
11
+# Loop through all environment variables
12
+while IFS='=' read -r key value; do
13
+ # Remove the prefix from the key, convert to lowercase, and replace underscores with dashes
14
+ arg_name=$(echo "${key#"${PREFIX}"}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')
15
+
16
+ # Add the argument name and value to the ARGS array
17
+ ARGS+=("${ARG_PREFIX}${arg_name}")
18
+ if [ -n "$value" ]; then
19
+ ARGS+=("$value")
20
+ fi
21
+done < <(env | grep "^${PREFIX}")
22
+
23
+# Pass the collected arguments to the main entrypoint
24
+exec python3 -m vllm.entrypoints.openai.api_server "${ARGS[@]}"
\ No newline at end of file
vllm/entrypoints/openai/api_server.py
+60
-1
@@ -16,7 +16,7 @@ from http import HTTPStatus
16
from typing import AsyncIterator, Optional, Set, Tuple
17
18
import uvloop
19
-from fastapi import APIRouter, FastAPI, Request
19
+from fastapi import APIRouter, FastAPI, HTTPException, Request
20
from fastapi.exceptions import RequestValidationError
21
from fastapi.middleware.cors import CORSMiddleware
22
from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -44,11 +44,15 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
44
CompletionResponse,
45
DetokenizeRequest,
46
DetokenizeResponse,
47
+ EmbeddingChatRequest,
48
+ EmbeddingCompletionRequest,
49
EmbeddingRequest,
50
EmbeddingResponse,
51
EmbeddingResponseData,
52
ErrorResponse,
53
LoadLoraAdapterRequest,
54
+ PoolingChatRequest,
55
+ PoolingCompletionRequest,
56
PoolingRequest, PoolingResponse,
57
ScoreRequest, ScoreResponse,
58
TokenizeRequest,
@@ -310,6 +314,12 @@ async def health(raw_request: Request) -> Response:
314
return Response(status_code=200)
315
316
317
+@router.api_route("/ping", methods=["GET", "POST"])
318
+async def ping(raw_request: Request) -> Response:
319
+ """Ping check. Endpoint required for SageMaker"""
320
+ return await health(raw_request)
321
+
322
+
323
@router.post("/tokenize")
324
@with_cancellation
325
async def tokenize(request: TokenizeRequest, raw_request: Request):
@@ -483,6 +493,54 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
493
return await create_score(request, raw_request)
494
495
496
+TASK_HANDLERS = {
497
+ "generate": {
498
+ "messages": (ChatCompletionRequest, create_chat_completion),
499
+ "default": (CompletionRequest, create_completion),
500
+ },
501
+ "embed": {
502
+ "messages": (EmbeddingChatRequest, create_embedding),
503
+ "default": (EmbeddingCompletionRequest, create_embedding),
504
+ },
505
+ "score": {
506
+ "default": (ScoreRequest, create_score),
507
+ },
508
+ "reward": {
509
+ "messages": (PoolingChatRequest, create_pooling),
510
+ "default": (PoolingCompletionRequest, create_pooling),
511
+ },
512
+ "classify": {
513
+ "messages": (PoolingChatRequest, create_pooling),
514
+ "default": (PoolingCompletionRequest, create_pooling),
515
+ },
516
+}
517
+
518
+
519
+@router.post("/invocations")
520
+async def invocations(raw_request: Request):
521
+ """
522
+ For SageMaker, routes requests to other handlers based on model `task`.
523
+ """
524
+ body = await raw_request.json()
525
+ task = raw_request.app.state.task
526
+
527
+ if task not in TASK_HANDLERS:
528
+ raise HTTPException(
529
+ status_code=400,
530
+ detail=f"Unsupported task: '{task}' for '/invocations'. "
531
+ f"Expected one of {set(TASK_HANDLERS.keys())}")
532
+
533
+ handler_config = TASK_HANDLERS[task]
534
+ if "messages" in body:
535
+ request_model, handler = handler_config["messages"]
536
+ else:
537
+ request_model, handler = handler_config["default"]
538
+
539
+ # this is required since we lose the FastAPI automatic casting
540
+ request = request_model.model_validate(body)
541
+ return await handler(request, raw_request)
542
+
543
+
544
if envs.VLLM_TORCH_PROFILER_DIR:
545
logger.warning(
546
"Torch Profiler is enabled in the API server. This should ONLY be "
@@ -687,6 +745,7 @@ def init_app_state(
745
chat_template=resolved_chat_template,
746
chat_template_content_format=args.chat_template_content_format,
747
)
748
+ state.task = model_config.task
749
750
751
def create_server_socket(addr: Tuple[str, int]) -> socket.socket: