A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
fix python 3.8 syntax (#2716)
Simon Mo committed
Feb 1, 2024 at 14:00 UTC
b9e96b17de4c555e2249db9f6149b346232c000e
2 files changed
+21
-8
Dockerfile
+14
-1
@@ -4,8 +4,21 @@
4
#################### BASE BUILD IMAGE ####################
5
FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS dev
6
7
+# Set the DEBIAN_FRONTEND variable to noninteractive to avoid interactive prompts
8
+ENV DEBIAN_FRONTEND=noninteractive
9
+
10
+# Preconfigure tzdata for US Central Time (build running in us-central-1 but this really doesn't matter.)
11
+RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
12
+ && echo 'tzdata tzdata/Zones/America select Chicago' | debconf-set-selections
13
+
14
+# We install an older version of python here for testing to make sure vllm works with older versions of Python.
15
+# For the actual openai compatible server, we will use the latest version of Python.
16
RUN apt-get update -y \
8
- && apt-get install -y python3-pip git
17
+ && apt-get install -y software-properties-common \
18
+ && add-apt-repository ppa:deadsnakes/ppa -y \
19
+ && apt-get update -y \
20
+ && apt-get install -y python3.8 python3.8-dev python3.8-venv python3-pip git \
21
+ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
22
23
# Workaround for https://github.com/openai/triton/issues/2507 and
24
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
vllm/entrypoints/openai/serving_completion.py
+7
-7
@@ -1,7 +1,7 @@
1
import asyncio
2
import time
3
from fastapi import Request
4
-from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional
4
+from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional, Dict, Tuple
5
from vllm.logger import init_logger
6
from vllm.utils import random_uuid
7
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -19,8 +19,8 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing
19
20
logger = init_logger(__name__)
21
22
-TypeTokenIDs = list[int]
23
-TypeTopLogProbs = List[Optional[dict[int, float]]]
22
+TypeTokenIDs = List[int]
23
+TypeTopLogProbs = List[Optional[Dict[int, float]]]
24
TypeCreateLogProbsFn = Callable[
25
[TypeTokenIDs, TypeTopLogProbs, Optional[int], int], LogProbs]
26
@@ -29,7 +29,7 @@ async def completion_stream_generator(
29
request: CompletionRequest,
30
raw_request: Request,
31
on_abort,
32
- result_generator: AsyncIterator[tuple[int, RequestOutput]],
32
+ result_generator: AsyncIterator[Tuple[int, RequestOutput]],
33
create_logprobs_fn: TypeCreateLogProbsFn,
34
request_id: str,
35
created_time: int,
@@ -126,7 +126,7 @@ async def completion_stream_generator(
126
yield "data: [DONE]\n\n"
127
128
129
-def parse_prompt_format(prompt) -> tuple[bool, list]:
129
+def parse_prompt_format(prompt) -> Tuple[bool, list]:
130
# get the prompt, openai supports the following
131
# "a string, array of strings, array of tokens, or array of token arrays."
132
prompt_is_tokens = False
@@ -151,7 +151,7 @@ def parse_prompt_format(prompt) -> tuple[bool, list]:
151
152
153
def request_output_to_completion_response(
154
- final_res_batch: list[RequestOutput],
154
+ final_res_batch: List[RequestOutput],
155
request: CompletionRequest,
156
create_logprobs_fn: TypeCreateLogProbsFn,
157
request_id: str,
@@ -302,7 +302,7 @@ class OpenAIServingCompletion(OpenAIServing):
302
except ValueError as e:
303
return self.create_error_response(str(e))
304
305
- result_generator: AsyncIterator[tuple[
305
+ result_generator: AsyncIterator[Tuple[
306
int, RequestOutput]] = merge_async_iterators(*generators)
307
308
# Similar to the OpenAI API, when n != best_of, we do not stream the