@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b9e96b17de
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

fix python 3.8 syntax (#2716)

Simon Mo committed Feb 1, 2024 at 14:00 UTC b9e96b17de4c555e2249db9f6149b346232c000e
2 files changed +21 -8
Dockerfile
+14 -1
@@ -4,8 +4,21 @@
4 #################### BASE BUILD IMAGE ####################
5 FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS dev
6
7 +# Set the DEBIAN_FRONTEND variable to noninteractive to avoid interactive prompts
8 +ENV DEBIAN_FRONTEND=noninteractive
9 +
10 +# Preconfigure tzdata for US Central Time (build running in us-central-1 but this really doesn't matter.)
11 +RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
12 + && echo 'tzdata tzdata/Zones/America select Chicago' | debconf-set-selections
13 +
14 +# We install an older version of python here for testing to make sure vllm works with older versions of Python.
15 +# For the actual openai compatible server, we will use the latest version of Python.
16 RUN apt-get update -y \
8 - && apt-get install -y python3-pip git
17 + && apt-get install -y software-properties-common \
18 + && add-apt-repository ppa:deadsnakes/ppa -y \
19 + && apt-get update -y \
20 + && apt-get install -y python3.8 python3.8-dev python3.8-venv python3-pip git \
21 + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
22
23 # Workaround for https://github.com/openai/triton/issues/2507 and
24 # https://github.com/pytorch/pytorch/issues/107960 -- hopefully
vllm/entrypoints/openai/serving_completion.py
+7 -7
@@ -1,7 +1,7 @@
1 import asyncio
2 import time
3 from fastapi import Request
4 -from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional
4 +from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional, Dict, Tuple
5 from vllm.logger import init_logger
6 from vllm.utils import random_uuid
7 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -19,8 +19,8 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing
19
20 logger = init_logger(__name__)
21
22 -TypeTokenIDs = list[int]
23 -TypeTopLogProbs = List[Optional[dict[int, float]]]
22 +TypeTokenIDs = List[int]
23 +TypeTopLogProbs = List[Optional[Dict[int, float]]]
24 TypeCreateLogProbsFn = Callable[
25 [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], LogProbs]
26
@@ -29,7 +29,7 @@ async def completion_stream_generator(
29 request: CompletionRequest,
30 raw_request: Request,
31 on_abort,
32 - result_generator: AsyncIterator[tuple[int, RequestOutput]],
32 + result_generator: AsyncIterator[Tuple[int, RequestOutput]],
33 create_logprobs_fn: TypeCreateLogProbsFn,
34 request_id: str,
35 created_time: int,
@@ -126,7 +126,7 @@ async def completion_stream_generator(
126 yield "data: [DONE]\n\n"
127
128
129 -def parse_prompt_format(prompt) -> tuple[bool, list]:
129 +def parse_prompt_format(prompt) -> Tuple[bool, list]:
130 # get the prompt, openai supports the following
131 # "a string, array of strings, array of tokens, or array of token arrays."
132 prompt_is_tokens = False
@@ -151,7 +151,7 @@ def parse_prompt_format(prompt) -> tuple[bool, list]:
151
152
153 def request_output_to_completion_response(
154 - final_res_batch: list[RequestOutput],
154 + final_res_batch: List[RequestOutput],
155 request: CompletionRequest,
156 create_logprobs_fn: TypeCreateLogProbsFn,
157 request_id: str,
@@ -302,7 +302,7 @@ class OpenAIServingCompletion(OpenAIServing):
302 except ValueError as e:
303 return self.create_error_response(str(e))
304
305 - result_generator: AsyncIterator[tuple[
305 + result_generator: AsyncIterator[Tuple[
306 int, RequestOutput]] = merge_async_iterators(*generators)
307
308 # Similar to the OpenAI API, when n != best_of, we do not stream the