@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82ad323dee
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Fix] Add chat completion Example and simplify dependencies (#576)

Zhuohan Li committed Jul 25, 2023 at 23:45 UTC 82ad323deed0d4f5fbdb6592f14314ca5b1118ad
4 files changed +52 -11
examples/openai_chatcompletion_client.py new
+33
@@ -0,0 +1,33 @@
1 +import openai
2 +
3 +# Modify OpenAI's API key and API base to use vLLM's API server.
4 +openai.api_key = "EMPTY"
5 +openai.api_base = "http://localhost:8000/v1"
6 +
7 +# List models API
8 +models = openai.Model.list()
9 +print("Models:", models)
10 +
11 +model = models["data"][0]["id"]
12 +
13 +# Chat completion API
14 +chat_completion = openai.ChatCompletion.create(
15 + model=model,
16 + messages=[{
17 + "role": "system",
18 + "content": "You are a helpful assistant."
19 + }, {
20 + "role": "user",
21 + "content": "Who won the world series in 2020?"
22 + }, {
23 + "role":
24 + "assistant",
25 + "content":
26 + "The Los Angeles Dodgers won the World Series in 2020."
27 + }, {
28 + "role": "user",
29 + "content": "Where was it played?"
30 + }])
31 +
32 +print("Chat completion results:")
33 +print(chat_completion)
examples/openai_completion_client.py renamed
+7 -7
@@ -3,26 +3,26 @@ import openai
3 # Modify OpenAI's API key and API base to use vLLM's API server.
4 openai.api_key = "EMPTY"
5 openai.api_base = "http://localhost:8000/v1"
6 -model = "facebook/opt-125m"
6
8 -# Test list models API
7 +# List models API
8 models = openai.Model.list()
9 print("Models:", models)
10
12 -# Test completion API
13 -stream = True
11 +model = models["data"][0]["id"]
12 +
13 +# Completion API
14 +stream = False
15 completion = openai.Completion.create(
16 model=model,
17 prompt="A robot may not injure a human being",
18 echo=False,
19 n=2,
19 - best_of=3,
20 stream=stream,
21 logprobs=3)
22
23 -# print the completion
23 +print("Completion results:")
24 if stream:
25 for c in completion:
26 print(c)
27 else:
28 - print("Completion result:", completion)
28 + print(completion)
requirements.txt
-1
@@ -9,4 +9,3 @@ xformers >= 0.0.19
9 fastapi
10 uvicorn
11 pydantic < 2 # Required for OpenAI server.
12 -fschat # Required for OpenAI ChatCompletion Endpoint.
vllm/entrypoints/openai/api_server.py
+12 -3
@@ -13,9 +13,6 @@ from fastapi import BackgroundTasks, Request
13 from fastapi.exceptions import RequestValidationError
14 from fastapi.middleware.cors import CORSMiddleware
15 from fastapi.responses import JSONResponse, StreamingResponse
16 -from fastchat.conversation import Conversation, SeparatorStyle
17 -from fastchat.model.model_adapter import get_conversation_template
18 -
16 import uvicorn
17
18 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -33,6 +30,13 @@ from vllm.sampling_params import SamplingParams
30 from vllm.transformers_utils.tokenizer import get_tokenizer
31 from vllm.utils import random_uuid
32
33 +try:
34 + from fastchat.conversation import Conversation, SeparatorStyle
35 + from fastchat.model.model_adapter import get_conversation_template
36 + _fastchat_available = True
37 +except ImportError:
38 + _fastchat_available = False
39 +
40 TIMEOUT_KEEP_ALIVE = 5 # seconds
41
42 logger = init_logger(__name__)
@@ -63,6 +67,11 @@ async def check_model(request) -> Optional[JSONResponse]:
67
68
69 async def get_gen_prompt(request) -> str:
70 + if not _fastchat_available:
71 + raise ModuleNotFoundError(
72 + "fastchat is not installed. Please install fastchat to use "
73 + "the chat completion and conversation APIs: `$ pip install fschat`"
74 + )
75 conv = get_conversation_template(request.model)
76 conv = Conversation(
77 name=conv.name,