A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Add chat completion Example and simplify dependencies (#576)
Zhuohan Li committed
Jul 25, 2023 at 23:45 UTC
82ad323deed0d4f5fbdb6592f14314ca5b1118ad
4 files changed
+52
-11
examples/openai_chatcompletion_client.py
new
+33
@@ -0,0 +1,33 @@
1
+import openai
2
+
3
+# Modify OpenAI's API key and API base to use vLLM's API server.
4
+openai.api_key = "EMPTY"
5
+openai.api_base = "http://localhost:8000/v1"
6
+
7
+# List models API
8
+models = openai.Model.list()
9
+print("Models:", models)
10
+
11
+model = models["data"][0]["id"]
12
+
13
+# Chat completion API
14
+chat_completion = openai.ChatCompletion.create(
15
+ model=model,
16
+ messages=[{
17
+ "role": "system",
18
+ "content": "You are a helpful assistant."
19
+ }, {
20
+ "role": "user",
21
+ "content": "Who won the world series in 2020?"
22
+ }, {
23
+ "role":
24
+ "assistant",
25
+ "content":
26
+ "The Los Angeles Dodgers won the World Series in 2020."
27
+ }, {
28
+ "role": "user",
29
+ "content": "Where was it played?"
30
+ }])
31
+
32
+print("Chat completion results:")
33
+print(chat_completion)
examples/openai_completion_client.py
renamed
+7
-7
@@ -3,26 +3,26 @@ import openai
3
# Modify OpenAI's API key and API base to use vLLM's API server.
4
openai.api_key = "EMPTY"
5
openai.api_base = "http://localhost:8000/v1"
6
-model = "facebook/opt-125m"
6
8
-# Test list models API
7
+# List models API
8
models = openai.Model.list()
9
print("Models:", models)
10
12
-# Test completion API
13
-stream = True
11
+model = models["data"][0]["id"]
12
+
13
+# Completion API
14
+stream = False
15
completion = openai.Completion.create(
16
model=model,
17
prompt="A robot may not injure a human being",
18
echo=False,
19
n=2,
19
- best_of=3,
20
stream=stream,
21
logprobs=3)
22
23
-# print the completion
23
+print("Completion results:")
24
if stream:
25
for c in completion:
26
print(c)
27
else:
28
- print("Completion result:", completion)
28
+ print(completion)
requirements.txt
-1
@@ -9,4 +9,3 @@ xformers >= 0.0.19
9
fastapi
10
uvicorn
11
pydantic < 2 # Required for OpenAI server.
12
-fschat # Required for OpenAI ChatCompletion Endpoint.
vllm/entrypoints/openai/api_server.py
+12
-3
@@ -13,9 +13,6 @@ from fastapi import BackgroundTasks, Request
13
from fastapi.exceptions import RequestValidationError
14
from fastapi.middleware.cors import CORSMiddleware
15
from fastapi.responses import JSONResponse, StreamingResponse
16
-from fastchat.conversation import Conversation, SeparatorStyle
17
-from fastchat.model.model_adapter import get_conversation_template
18
-
16
import uvicorn
17
18
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -33,6 +30,13 @@ from vllm.sampling_params import SamplingParams
30
from vllm.transformers_utils.tokenizer import get_tokenizer
31
from vllm.utils import random_uuid
32
33
+try:
34
+ from fastchat.conversation import Conversation, SeparatorStyle
35
+ from fastchat.model.model_adapter import get_conversation_template
36
+ _fastchat_available = True
37
+except ImportError:
38
+ _fastchat_available = False
39
+
40
TIMEOUT_KEEP_ALIVE = 5 # seconds
41
42
logger = init_logger(__name__)
@@ -63,6 +67,11 @@ async def check_model(request) -> Optional[JSONResponse]:
67
68
69
async def get_gen_prompt(request) -> str:
70
+ if not _fastchat_available:
71
+ raise ModuleNotFoundError(
72
+ "fastchat is not installed. Please install fastchat to use "
73
+ "the chat completion and conversation APIs: `$ pip install fschat`"
74
+ )
75
conv = get_conversation_template(request.model)
76
conv = Conversation(
77
name=conv.name,