A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[BUG FIX] upgrade fschat version to 0.2.23 (#650)
Co-authored-by: hao.yu <hao.yu@cn-c017.server.mila.quebec>
YHPeter committed
Aug 2, 2023 at 17:05 UTC
e8ddc08ec85495e5faca31bdf9129e0bf59a4fac
1 file changed
+10
-2
vllm/entrypoints/openai/api_server.py
+10
-2
@@ -7,6 +7,7 @@ from http import HTTPStatus
7
import json
8
import time
9
from typing import AsyncGenerator, Dict, List, Optional
10
+from packaging import version
11
12
import fastapi
13
from fastapi import BackgroundTasks, Request
@@ -31,6 +32,7 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
32
from vllm.utils import random_uuid
33
34
try:
35
+ import fastchat
36
from fastchat.conversation import Conversation, SeparatorStyle
37
from fastchat.model.model_adapter import get_conversation_template
38
_fastchat_available = True
@@ -72,10 +74,16 @@ async def get_gen_prompt(request) -> str:
74
"fastchat is not installed. Please install fastchat to use "
75
"the chat completion and conversation APIs: `$ pip install fschat`"
76
)
77
+ if version.parse(fastchat.__version__) < version.parse("0.2.23"):
78
+ raise ImportError(
79
+ f"fastchat version is low. Current version: {fastchat.__version__} "
80
+ "Please upgrade fastchat to use: `$ pip install -U fschat`")
81
+
82
conv = get_conversation_template(request.model)
83
conv = Conversation(
84
name=conv.name,
78
- system=conv.system,
85
+ system_template=conv.system_template,
86
+ system_message=conv.system_message,
87
roles=conv.roles,
88
messages=list(conv.messages), # prevent in-place modification
89
offset=conv.offset,
@@ -92,7 +100,7 @@ async def get_gen_prompt(request) -> str:
100
for message in request.messages:
101
msg_role = message["role"]
102
if msg_role == "system":
95
- conv.system = message["content"]
103
+ conv.system_message = message["content"]
104
elif msg_role == "user":
105
conv.append_message(conv.roles[0], message["content"])
106
elif msg_role == "assistant":