@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e8ddc08ec8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[BUG FIX] upgrade fschat version to 0.2.23 (#650)

Co-authored-by: hao.yu <hao.yu@cn-c017.server.mila.quebec>

YHPeter committed Aug 2, 2023 at 17:05 UTC e8ddc08ec85495e5faca31bdf9129e0bf59a4fac
1 file changed +10 -2
vllm/entrypoints/openai/api_server.py
+10 -2
@@ -7,6 +7,7 @@ from http import HTTPStatus
7 import json
8 import time
9 from typing import AsyncGenerator, Dict, List, Optional
10 +from packaging import version
11
12 import fastapi
13 from fastapi import BackgroundTasks, Request
@@ -31,6 +32,7 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
32 from vllm.utils import random_uuid
33
34 try:
35 + import fastchat
36 from fastchat.conversation import Conversation, SeparatorStyle
37 from fastchat.model.model_adapter import get_conversation_template
38 _fastchat_available = True
@@ -72,10 +74,16 @@ async def get_gen_prompt(request) -> str:
74 "fastchat is not installed. Please install fastchat to use "
75 "the chat completion and conversation APIs: `$ pip install fschat`"
76 )
77 + if version.parse(fastchat.__version__) < version.parse("0.2.23"):
78 + raise ImportError(
79 + f"fastchat version is low. Current version: {fastchat.__version__} "
80 + "Please upgrade fastchat to use: `$ pip install -U fschat`")
81 +
82 conv = get_conversation_template(request.model)
83 conv = Conversation(
84 name=conv.name,
78 - system=conv.system,
85 + system_template=conv.system_template,
86 + system_message=conv.system_message,
87 roles=conv.roles,
88 messages=list(conv.messages), # prevent in-place modification
89 offset=conv.offset,
@@ -92,7 +100,7 @@ async def get_gen_prompt(request) -> str:
100 for message in request.messages:
101 msg_role = message["role"]
102 if msg_role == "system":
95 - conv.system = message["content"]
103 + conv.system_message = message["content"]
104 elif msg_role == "user":
105 conv.append_message(conv.roles[0], message["content"])
106 elif msg_role == "assistant":