A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Server] use fastchat.model.model_adapter.get_conversation_template method to get model template (#357)
akxxsb committed
Jul 5, 2023 at 12:39 UTC
3d64cf019e85385e33b522f65ad9e1e7c665c3e3
1 file changed
+5
-17
vllm/entrypoints/openai/api_server.py
+5
-17
@@ -13,8 +13,9 @@ from fastapi import BackgroundTasks, Request
13
from fastapi.exceptions import RequestValidationError
14
from fastapi.middleware.cors import CORSMiddleware
15
from fastapi.responses import JSONResponse, StreamingResponse
16
-from fastchat.conversation import (Conversation, SeparatorStyle,
17
- get_conv_template)
16
+from fastchat.conversation import Conversation, SeparatorStyle
17
+from fastchat.model.model_adapter import get_conversation_template
18
+
19
import uvicorn
20
21
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -36,7 +37,6 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
37
38
logger = init_logger(__name__)
39
served_model = None
39
-chat_template = None
40
app = fastapi.FastAPI()
41
42
@@ -63,7 +63,7 @@ async def check_model(request) -> Optional[JSONResponse]:
63
64
65
async def get_gen_prompt(request) -> str:
66
- conv = get_conv_template(chat_template)
66
+ conv = get_conversation_template(request.model)
67
conv = Conversation(
68
name=conv.name,
69
system=conv.system,
@@ -560,14 +560,7 @@ if __name__ == "__main__":
560
help="The model name used in the API. If not "
561
"specified, the model name will be the same as "
562
"the huggingface name.")
563
- parser.add_argument(
564
- "--chat-template",
565
- type=str,
566
- default=None,
567
- help="The chat template name used in the ChatCompletion endpoint. If "
568
- "not specified, we use the API model name as the template name. See "
569
- "https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py "
570
- "for the list of available templates.")
563
+
564
parser = AsyncEngineArgs.add_cli_args(parser)
565
args = parser.parse_args()
566
@@ -586,11 +579,6 @@ if __name__ == "__main__":
579
else:
580
served_model = args.model
581
589
- if args.chat_template is not None:
590
- chat_template = args.chat_template
591
- else:
592
- chat_template = served_model
593
-
582
engine_args = AsyncEngineArgs.from_cli_args(args)
583
engine = AsyncLLMEngine.from_engine_args(engine_args)
584
engine_model_config = asyncio.run(engine.get_model_config())