@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3d64cf019e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Server] use fastchat.model.model_adapter.get_conversation_template method to get model template (#357)

akxxsb committed Jul 5, 2023 at 12:39 UTC 3d64cf019e85385e33b522f65ad9e1e7c665c3e3
1 file changed +5 -17
vllm/entrypoints/openai/api_server.py
+5 -17
@@ -13,8 +13,9 @@ from fastapi import BackgroundTasks, Request
13 from fastapi.exceptions import RequestValidationError
14 from fastapi.middleware.cors import CORSMiddleware
15 from fastapi.responses import JSONResponse, StreamingResponse
16 -from fastchat.conversation import (Conversation, SeparatorStyle,
17 - get_conv_template)
16 +from fastchat.conversation import Conversation, SeparatorStyle
17 +from fastchat.model.model_adapter import get_conversation_template
18 +
19 import uvicorn
20
21 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -36,7 +37,6 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
37
38 logger = init_logger(__name__)
39 served_model = None
39 -chat_template = None
40 app = fastapi.FastAPI()
41
42
@@ -63,7 +63,7 @@ async def check_model(request) -> Optional[JSONResponse]:
63
64
65 async def get_gen_prompt(request) -> str:
66 - conv = get_conv_template(chat_template)
66 + conv = get_conversation_template(request.model)
67 conv = Conversation(
68 name=conv.name,
69 system=conv.system,
@@ -560,14 +560,7 @@ if __name__ == "__main__":
560 help="The model name used in the API. If not "
561 "specified, the model name will be the same as "
562 "the huggingface name.")
563 - parser.add_argument(
564 - "--chat-template",
565 - type=str,
566 - default=None,
567 - help="The chat template name used in the ChatCompletion endpoint. If "
568 - "not specified, we use the API model name as the template name. See "
569 - "https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py "
570 - "for the list of available templates.")
563 +
564 parser = AsyncEngineArgs.add_cli_args(parser)
565 args = parser.parse_args()
566
@@ -586,11 +579,6 @@ if __name__ == "__main__":
579 else:
580 served_model = args.model
581
589 - if args.chat_template is not None:
590 - chat_template = args.chat_template
591 - else:
592 - chat_template = served_model
593 -
582 engine_args = AsyncEngineArgs.from_cli_args(args)
583 engine = AsyncLLMEngine.from_engine_args(engine_args)
584 engine_model_config = asyncio.run(engine.get_model_config())