@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 98fe8cb542
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Server] Add option to specify chat template for chat endpoint (#345)

Zhuohan Li committed Jul 3, 2023 at 23:01 UTC 98fe8cb5420c28fa8dcc3110b6c898848dd57e45
2 files changed +23 -6
requirements.txt
+1
@@ -9,3 +9,4 @@ xformers >= 0.0.19
9 fastapi
10 uvicorn
11 pydantic # Required for OpenAI server.
12 +fschat # Required for OpenAI ChatCompletion Endpoint.
vllm/entrypoints/openai/api_server.py
+22 -6
@@ -36,6 +36,7 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
36
37 logger = init_logger(__name__)
38 served_model = None
39 +chat_template = None
40 app = fastapi.FastAPI()
41
42
@@ -62,7 +63,7 @@ async def check_model(request) -> Optional[JSONResponse]:
63
64
65 async def get_gen_prompt(request) -> str:
65 - conv = get_conv_template(request.model)
66 + conv = get_conv_template(chat_template)
67 conv = Conversation(
68 name=conv.name,
69 system=conv.system,
@@ -553,13 +554,20 @@ if __name__ == "__main__":
554 type=json.loads,
555 default=["*"],
556 help="allowed headers")
557 + parser.add_argument("--served-model-name",
558 + type=str,
559 + default=None,
560 + help="The model name used in the API. If not "
561 + "specified, the model name will be the same as "
562 + "the huggingface name.")
563 parser.add_argument(
557 - "--served-model-name",
564 + "--chat-template",
565 type=str,
566 default=None,
560 - help="The model name used in the API. If not specified, "
561 - "the model name will be the same as the "
562 - "huggingface name.")
567 + help="The chat template name used in the ChatCompletion endpoint. If "
568 + "not specified, we use the API model name as the template name. See "
569 + "https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py "
570 + "for the list of available templates.")
571 parser = AsyncEngineArgs.add_cli_args(parser)
572 args = parser.parse_args()
573
@@ -573,7 +581,15 @@ if __name__ == "__main__":
581
582 logger.info(f"args: {args}")
583
576 - served_model = args.served_model_name or args.model
584 + if args.served_model_name is not None:
585 + served_model = args.served_model_name
586 + else:
587 + served_model = args.model
588 +
589 + if args.chat_template is not None:
590 + chat_template = args.chat_template
591 + else:
592 + chat_template = served_model
593
594 engine_args = AsyncEngineArgs.from_cli_args(args)
595 engine = AsyncLLMEngine.from_engine_args(engine_args)