A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Server] Add option to specify chat template for chat endpoint (#345)
Zhuohan Li committed
Jul 3, 2023 at 23:01 UTC
98fe8cb5420c28fa8dcc3110b6c898848dd57e45
2 files changed
+23
-6
requirements.txt
+1
@@ -9,3 +9,4 @@ xformers >= 0.0.19
9
fastapi
10
uvicorn
11
pydantic # Required for OpenAI server.
12
+fschat # Required for OpenAI ChatCompletion Endpoint.
vllm/entrypoints/openai/api_server.py
+22
-6
@@ -36,6 +36,7 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
36
37
logger = init_logger(__name__)
38
served_model = None
39
+chat_template = None
40
app = fastapi.FastAPI()
41
42
@@ -62,7 +63,7 @@ async def check_model(request) -> Optional[JSONResponse]:
63
64
65
async def get_gen_prompt(request) -> str:
65
- conv = get_conv_template(request.model)
66
+ conv = get_conv_template(chat_template)
67
conv = Conversation(
68
name=conv.name,
69
system=conv.system,
@@ -553,13 +554,20 @@ if __name__ == "__main__":
554
type=json.loads,
555
default=["*"],
556
help="allowed headers")
557
+ parser.add_argument("--served-model-name",
558
+ type=str,
559
+ default=None,
560
+ help="The model name used in the API. If not "
561
+ "specified, the model name will be the same as "
562
+ "the huggingface name.")
563
parser.add_argument(
557
- "--served-model-name",
564
+ "--chat-template",
565
type=str,
566
default=None,
560
- help="The model name used in the API. If not specified, "
561
- "the model name will be the same as the "
562
- "huggingface name.")
567
+ help="The chat template name used in the ChatCompletion endpoint. If "
568
+ "not specified, we use the API model name as the template name. See "
569
+ "https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py "
570
+ "for the list of available templates.")
571
parser = AsyncEngineArgs.add_cli_args(parser)
572
args = parser.parse_args()
573
@@ -573,7 +581,15 @@ if __name__ == "__main__":
581
582
logger.info(f"args: {args}")
583
576
- served_model = args.served_model_name or args.model
584
+ if args.served_model_name is not None:
585
+ served_model = args.served_model_name
586
+ else:
587
+ served_model = args.model
588
+
589
+ if args.chat_template is not None:
590
+ chat_template = args.chat_template
591
+ else:
592
+ chat_template = served_model
593
594
engine_args = AsyncEngineArgs.from_cli_args(args)
595
engine = AsyncLLMEngine.from_engine_args(engine_args)