@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 112 lines 3.46 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Example for starting a Gradio OpenAI Chatbot Webserver
4 Start vLLM API server:
5 vllm serve meta-llama/Llama-2-7b-chat-hf
6
7 Start Gradio OpenAI Chatbot Webserver:
8 python examples/applications/chatbot/gradio_openai_chatbot_webserver.py \
9 -m meta-llama/Llama-2-7b-chat-hf
10
11 Note that `pip install --upgrade gradio` is needed to run this example.
12 More details: https://github.com/gradio-app/gradio
13
14 If your antivirus software blocks the download of frpc for gradio,
15 you can install it manually by following these steps:
16
17 1. Download this file: https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_amd64
18 2. Rename the downloaded file to: frpc_linux_amd64_v0.3
19 3. Move the file to this location: /home/user/.cache/huggingface/gradio/frpc
20 """
21
22 import argparse
23
24 import gradio as gr
25 from openai import OpenAI
26
27
28 def predict(message, history, client, model_name, temp, stop_token_ids):
29 messages = [
30 {"role": "system", "content": "You are a great AI assistant."},
31 *history,
32 {"role": "user", "content": message},
33 ]
34
35 # Send request to OpenAI API (vLLM server)
36 stream = client.chat.completions.create(
37 model=model_name,
38 messages=messages,
39 temperature=temp,
40 stream=True,
41 extra_body={
42 "repetition_penalty": 1,
43 "stop_token_ids": [int(id.strip()) for id in stop_token_ids.split(",")]
44 if stop_token_ids
45 else [],
46 },
47 )
48
49 # Collect all chunks and concatenate them into a full message
50 full_message = ""
51 for chunk in stream:
52 full_message += chunk.choices[0].delta.content or ""
53
54 # Return the full message as a single response
55 return full_message
56
57
58 def parse_args():
59 parser = argparse.ArgumentParser(
60 description="Chatbot Interface with Customizable Parameters"
61 )
62 parser.add_argument(
63 "--model-url", type=str, default="http://localhost:8000/v1", help="Model URL"
64 )
65 parser.add_argument(
66 "-m", "--model", type=str, required=True, help="Model name for the chatbot"
67 )
68 parser.add_argument(
69 "--temp", type=float, default=0.8, help="Temperature for text generation"
70 )
71 parser.add_argument(
72 "--stop-token-ids", type=str, default="", help="Comma-separated stop token IDs"
73 )
74 parser.add_argument("--host", type=str, default=None)
75 parser.add_argument("--port", type=int, default=8001)
76 return parser.parse_args()
77
78
79 def build_gradio_interface(client, model_name, temp, stop_token_ids):
80 def chat_predict(message, history):
81 return predict(message, history, client, model_name, temp, stop_token_ids)
82
83 return gr.ChatInterface(
84 fn=chat_predict,
85 title="Chatbot Interface",
86 description="A simple chatbot powered by vLLM",
87 )
88
89
90 def main():
91 # Parse the arguments
92 args = parse_args()
93
94 # Set OpenAI's API key and API base to use vLLM's API server
95 openai_api_key = "EMPTY"
96 openai_api_base = args.model_url
97
98 # Create an OpenAI client
99 client = OpenAI(api_key=openai_api_key, base_url=openai_api_base)
100
101 # Define the Gradio chatbot interface using the predict function
102 gradio_interface = build_gradio_interface(
103 client, args.model, args.temp, args.stop_token_ids
104 )
105
106 gradio_interface.queue().launch(
107 server_name=args.host, server_port=args.port, share=True
108 )
109
110
111 if __name__ == "__main__":
112 main()