A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add gradio chatbot for openai webserver (#2307)
arkohut committed
Jan 12, 2024 at 11:45 UTC
97460585d9ae2c79fb625d0e4ad48f17b753a2da
1 file changed
+81
examples/gradio_openai_chatbot_webserver.py
new
+81
@@ -0,0 +1,81 @@
1
+import argparse
2
+from openai import OpenAI
3
+import gradio as gr
4
+
5
+# Argument parser setup
6
+parser = argparse.ArgumentParser(
7
+ description='Chatbot Interface with Customizable Parameters')
8
+parser.add_argument('--model-url',
9
+ type=str,
10
+ default='http://localhost:8000/v1',
11
+ help='Model URL')
12
+parser.add_argument('-m',
13
+ '--model',
14
+ type=str,
15
+ required=True,
16
+ help='Model name for the chatbot')
17
+parser.add_argument('--temp',
18
+ type=float,
19
+ default=0.8,
20
+ help='Temperature for text generation')
21
+parser.add_argument('--stop-token-ids',
22
+ type=str,
23
+ default='',
24
+ help='Comma-separated stop token IDs')
25
+parser.add_argument("--host", type=str, default=None)
26
+parser.add_argument("--port", type=int, default=8001)
27
+
28
+# Parse the arguments
29
+args = parser.parse_args()
30
+
31
+# Set OpenAI's API key and API base to use vLLM's API server.
32
+openai_api_key = "EMPTY"
33
+openai_api_base = args.model_url
34
+
35
+# Create an OpenAI client to interact with the API server
36
+client = OpenAI(
37
+ api_key=openai_api_key,
38
+ base_url=openai_api_base,
39
+)
40
+
41
+
42
+def predict(message, history):
43
+ # Convert chat history to OpenAI format
44
+ history_openai_format = [{
45
+ "role": "system",
46
+ "content": "You are a great ai assistant."
47
+ }]
48
+ for human, assistant in history:
49
+ history_openai_format.append({"role": "user", "content": human})
50
+ history_openai_format.append({
51
+ "role": "assistant",
52
+ "content": assistant
53
+ })
54
+ history_openai_format.append({"role": "user", "content": message})
55
+
56
+ # Create a chat completion request and send it to the API server
57
+ stream = client.chat.completions.create(
58
+ model=args.model, # Model name to use
59
+ messages=history_openai_format, # Chat history
60
+ temperature=args.temp, # Temperature for text generation
61
+ stream=True, # Stream response
62
+ extra_body={
63
+ 'repetition_penalty':
64
+ 1,
65
+ 'stop_token_ids': [
66
+ int(id.strip()) for id in args.stop_token_ids.split(',')
67
+ if id.strip()
68
+ ] if args.stop_token_ids else []
69
+ })
70
+
71
+ # Read and return generated text from response stream
72
+ partial_message = ""
73
+ for chunk in stream:
74
+ partial_message += (chunk.choices[0].delta.content or "")
75
+ yield partial_message
76
+
77
+
78
+# Create and launch a chat interface with Gradio
79
+gr.ChatInterface(predict).queue().launch(server_name=args.host,
80
+ server_port=args.port,
81
+ share=True)