@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 97460585d9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add gradio chatbot for openai webserver (#2307)

arkohut committed Jan 12, 2024 at 11:45 UTC 97460585d9ae2c79fb625d0e4ad48f17b753a2da
1 file changed +81
examples/gradio_openai_chatbot_webserver.py new
+81
@@ -0,0 +1,81 @@
1 +import argparse
2 +from openai import OpenAI
3 +import gradio as gr
4 +
5 +# Argument parser setup
6 +parser = argparse.ArgumentParser(
7 + description='Chatbot Interface with Customizable Parameters')
8 +parser.add_argument('--model-url',
9 + type=str,
10 + default='http://localhost:8000/v1',
11 + help='Model URL')
12 +parser.add_argument('-m',
13 + '--model',
14 + type=str,
15 + required=True,
16 + help='Model name for the chatbot')
17 +parser.add_argument('--temp',
18 + type=float,
19 + default=0.8,
20 + help='Temperature for text generation')
21 +parser.add_argument('--stop-token-ids',
22 + type=str,
23 + default='',
24 + help='Comma-separated stop token IDs')
25 +parser.add_argument("--host", type=str, default=None)
26 +parser.add_argument("--port", type=int, default=8001)
27 +
28 +# Parse the arguments
29 +args = parser.parse_args()
30 +
31 +# Set OpenAI's API key and API base to use vLLM's API server.
32 +openai_api_key = "EMPTY"
33 +openai_api_base = args.model_url
34 +
35 +# Create an OpenAI client to interact with the API server
36 +client = OpenAI(
37 + api_key=openai_api_key,
38 + base_url=openai_api_base,
39 +)
40 +
41 +
42 +def predict(message, history):
43 + # Convert chat history to OpenAI format
44 + history_openai_format = [{
45 + "role": "system",
46 + "content": "You are a great ai assistant."
47 + }]
48 + for human, assistant in history:
49 + history_openai_format.append({"role": "user", "content": human})
50 + history_openai_format.append({
51 + "role": "assistant",
52 + "content": assistant
53 + })
54 + history_openai_format.append({"role": "user", "content": message})
55 +
56 + # Create a chat completion request and send it to the API server
57 + stream = client.chat.completions.create(
58 + model=args.model, # Model name to use
59 + messages=history_openai_format, # Chat history
60 + temperature=args.temp, # Temperature for text generation
61 + stream=True, # Stream response
62 + extra_body={
63 + 'repetition_penalty':
64 + 1,
65 + 'stop_token_ids': [
66 + int(id.strip()) for id in args.stop_token_ids.split(',')
67 + if id.strip()
68 + ] if args.stop_token_ids else []
69 + })
70 +
71 + # Read and return generated text from response stream
72 + partial_message = ""
73 + for chunk in stream:
74 + partial_message += (chunk.choices[0].delta.content or "")
75 + yield partial_message
76 +
77 +
78 +# Create and launch a chat interface with Gradio
79 +gr.ChatInterface(predict).queue().launch(server_name=args.host,
80 + server_port=args.port,
81 + share=True)