| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """vLLM Chat Assistant - A Streamlit Web Interface. |
| 4 | |
| 5 | A streamlined chat interface that quickly integrates |
| 6 | with vLLM API server. |
| 7 | |
| 8 | Features: |
| 9 | - Multiple chat sessions management |
| 10 | - Streaming response display |
| 11 | - Configurable API endpoint |
| 12 | - Real-time chat history |
| 13 | - Reasoning Display: Optional thinking process visualization |
| 14 | |
| 15 | Requirements: |
| 16 | pip install streamlit openai |
| 17 | |
| 18 | Usage: |
| 19 | # Start the app with default settings |
| 20 | streamlit run streamlit_openai_chatbot_webserver.py |
| 21 | |
| 22 | # Start with custom vLLM API endpoint |
| 23 | VLLM_API_BASE="http://your-server:8000/v1" \ |
| 24 | streamlit run streamlit_openai_chatbot_webserver.py |
| 25 | |
| 26 | # Enable debug mode |
| 27 | streamlit run streamlit_openai_chatbot_webserver.py \ |
| 28 | --logger.level=debug |
| 29 | """ |
| 30 | |
| 31 | import os |
| 32 | from datetime import datetime |
| 33 | |
| 34 | import streamlit as st |
| 35 | from openai import OpenAI |
| 36 | |
| 37 | # Get command line arguments from environment variables |
| 38 | openai_api_key = os.getenv("VLLM_API_KEY", "EMPTY") |
| 39 | openai_api_base = os.getenv("VLLM_API_BASE", "http://localhost:8000/v1") |
| 40 | |
| 41 | # Initialize session states for managing chat sessions |
| 42 | if "sessions" not in st.session_state: |
| 43 | st.session_state.sessions = {} |
| 44 | |
| 45 | if "current_session" not in st.session_state: |
| 46 | st.session_state.current_session = None |
| 47 | |
| 48 | if "messages" not in st.session_state: |
| 49 | st.session_state.messages = [] |
| 50 | |
| 51 | if "active_session" not in st.session_state: |
| 52 | st.session_state.active_session = None |
| 53 | |
| 54 | # Add new session state for reasoning |
| 55 | if "show_reasoning" not in st.session_state: |
| 56 | st.session_state.show_reasoning = {} |
| 57 | |
| 58 | # Initialize session state for API base URL |
| 59 | if "api_base_url" not in st.session_state: |
| 60 | st.session_state.api_base_url = openai_api_base |
| 61 | |
| 62 | |
| 63 | def create_new_chat_session(): |
| 64 | """Create a new chat session with timestamp as unique identifier. |
| 65 | |
| 66 | This function initializes a new chat session by: |
| 67 | 1. Generating a timestamp-based session ID |
| 68 | 2. Creating an empty message list for the new session |
| 69 | 3. Setting the new session as both current and active session |
| 70 | 4. Resetting the messages list for the new session |
| 71 | |
| 72 | Returns: |
| 73 | None |
| 74 | |
| 75 | Session State Updates: |
| 76 | - sessions: Adds new empty message list with timestamp key |
| 77 | - current_session: Sets to new session ID |
| 78 | - active_session: Sets to new session ID |
| 79 | - messages: Resets to empty list |
| 80 | |
| 81 | """ |
| 82 | session_id = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
| 83 | st.session_state.sessions[session_id] = [] |
| 84 | st.session_state.current_session = session_id |
| 85 | st.session_state.active_session = session_id |
| 86 | st.session_state.messages = [] |
| 87 | |
| 88 | |
| 89 | def switch_to_chat_session(session_id): |
| 90 | """Switch the active chat context to a different session. |
| 91 | |
| 92 | Args: |
| 93 | session_id (str): The timestamp ID of the session to switch to |
| 94 | |
| 95 | This function handles chat session switching by: |
| 96 | 1. Setting the specified session as current |
| 97 | 2. Updating the active session marker |
| 98 | 3. Loading the messages history from the specified session |
| 99 | |
| 100 | Session State Updates: |
| 101 | - current_session: Updated to specified session_id |
| 102 | - active_session: Updated to specified session_id |
| 103 | - messages: Loaded from sessions[session_id] |
| 104 | |
| 105 | """ |
| 106 | st.session_state.current_session = session_id |
| 107 | st.session_state.active_session = session_id |
| 108 | st.session_state.messages = st.session_state.sessions[session_id] |
| 109 | |
| 110 | |
| 111 | def get_llm_response(messages, model, reason, content_ph=None, reasoning_ph=None): |
| 112 | """Generate and stream LLM response with optional reasoning process. |
| 113 | |
| 114 | Args: |
| 115 | messages (list): List of conversation message dicts with 'role' and 'content' |
| 116 | model (str): The model identifier to use for generation |
| 117 | reason (bool): Whether to enable and display reasoning process |
| 118 | content_ph (streamlit.empty): Placeholder for streaming response content |
| 119 | reasoning_ph (streamlit.empty): Placeholder for streaming reasoning process |
| 120 | |
| 121 | Returns: |
| 122 | tuple: (str, str) |
| 123 | - First string contains the complete response text |
| 124 | - Second string contains the complete reasoning text (if enabled) |
| 125 | |
| 126 | Features: |
| 127 | - Streams both reasoning and response text in real-time |
| 128 | - Handles model API errors gracefully |
| 129 | - Supports live updating of thinking process |
| 130 | - Maintains separate content and reasoning displays |
| 131 | |
| 132 | Raises: |
| 133 | Exception: Wrapped in error message if API call fails |
| 134 | |
| 135 | Note: |
| 136 | The function uses streamlit placeholders for live updates. |
| 137 | When reason=True, the reasoning process appears above the response. |
| 138 | |
| 139 | """ |
| 140 | full_text = "" |
| 141 | think_text = "" |
| 142 | live_think = None |
| 143 | # Build request parameters |
| 144 | params = {"model": model, "messages": messages, "stream": True} |
| 145 | if reason: |
| 146 | params["extra_body"] = {"chat_template_kwargs": {"enable_thinking": True}} |
| 147 | |
| 148 | try: |
| 149 | response = client.chat.completions.create(**params) |
| 150 | if isinstance(response, str): |
| 151 | if content_ph: |
| 152 | content_ph.markdown(response) |
| 153 | return response, "" |
| 154 | |
| 155 | # Prepare reasoning expander above content |
| 156 | if reason and reasoning_ph: |
| 157 | exp = reasoning_ph.expander("💭 Thinking Process (live)", expanded=True) |
| 158 | live_think = exp.empty() |
| 159 | |
| 160 | # Stream chunks |
| 161 | for chunk in response: |
| 162 | delta = chunk.choices[0].delta |
| 163 | # Stream reasoning first |
| 164 | if reason and hasattr(delta, "reasoning") and live_think: |
| 165 | rc = delta.reasoning |
| 166 | if rc: |
| 167 | think_text += rc |
| 168 | live_think.markdown(think_text + "▌") |
| 169 | # Then stream content |
| 170 | if hasattr(delta, "content") and delta.content and content_ph: |
| 171 | full_text += delta.content |
| 172 | content_ph.markdown(full_text + "▌") |
| 173 | |
| 174 | # Finalize displays: reasoning remains above, content below |
| 175 | if reason and live_think: |
| 176 | live_think.markdown(think_text) |
| 177 | if content_ph: |
| 178 | content_ph.markdown(full_text) |
| 179 | |
| 180 | return full_text, think_text |
| 181 | except Exception as e: |
| 182 | st.error(f"Error details: {str(e)}") |
| 183 | return f"Error: {str(e)}", "" |
| 184 | |
| 185 | |
| 186 | # Sidebar - API Settings first |
| 187 | st.sidebar.title("API Settings") |
| 188 | new_api_base = st.sidebar.text_input( |
| 189 | "API Base URL:", value=st.session_state.api_base_url |
| 190 | ) |
| 191 | if new_api_base != st.session_state.api_base_url: |
| 192 | st.session_state.api_base_url = new_api_base |
| 193 | st.rerun() |
| 194 | |
| 195 | st.sidebar.divider() |
| 196 | |
| 197 | # Sidebar - Session Management |
| 198 | st.sidebar.title("Chat Sessions") |
| 199 | if st.sidebar.button("New Session"): |
| 200 | create_new_chat_session() |
| 201 | |
| 202 | |
| 203 | # Display all sessions in reverse chronological order |
| 204 | for session_id in sorted(st.session_state.sessions.keys(), reverse=True): |
| 205 | # Mark the active session with a pinned button |
| 206 | if session_id == st.session_state.active_session: |
| 207 | st.sidebar.button( |
| 208 | f"📍 {session_id}", |
| 209 | key=session_id, |
| 210 | type="primary", |
| 211 | on_click=switch_to_chat_session, |
| 212 | args=(session_id,), |
| 213 | ) |
| 214 | else: |
| 215 | st.sidebar.button( |
| 216 | f"Session {session_id}", |
| 217 | key=session_id, |
| 218 | on_click=switch_to_chat_session, |
| 219 | args=(session_id,), |
| 220 | ) |
| 221 | |
| 222 | # Main interface |
| 223 | st.title("vLLM Chat Assistant") |
| 224 | |
| 225 | # Initialize OpenAI client with API settings |
| 226 | client = OpenAI(api_key=openai_api_key, base_url=st.session_state.api_base_url) |
| 227 | |
| 228 | # Get and display current model id |
| 229 | models = client.models.list() |
| 230 | model = models.data[0].id |
| 231 | st.markdown(f"**Model**: {model}") |
| 232 | |
| 233 | # Initialize first session if none exists |
| 234 | if st.session_state.current_session is None: |
| 235 | create_new_chat_session() |
| 236 | st.session_state.active_session = st.session_state.current_session |
| 237 | |
| 238 | # Update the chat history display section |
| 239 | for idx, msg in enumerate(st.session_state.messages): |
| 240 | # Render user messages normally |
| 241 | if msg["role"] == "user": |
| 242 | with st.chat_message("user"): |
| 243 | st.write(msg["content"]) |
| 244 | # Render assistant messages with reasoning above |
| 245 | else: |
| 246 | # If reasoning exists for this assistant message, show it above the content |
| 247 | if idx in st.session_state.show_reasoning: |
| 248 | with st.expander("💭 Thinking Process", expanded=False): |
| 249 | st.markdown(st.session_state.show_reasoning[idx]) |
| 250 | with st.chat_message("assistant"): |
| 251 | st.write(msg["content"]) |
| 252 | |
| 253 | |
| 254 | # Setup & Cache reasoning support check |
| 255 | @st.cache_data(show_spinner=False) |
| 256 | def server_supports_reasoning(): |
| 257 | """Check if the current model supports reasoning capability. |
| 258 | |
| 259 | Returns: |
| 260 | bool: True if the model supports reasoning, False otherwise |
| 261 | |
| 262 | """ |
| 263 | resp = client.chat.completions.create( |
| 264 | model=model, |
| 265 | messages=[{"role": "user", "content": "Hi"}], |
| 266 | stream=False, |
| 267 | ) |
| 268 | return hasattr(resp.choices[0].message, "reasoning") and bool( |
| 269 | resp.choices[0].message.reasoning |
| 270 | ) |
| 271 | |
| 272 | |
| 273 | # Check support |
| 274 | supports_reasoning = server_supports_reasoning() |
| 275 | |
| 276 | # Add reasoning toggle in sidebar if supported |
| 277 | reason = False # Default to False |
| 278 | if supports_reasoning: |
| 279 | reason = st.sidebar.checkbox("Enable Reasoning", value=False) |
| 280 | else: |
| 281 | st.sidebar.markdown( |
| 282 | "<span style='color:gray;'>Reasoning unavailable for this model.</span>", |
| 283 | unsafe_allow_html=True, |
| 284 | ) |
| 285 | # reason remains False |
| 286 | |
| 287 | # Update the input handling section |
| 288 | if prompt := st.chat_input("Type your message here..."): |
| 289 | # Save and display user message |
| 290 | st.session_state.messages.append({"role": "user", "content": prompt}) |
| 291 | st.session_state.sessions[st.session_state.current_session] = ( |
| 292 | st.session_state.messages |
| 293 | ) |
| 294 | with st.chat_message("user"): |
| 295 | st.write(prompt) |
| 296 | |
| 297 | # Prepare LLM messages |
| 298 | msgs = [ |
| 299 | {"role": m["role"], "content": m["content"]} for m in st.session_state.messages |
| 300 | ] |
| 301 | |
| 302 | # Stream assistant response |
| 303 | with st.chat_message("assistant"): |
| 304 | # Placeholders: reasoning above, content below |
| 305 | reason_ph = st.empty() |
| 306 | content_ph = st.empty() |
| 307 | full, think = get_llm_response(msgs, model, reason, content_ph, reason_ph) |
| 308 | # Determine index for this new assistant message |
| 309 | message_index = len(st.session_state.messages) |
| 310 | # Save assistant reply |
| 311 | st.session_state.messages.append({"role": "assistant", "content": full}) |
| 312 | # Persist reasoning in session state if any |
| 313 | if reason and think: |
| 314 | st.session_state.show_reasoning[message_index] = think |