feature: attachment setup
missing - double user message when sending imgs - base 64 images implementation fix: fonts consistency
Alessandro committed
Nov 8, 2024 at 01:46 UTC
3c6a5bee64296559da274a9799afb139a7bdc98b
7 files changed
+294
-86
agent.py
+9
@@ -116,6 +116,7 @@ class AgentContext:
116
except Exception as e:
117
agent.handle_critical_exception(e)
118
119
+
120
@dataclass
121
class AgentConfig:
122
chat_model: BaseChatModel | BaseLLM
@@ -194,6 +195,7 @@ class LoopData:
195
self.message = ""
196
self.history_from = 0
197
self.history = []
198
+ self.attachments = [] # Add attachments field
199
200
201
# intervention exception class - skips rest of message loop iteration
@@ -251,6 +253,13 @@ class Agent:
253
254
printer = PrintStyle(italic=True, font_color="#b3ffd9", padding=False)
255
user_message = loop_data.message
256
+
257
+ # Include attachments in user message if available
258
+ if loop_data.attachments:
259
+ user_message += "\n" + "\n".join(loop_data.attachments) # Add attachments to message
260
+ loop_data.attachments = [] # Clear attachments after adding to message
261
+
262
+
263
await self.append_message(user_message, human=True)
264
265
# let the agent run message loop until he stops it with a response tool
python/extensions/message_loop_prompts/_30_include_attachments.py
new
+53
@@ -0,0 +1,53 @@
1
+from python.helpers.extension import Extension
2
+from agent import Agent, LoopData
3
+import os
4
+import io
5
+import base64
6
+from PIL import Image
7
+
8
+class IncludeAttachments(Extension):
9
+ async def execute(self, loop_data: LoopData = LoopData(), **kwargs):
10
+ # Check if there are attachments in agent data
11
+ attachments = self.agent.get_data('attachments') or []
12
+ if attachments:
13
+ loop_data.attachments = [] # Initialize attachments list for loop_data
14
+
15
+ # For each attachment, compress and encode the image
16
+ for attachment_path in attachments:
17
+ if os.path.exists(attachment_path):
18
+ # Prepare the base64-encoded image
19
+ compressed_image_base64 = self.compress_and_encode_image(attachment_path)
20
+ if compressed_image_base64:
21
+ # Append the image data to loop_data.attachments
22
+ loop_data.attachments.append(f"<image>{compressed_image_base64}</image>")
23
+
24
+ # Clear attachments from agent data
25
+ self.agent.set_data('attachments', [])
26
+
27
+ def compress_and_encode_image(self, image_path: str) -> str:
28
+ try:
29
+ # Open an image file
30
+ with Image.open(image_path) as img:
31
+ # Convert image to RGB if it's in RGBA mode
32
+ if img.mode in ('RGBA', 'P'):
33
+ img = img.convert('RGB')
34
+
35
+ # Resize the image to a reasonable size
36
+ max_dimension = 800 # You can adjust this value
37
+ img.thumbnail((max_dimension, max_dimension))
38
+
39
+ # Compress the image
40
+ buffered = io.BytesIO()
41
+ # Save as JPEG to ensure compression; you can adjust quality
42
+ img.save(buffered, format="JPEG", quality=70, optimize=True)
43
+ compressed_image = buffered.getvalue()
44
+
45
+ # Encode the compressed image in base64
46
+ return base64.b64encode(compressed_image).decode('utf-8')
47
+ except Exception as e:
48
+ print(f"Error compressing and encoding image {image_path}: {e}")
49
+ return ""
50
+
51
+ def estimate_token_count(self, message: str) -> int:
52
+ # Simple estimation: assume 4 characters per token
53
+ return len(message) // 4
run_ui.py
+66
-46
@@ -143,8 +143,7 @@ async def health_check():
143
# send message to agent (async UI)
144
@app.route("/msg", methods=["POST"])
145
async def handle_message_async():
146
- return await handle_message(False)
147
-
146
+ return await handle_message(False)
147
148
# send message to agent (synchronous API)
149
@app.route("/msg_sync", methods=["POST"])
@@ -153,50 +152,71 @@ async def handle_msg_sync():
152
153
154
async def handle_message(sync: bool):
156
- try:
157
-
158
- # data sent to the server
159
- input = request.get_json()
160
- text = input.get("text", "")
161
- ctxid = input.get("context", "")
162
- blev = input.get("broadcast", 1)
163
-
164
- # context instance - get or create
165
- context = get_context(ctxid)
166
-
167
- # print to console and log
168
- PrintStyle(
169
- background_color="#6C3483", font_color="white", bold=True, padding=True
170
- ).print(f"User message:")
171
- PrintStyle(font_color="white", padding=False).print(f"> {text}")
172
- context.log.log(type="user", heading="User message", content=text)
173
-
174
- if sync:
175
- context.communicate(text)
176
- result = await context.process.result() # type: ignore
177
- response = {
178
- "ok": True,
179
- "message": result,
180
- "context": context.id,
181
- }
182
- else:
183
-
184
- context.communicate(text)
185
- response = {
186
- "ok": True,
187
- "message": "Message received.",
188
- "context": context.id,
189
- }
190
-
191
- except Exception as e:
192
- response = {
193
- "ok": False,
194
- "message": str(e),
195
- }
196
- PrintStyle.error(str(e))
197
-
198
- # respond with json
199
- return jsonify(response)
155
+ try:
156
+ # Handle both JSON and multipart/form-data
157
+ if request.content_type.startswith('multipart/form-data'):
158
+ text = request.form.get('text', '')
159
+ ctxid = request.form.get('context', '')
160
+ attachments = request.files.getlist('attachments')
161
+ attachment_paths = []
162
+
163
+ upload_folder = os.path.join(os.getcwd(), 'work_dir', 'uploads')
164
+
165
+ if attachments:
166
+ os.makedirs(upload_folder, exist_ok=True)
167
+ for attachment in attachments:
168
+ filename = secure_filename(attachment.filename)
169
+ save_path = os.path.join(upload_folder, filename)
170
+ attachment.save(save_path)
171
+ attachment_paths.append(save_path)
172
+ else:
173
+ # Handle JSON request as before
174
+ input_data = request.get_json()
175
+ text = input_data.get('text', '')
176
+ ctxid = input_data.get('context', '')
177
+ attachment_paths = []
178
+
179
+ # Now process the message
180
+ message = text
181
+
182
+ # Obtain agent context
183
+ context = get_context(ctxid)
184
+
185
+ # Store attachments in agent data
186
+ context.agent0.set_data('attachments', attachment_paths)
187
+
188
+ # Print to console and log
189
+ PrintStyle(
190
+ background_color="#6C3483", font_color="white", bold=True, padding=True
191
+ ).print(f"User message:")
192
+ PrintStyle(font_color="white", padding=False).print(f"> {message}")
193
+ context.log.log(type="user", heading="User message", content=message)
194
+
195
+ if sync:
196
+ context.communicate(message)
197
+ result = await context.process.result() # type: ignore
198
+ response = {
199
+ "ok": True,
200
+ "message": result,
201
+ "context": context.id,
202
+ }
203
+ else:
204
+ context.communicate(message)
205
+ response = {
206
+ "ok": True,
207
+ "message": "Message received.",
208
+ "context": context.id,
209
+ }
210
+
211
+ except Exception as e:
212
+ response = {
213
+ "ok": False,
214
+ "message": str(e),
215
+ }
216
+ PrintStyle.error(str(e))
217
+
218
+ # respond with json
219
+ return jsonify(response)
220
221
222
# pausing/unpausing the agent
webui/index.css
+27
-5
@@ -807,16 +807,36 @@ pre {
807
opacity: 0.5;
808
}
809
810
+.attachments-container {
811
+ margin-top: 10px;
812
+ padding: 10px;
813
+ border-radius: 5px;
814
+}
815
+
816
+.message-attachment {
817
+ max-width: 100%;
818
+ max-height: 400px;
819
+ margin: 5px 0;
820
+ border-radius: 5px;
821
+ object-fit: contain;
822
+}
823
+
824
+.image-error {
825
+ border: 1px solid #ff0000;
826
+ padding: 10px;
827
+ color: #ff0000;
828
+}
829
+
830
/* Text input */
831
#chat-input {
832
flex-grow: 1;
833
min-height: 2.7rem;
814
- padding: var(--spacing-xs) var(--spacing-sm);
815
- background-color: var(--color-input);
834
+ padding: var(--spacing-sm) var(--spacing-sm);
835
+ padding-top: 0.70rem;
836
border: 1px solid var(--color-border);
837
border-radius: 8px;
838
resize: none;
819
- align-content: center;
839
+ align-content: start;
840
}
841
842
.input-row {
@@ -838,7 +858,7 @@ pre {
858
background: none;
859
border: none;
860
color: var(--color-text);
841
- font-family: Rubik;
861
+ font-family: "Rubik", Arial, Helvetica, sans-serif;
862
font-size: 0.7rem;
863
padding: 6px var(--spacing-sm);
864
cursor: pointer;
@@ -1284,7 +1304,7 @@ input:checked + .slider:before {
1304
.katex {
1305
font-family: Roboto Mono !important;
1306
line-height: 1rem !important;
1287
- font-size: var(--font-size-normal) !important;
1307
+ font-size: 3rem;
1308
}
1309
1310
/* Animations */
@@ -1316,6 +1336,8 @@ input:checked + .slider:before {
1336
#chat-input {
1337
min-height: 5.3rem;
1338
align-content: start;
1339
+
1340
+
1341
}
1342
1343
#chat-buttons-wrapper {
webui/index.js
+61
-11
@@ -60,31 +60,81 @@ function setupSidebarToggle() {
60
// Make sure to call this function
61
document.addEventListener('DOMContentLoaded', setupSidebarToggle);
62
63
+ // index.js
64
async function sendMessage() {
65
try {
66
const message = chatInput.value.trim();
66
- if (message) {
67
+ const inputAD = Alpine.$data(inputSection);
68
+ const attachments = inputAD.attachments;
69
+ const hasAttachments = attachments && attachments.length > 0;
70
+
71
+ if (message || hasAttachments) {
72
+ let response;
73
+ const messageId = generateGUID();
74
+
75
+ // Only render immediately for attachments
76
+ if (hasAttachments) {
77
+ const attachmentsWithUrls = attachments.map(attachment => ({
78
+ ...attachment,
79
+ url: URL.createObjectURL(attachment.file)
80
+ }));
81
+
82
+ // Only render if there's text content or it's an image-only message
83
+ setMessage(messageId, 'user', '', message, false, {
84
+ attachments: attachmentsWithUrls
85
+ });
86
+
87
+ const formData = new FormData();
88
+ formData.append('text', message);
89
+ formData.append('context', context);
90
+ formData.append('message_id', messageId);
91
68
- const response = await sendJsonData("/msg", { text: message, context });
92
+ for (let i = 0; i < attachments.length; i++) {
93
+ formData.append('attachments', attachments[i].file);
94
+ }
95
70
- if (!response) {
71
- toast("No response returned.", "error")
72
- } else if (!response.ok) {
73
- if (response.message) {
74
- toast(response.message, "error")
96
+ response = await fetch('/msg', {
97
+ method: 'POST',
98
+ body: formData
99
+ });
100
+ } else {
101
+ // For text-only messages, let polling handle the rendering
102
+ const data = {
103
+ text: message,
104
+ context,
105
+ message_id: messageId
106
+ };
107
+ response = await fetch('/msg', {
108
+ method: 'POST',
109
+ headers: {
110
+ 'Content-Type': 'application/json'
111
+ },
112
+ body: JSON.stringify(data)
113
+ });
114
+ }
115
+
116
+ // Handle response
117
+ const jsonResponse = await response.json();
118
+ if (!jsonResponse) {
119
+ toast("No response returned.", "error");
120
+ } else if (!jsonResponse.ok) {
121
+ if (jsonResponse.message) {
122
+ toast(jsonResponse.message, "error");
123
} else {
76
- toast("Undefined error.", "error")
124
+ toast("Undefined error.", "error");
125
}
126
} else {
79
- setContext(response.context)
127
+ setContext(jsonResponse.context);
128
}
129
82
- //setMessage('user', message);
130
+ // Clear input and attachments
131
chatInput.value = '';
132
+ inputAD.attachments = [];
133
+ inputAD.hasAttachments = false;
134
adjustTextareaHeight();
135
}
136
} catch (e) {
87
- toast(e.message, "error")
137
+ toast(e.message, "error");
138
}
139
}
140
webui/messages.js
+75
-21
@@ -27,6 +27,8 @@ export function getHandler(type) {
27
}
28
}
29
30
+
31
+// draw a message with a specific type
32
export function _drawMessage(messageContainer, heading, content, temp, followUp, kvps = null, messageClasses = [], contentClasses = []) {
33
34
const messageDiv = document.createElement('div');
@@ -60,10 +62,10 @@ export function _drawMessage(messageContainer, heading, content, temp, followUp,
62
if (window.renderMathInElement) {
63
renderMathInElement(spanElement, {
64
delimiters: [
63
- {left: "$$", right: "$$", display: true},
64
- {left: "\$$", right: "\$$", display: true},
65
+ {left: "$", right: "$", display: true},
66
+ {left: "\\$", right: "\\$", display: true},
67
{left: "$", right: "$", display: false},
66
- {left: "\$$", right: "\$$", display: false}
68
+ {left: "\\$", right: "\\$", display: false}
69
],
70
throwOnError: false // Prevent KaTeX from throwing errors
71
});
@@ -72,43 +74,85 @@ export function _drawMessage(messageContainer, heading, content, temp, followUp,
74
return messageDiv;
75
}
76
77
+
78
export function drawMessageDefault(messageContainer, id, type, heading, content, temp, kvps = null) {
76
- _drawMessage(messageContainer, heading, content, temp, false, kvps, ['message-ai', 'message-default'], ['msg-json']);
79
+ const messageContent = convertImageTags(content); // Convert image tags
80
+ _drawMessage(messageContainer, heading, messageContent, temp, false, kvps, ['message-ai', 'message-default'], ['msg-json']);
81
}
82
83
export function drawMessageAgent(messageContainer, id, type, heading, content, temp, kvps = null) {
80
- let kvpsFlat = null
84
+ let kvpsFlat = null;
85
if (kvps) {
82
- kvpsFlat = { ...kvps, ...kvps['tool_args'] || {} }
83
- delete kvpsFlat['tool_args']
86
+ kvpsFlat = { ...kvps, ...kvps['tool_args'] || {} };
87
+ delete kvpsFlat['tool_args'];
88
}
89
86
- _drawMessage(messageContainer, heading, content, temp, false, kvpsFlat, ['message-ai', 'message-agent'], ['msg-json']);
90
+ const messageContent = convertImageTags(content); // Convert image tags
91
+ _drawMessage(messageContainer, heading, messageContent, temp, false, kvpsFlat, ['message-ai', 'message-agent'], ['msg-json']);
92
}
93
94
export function drawMessageResponse(messageContainer, id, type, heading, content, temp, kvps = null) {
90
- _drawMessage(messageContainer, heading, content, temp, true, null, ['message-ai', 'message-agent-response']);
95
+ const messageContent = convertImageTags(content); // Convert image tags
96
+ _drawMessage(messageContainer, heading, messageContent, temp, true, null, ['message-ai', 'message-agent-response']);
97
}
98
99
export function drawMessageDelegation(messageContainer, id, type, heading, content, temp, kvps = null) {
94
- _drawMessage(messageContainer, heading, content, temp, true, kvps, ['message-ai', 'message-agent', 'message-agent-delegation']);
100
+ const messageContent = convertImageTags(content); // Convert image tags
101
+ _drawMessage(messageContainer, heading, messageContent, temp, true, kvps, ['message-ai', 'message-agent', 'message-agent-delegation']);
102
}
103
104
export function drawMessageUser(messageContainer, id, type, heading, content, temp, kvps = null) {
98
- _drawMessage(messageContainer, heading, content, temp, false, kvps, ['message-user']);
105
+ const hasContent = content && content.trim().length > 0;
106
+ const hasAttachments = kvps && kvps.attachments && kvps.attachments.length > 0;
107
+
108
+ // Only create message container if there's content or it's the initial message
109
+ if (hasContent || !messageContainer.hasChildNodes()) {
110
+ // Create the message with user heading and content
111
+ _drawMessage(messageContainer, "User message", content, temp, false, null, ['message-user'], []);
112
+ }
113
+
114
+ // Render image attachments for user messages
115
+ if (hasAttachments) {
116
+ const attachmentsContainer = document.createElement('div');
117
+ attachmentsContainer.classList.add('attachments-container');
118
+
119
+ // Add "Attachments" heading if there's text content
120
+ if (hasContent) {
121
+ const attachmentsHeading = document.createElement('h4');
122
+ attachmentsHeading.textContent = "Attachments";
123
+ attachmentsContainer.appendChild(attachmentsHeading);
124
+ }
125
+
126
+ kvps.attachments.forEach(attachment => {
127
+ const image = document.createElement('img');
128
+ if (attachment.url) {
129
+ image.src = attachment.url;
130
+ } else if (attachment.data) {
131
+ image.src = `data:image/jpeg;base64,${attachment.data}`;
132
+ }
133
+ image.alt = 'Attachment';
134
+ image.classList.add('message-attachment');
135
+ attachmentsContainer.appendChild(image);
136
+ });
137
+
138
+ messageContainer.appendChild(attachmentsContainer);
139
+ }
140
}
141
142
export function drawMessageTool(messageContainer, id, type, heading, content, temp, kvps = null) {
102
- _drawMessage(messageContainer, heading, content, temp, true, kvps, ['message-ai', 'message-tool'], ['msg-output']);
143
+ const messageContent = convertImageTags(content); // Convert image tags
144
+ _drawMessage(messageContainer, heading, messageContent, temp, true, kvps, ['message-ai', 'message-tool'], ['msg-output']);
145
}
146
147
export function drawMessageCodeExe(messageContainer, id, type, heading, content, temp, kvps = null) {
106
- _drawMessage(messageContainer, heading, content, temp, true, null, ['message-ai', 'message-code-exe']);
148
+ const messageContent = convertImageTags(content); // Convert image tags
149
+ _drawMessage(messageContainer, heading, messageContent, temp, true, null, ['message-ai', 'message-code-exe']);
150
}
151
152
export function drawMessageAgentPlain(classes, messageContainer, id, type, heading, content, temp, kvps = null) {
110
- _drawMessage(messageContainer, heading, content, temp, false, null, [...classes]);
111
- messageContainer.classList.add('center-container')
153
+ const messageContent = convertImageTags(content); // Convert image tags
154
+ _drawMessage(messageContainer, heading, messageContent, temp, false, null, [...classes]);
155
+ messageContainer.classList.add('center-container');
156
}
157
158
export function drawMessageInfo(messageContainer, id, type, heading, content, temp, kvps = null) {
@@ -116,12 +160,9 @@ export function drawMessageInfo(messageContainer, id, type, heading, content, te
160
}
161
162
export function drawMessageUtil(messageContainer, id, type, heading, content, temp, kvps = null) {
119
- //if kvps is not null and contains "query"
120
- if (kvps && kvps["query"]) {
121
- const a = 1+1
122
- }
123
- _drawMessage(messageContainer, heading, content, temp, false, kvps, ['message-util'], ['msg-json']);
124
- messageContainer.classList.add('center-container')
163
+ const messageContent = convertImageTags(content); // Convert image tags
164
+ _drawMessage(messageContainer, heading, messageContent, temp, false, kvps, ['message-util'], ['msg-json']);
165
+ messageContainer.classList.add('center-container');
166
}
167
168
export function drawMessageWarning(messageContainer, id, type, heading, content, temp, kvps = null) {
@@ -189,3 +230,16 @@ function convertToTitleCase(str) {
230
return match.toUpperCase(); // Capitalize the first letter of each word
231
});
232
}
233
+
234
+
235
+function convertImageTags(content) {
236
+ // Regular expression to match <image> tags and extract base64 content
237
+ const imageTagRegex = /<image>(.*?)<\/image>/g;
238
+
239
+ // Replace <image> tags with <img> tags with base64 source
240
+ const updatedContent = content.replace(imageTagRegex, (match, base64Content) => {
241
+ return `<img src="data:image/jpeg;base64,${base64Content}" alt="Image Attachment" style="max-width: 250px !important;"/>`;
242
+ });
243
+
244
+ return updatedContent;
245
+}
webui/settings.css
+3
-3
@@ -128,7 +128,7 @@ select {
128
padding: 0.5rem;
129
border: 1px solid #ddd;
130
border-radius: 0.25rem;
131
- font-family: Rubik;
131
+ font-family: "Rubik", Arial, Helvetica, sans-serif;
132
}
133
134
textarea {
@@ -212,7 +212,7 @@ select {
212
cursor: pointer;
213
border: none;
214
font-size: 0.875rem;
215
- font-family: Rubik;
215
+ font-family: "Rubik", Arial, Helvetica, sans-serif;
216
}
217
218
.btn-ok {
@@ -262,7 +262,7 @@ select {
262
background-color: white;
263
font-size: inherit;
264
cursor: pointer;
265
- font-family: Rubik;
265
+ font-family: "Rubik", Arial, Helvetica, sans-serif;
266
outline: none;
267
}
268