@samitouri / QOSami_Agents_Zero-2 / commits / a57f0c11

feature: attachments preview and sending (file, code, imgs)

Alessandro committed Nov 12, 2024 at 15:02 UTC a57f0c11988b06efdc6b8da8dad44969996d4b92
8 files changed +649 -329
python/extensions/message_loop_prompts/_30_include_attachments.py
+35 -48
@@ -1,53 +1,40 @@
1 +# python/extensions/monologue_start/include_attachments.py
2 from python.helpers.extension import Extension
3 +from python.helpers.attachment_manager import AttachmentManager
4 from agent import Agent, LoopData
5 import os
4 -import io
5 -import base64
6 -from PIL import Image
6
7 class IncludeAttachments(Extension):
9 - async def execute(self, loop_data: LoopData = LoopData(), **kwargs):
10 - # Check if there are attachments in agent data
11 - attachments = self.agent.get_data('attachments') or []
12 - if attachments:
13 - loop_data.attachments = [] # Initialize attachments list for loop_data
14 -
15 - # For each attachment, compress and encode the image
16 - for attachment_path in attachments:
17 - if os.path.exists(attachment_path):
18 - # Prepare the base64-encoded image
19 - compressed_image_base64 = self.compress_and_encode_image(attachment_path)
20 - if compressed_image_base64:
21 - # Append the image data to loop_data.attachments
22 - loop_data.attachments.append(f"<image>{compressed_image_base64}</image>")
23 -
24 - # Clear attachments from agent data
25 - self.agent.set_data('attachments', [])
26 -
27 - def compress_and_encode_image(self, image_path: str) -> str:
28 - try:
29 - # Open an image file
30 - with Image.open(image_path) as img:
31 - # Convert image to RGB if it's in RGBA mode
32 - if img.mode in ('RGBA', 'P'):
33 - img = img.convert('RGB')
34 -
35 - # Resize the image to a reasonable size
36 - max_dimension = 800 # You can adjust this value
37 - img.thumbnail((max_dimension, max_dimension))
38 -
39 - # Compress the image
40 - buffered = io.BytesIO()
41 - # Save as JPEG to ensure compression; you can adjust quality
42 - img.save(buffered, format="JPEG", quality=70, optimize=True)
43 - compressed_image = buffered.getvalue()
44 -
45 - # Encode the compressed image in base64
46 - return base64.b64encode(compressed_image).decode('utf-8')
47 - except Exception as e:
48 - print(f"Error compressing and encoding image {image_path}: {e}")
49 - return ""
50 -
51 - def estimate_token_count(self, message: str) -> int:
52 - # Simple estimation: assume 4 characters per token
53 - return len(message) // 4
8 + async def execute(self, loop_data: LoopData = LoopData(), **kwargs):
9 + attachments = self.agent.get_data('attachments') or []
10 + if attachments:
11 + loop_data.attachments = []
12 + file_manager = AttachmentManager(os.path.join(os.getcwd(), 'work_dir'))
13 +
14 + for attachment in attachments:
15 + if os.path.exists(attachment):
16 + filename = os.path.basename(attachment)
17 + file_type = file_manager.get_file_type(filename)
18 +
19 + attachment_html = f'<div class="attachment-item attachment-{file_type}">'
20 + if file_type == 'image':
21 + preview = file_manager.generate_image_preview(attachment)
22 + if preview:
23 + attachment_html += f'<img src="data:image/jpeg;base64,{preview}" alt="{filename}" class="attachment-preview"/>'
24 + else:
25 + # Add placeholder for non-image files
26 + attachment_html += f'<div class="attachment-placeholder">{file_type.upper()}</div>'
27 +
28 + # Add filename and extension badge
29 + ext = file_manager.get_file_extension(filename)
30 + attachment_html += f'''
31 + <div class="attachment-info">
32 + <span class="attachment-name">{filename}</span>
33 + <span class="attachment-badge">{ext}</span>
34 + </div>
35 + </div>'''
36 +
37 + loop_data.attachments.append(attachment_html)
38 +
39 + # Clear attachments after processing
40 + self.agent.set_data('attachments', [])
\ No newline at end of file
python/helpers/attachment_manager.py new
+91
@@ -0,0 +1,91 @@
1 +import os
2 +import io
3 +import base64
4 +from PIL import Image
5 +from typing import Dict, List, Optional, Tuple
6 +from werkzeug.utils import secure_filename
7 +
8 +class AttachmentManager:
9 + ALLOWED_EXTENSIONS = {
10 + 'image': {'jpg', 'jpeg', 'png', 'bmp'},
11 + 'code': {'py', 'js', 'sh', 'html', 'css'},
12 + 'document': {'md', 'pdf', 'txt', 'csv', 'json'}
13 + }
14 +
15 + def __init__(self, work_dir: str):
16 + self.work_dir = work_dir
17 + os.makedirs(work_dir, exist_ok=True)
18 +
19 + def is_allowed_file(self, filename: str) -> bool:
20 + ext = self.get_file_extension(filename)
21 + all_allowed = set().union(*self.ALLOWED_EXTENSIONS.values())
22 + return ext in all_allowed
23 +
24 + def get_file_type(self, filename: str) -> str:
25 + ext = self.get_file_extension(filename)
26 + for file_type, extensions in self.ALLOWED_EXTENSIONS.items():
27 + if ext in extensions:
28 + return file_type
29 + return 'unknown'
30 +
31 + @staticmethod
32 + def get_file_extension(filename: str) -> str:
33 + return filename.rsplit('.', 1)[1].lower() if '.' in filename else ''
34 +
35 + def validate_mime_type(self, file) -> bool:
36 + try:
37 + mime_type = file.content_type
38 + return mime_type.split('/')[0] in ['image', 'text', 'application']
39 + except AttributeError:
40 + return False
41 +
42 + def save_file(self, file, filename: str) -> Tuple[str, Dict]:
43 + """Save file and return path and metadata"""
44 + try:
45 + filename = secure_filename(filename)
46 + if not filename:
47 + raise ValueError("Invalid filename")
48 +
49 + file_path = os.path.join(self.work_dir, filename)
50 +
51 + file_type = self.get_file_type(filename)
52 + metadata = {
53 + 'filename': filename,
54 + 'type': file_type,
55 + 'extension': self.get_file_extension(filename),
56 + 'preview': None
57 + }
58 +
59 + # Save file
60 + file.save(file_path)
61 +
62 + # Generate preview for images
63 + if file_type == 'image':
64 + metadata['preview'] = self.generate_image_preview(file_path)
65 +
66 + return file_path, metadata
67 +
68 + except Exception as e:
69 + print(f"Error saving file {filename}: {e}")
70 + return None, {} # type: ignore
71 +
72 + def generate_image_preview(self, image_path: str, max_size: int = 800) -> Optional[str]:
73 + try:
74 + with Image.open(image_path) as img:
75 + # Convert image if needed
76 + if img.mode in ('RGBA', 'P'):
77 + img = img.convert('RGB')
78 +
79 + # Resize for preview
80 + img.thumbnail((max_size, max_size))
81 +
82 + # Save to buffer
83 + buffer = io.BytesIO()
84 + img.save(buffer, format="JPEG", quality=70, optimize=True)
85 +
86 + # Convert to base64
87 + return base64.b64encode(buffer.getvalue()).decode('utf-8')
88 + except Exception as e:
89 + print(f"Error generating preview for {image_path}: {e}")
90 + return None
91 +
\ No newline at end of file
python/helpers/log.py
+155 -154
@@ -4,165 +4,166 @@ from typing import Any, Literal, Optional, Dict
4 import uuid
5 from collections import OrderedDict # Import OrderedDict
6
7 -
7 Type = Literal[
9 - "agent",
10 - "code_exe",
11 - "error",
12 - "hint",
13 - "info",
14 - "progress",
15 - "response",
16 - "tool",
17 - "user",
18 - "util",
19 - "warning",
8 + "agent",
9 + "code_exe",
10 + "error",
11 + "hint",
12 + "info",
13 + "progress",
14 + "response",
15 + "tool",
16 + "user",
17 + "util",
18 + "warning",
19 ]
20
22 -
21 @dataclass
22 class LogItem:
25 - log: "Log"
26 - no: int
27 - type: str
28 - heading: str
29 - content: str
30 - temp: bool
31 - kvps: Optional[OrderedDict] = None # Use OrderedDict for kvps
32 - guid: str = ""
33 -
34 - def __post_init__(self):
35 - self.guid = self.log.guid
36 -
37 - def update(
38 - self,
39 - type: Type | None = None,
40 - heading: str | None = None,
41 - content: str | None = None,
42 - kvps: dict | None = None,
43 - temp: bool | None = None,
44 - **kwargs,
45 - ):
46 - if self.guid == self.log.guid:
47 - self.log.update_item(
48 - self.no,
49 - type=type,
50 - heading=heading,
51 - content=content,
52 - kvps=kvps,
53 - temp=temp,
54 - **kwargs,
55 - )
56 -
57 - def stream(self, heading: str | None = None, content: str | None = None, **kwargs):
58 - if heading is not None:
59 - self.update(heading=self.heading + heading)
60 - if content is not None:
61 - self.update(content=self.content + content)
62 -
63 - for k, v in kwargs.items():
64 - prev = self.kvps.get(k, "") if self.kvps else ""
65 - self.update(**{k: prev + v})
66 -
67 - def output(self):
68 - return {
69 - "no": self.no,
70 - "type": self.type,
71 - "heading": self.heading,
72 - "content": self.content,
73 - "temp": self.temp,
74 - "kvps": self.kvps,
75 - }
76 -
23 + log: "Log"
24 + no: int
25 + type: str
26 + heading: str
27 + content: str
28 + temp: bool
29 + kvps: Optional[OrderedDict] = None # Use OrderedDict for kvps
30 + id: Optional[str] = None # Add id field
31 + guid: str = ""
32 +
33 + def __post_init__(self):
34 + self.guid = self.log.guid
35 +
36 + def update(
37 + self,
38 + type: Type | None = None,
39 + heading: str | None = None,
40 + content: str | None = None,
41 + kvps: dict | None = None,
42 + temp: bool | None = None,
43 + **kwargs,
44 + ):
45 + if self.guid == self.log.guid:
46 + self.log.update_item(
47 + self.no,
48 + type=type,
49 + heading=heading,
50 + content=content,
51 + kvps=kvps,
52 + temp=temp,
53 + **kwargs,
54 + )
55 +
56 + def stream(self, heading: str | None = None, content: str | None = None, **kwargs):
57 + if heading is not None:
58 + self.update(heading=self.heading + heading)
59 + if content is not None:
60 + self.update(content=self.content + content)
61 +
62 + for k, v in kwargs.items():
63 + prev = self.kvps.get(k, "") if self.kvps else ""
64 + self.update(**{k: prev + v})
65 +
66 + def output(self):
67 + return {
68 + "no": self.no,
69 + "id": self.id, # Include id in output
70 + "type": self.type,
71 + "heading": self.heading,
72 + "content": self.content,
73 + "temp": self.temp,
74 + "kvps": self.kvps,
75 + }
76
77 class Log:
78
80 - def __init__(self):
81 - self.guid: str = str(uuid.uuid4())
82 - self.updates: list[int] = []
83 - self.logs: list[LogItem] = []
84 - self.progress = ""
85 - self.progress_no = 0
86 -
87 - def log(
88 - self,
89 - type: Type,
90 - heading: str | None = None,
91 - content: str | None = None,
92 - kvps: dict | None = None,
93 - temp: bool | None = None,
94 - ) -> LogItem:
95 - # Use OrderedDict if kvps is provided
96 - if kvps is not None:
97 - kvps = OrderedDict(kvps)
98 - item = LogItem(
99 - log=self,
100 - no=len(self.logs),
101 - type=type,
102 - heading=heading or "",
103 - content=content or "",
104 - kvps=kvps,
105 - temp=temp or False,
106 - )
107 - self.logs.append(item)
108 - self.updates += [item.no]
109 - if heading and item.no >= self.progress_no:
110 - self.progress = heading
111 - self.progress_no = item.no
112 - return item
113 -
114 - def update_item(
115 - self,
116 - no: int,
117 - type: str | None = None,
118 - heading: str | None = None,
119 - content: str | None = None,
120 - kvps: dict | None = None,
121 - temp: bool | None = None,
122 - **kwargs,
123 - ):
124 - item = self.logs[no]
125 - if type is not None:
126 - item.type = type
127 - if heading is not None:
128 - item.heading = heading
129 - if no >= self.progress_no:
130 - self.progress = heading
131 - self.progress_no = no
132 - if content is not None:
133 - item.content = content
134 - if kvps is not None:
135 - item.kvps = OrderedDict(kvps) # Use OrderedDict to keep the order
136 -
137 - if temp is not None:
138 - item.temp = temp
139 -
140 - if kwargs:
141 - if item.kvps is None:
142 - item.kvps = OrderedDict() # Ensure kvps is an OrderedDict
143 - for k, v in kwargs.items():
144 - item.kvps[k] = v
145 -
146 - self.updates += [item.no]
147 -
148 - def output(self, start=None, end=None):
149 - if start is None:
150 - start = 0
151 - if end is None:
152 - end = len(self.updates)
153 -
154 - out = []
155 - seen = set()
156 - for update in self.updates[start:end]:
157 - if update not in seen:
158 - out.append(self.logs[update].output())
159 - seen.add(update)
160 -
161 - return out
162 -
163 - def reset(self):
164 - self.guid = str(uuid.uuid4())
165 - self.updates = []
166 - self.logs = []
167 - self.progress = ""
168 - self.progress_no = 0
79 + def __init__(self):
80 + self.guid: str = str(uuid.uuid4())
81 + self.updates: list[int] = []
82 + self.logs: list[LogItem] = []
83 + self.progress = ""
84 + self.progress_no = 0
85 +
86 + def log(
87 + self,
88 + type: Type,
89 + heading: str | None = None,
90 + content: str | None = None,
91 + kvps: dict | None = None,
92 + temp: bool | None = None,
93 + id: Optional[str] = None, # Add id parameter
94 + ) -> LogItem:
95 + # Use OrderedDict if kvps is provided
96 + if kvps is not None:
97 + kvps = OrderedDict(kvps)
98 + item = LogItem(
99 + log=self,
100 + no=len(self.logs),
101 + type=type,
102 + heading=heading or "",
103 + content=content or "",
104 + kvps=kvps,
105 + temp=temp or False,
106 + id=id, # Pass id to LogItem
107 + )
108 + self.logs.append(item)
109 + self.updates += [item.no]
110 + if heading and item.no >= self.progress_no:
111 + self.progress = heading
112 + self.progress_no = item.no
113 + return item
114 +
115 + def update_item(
116 + self,
117 + no: int,
118 + type: str | None = None,
119 + heading: str | None = None,
120 + content: str | None = None,
121 + kvps: dict | None = None,
122 + temp: bool | None = None,
123 + **kwargs,
124 + ):
125 + item = self.logs[no]
126 + if type is not None:
127 + item.type = type
128 + if heading is not None:
129 + item.heading = heading
130 + if no >= self.progress_no:
131 + self.progress = heading
132 + self.progress_no = no
133 + if content is not None:
134 + item.content = content
135 + if kvps is not None:
136 + item.kvps = OrderedDict(kvps) # Use OrderedDict to keep the order
137 +
138 + if temp is not None:
139 + item.temp = temp
140 +
141 + if kwargs:
142 + if item.kvps is None:
143 + item.kvps = OrderedDict() # Ensure kvps is an OrderedDict
144 + for k, v in kwargs.items():
145 + item.kvps[k] = v
146 +
147 + self.updates += [item.no]
148 +
149 + def output(self, start=None, end=None):
150 + if start is None:
151 + start = 0
152 + if end is None:
153 + end = len(self.updates)
154 +
155 + out = []
156 + seen = set()
157 + for update in self.updates[start:end]:
158 + if update not in seen:
159 + out.append(self.logs[update].output())
160 + seen.add(update)
161 +
162 + return out
163 +
164 + def reset(self):
165 + self.guid = str(uuid.uuid4())
166 + self.updates = []
167 + self.logs = []
168 + self.progress = ""
169 + self.progress_no = 0
\ No newline at end of file
run_ui.py
+12 -2
@@ -225,13 +225,13 @@ async def handle_message_async():
225 async def handle_msg_sync():
226 return await handle_message(True)
227
228 -
228 async def handle_message(sync: bool):
229 try:
230 # Handle both JSON and multipart/form-data
231 if request.content_type.startswith('multipart/form-data'):
232 text = request.form.get('text', '')
233 ctxid = request.form.get('context', '')
234 + message_id = request.form.get('message_id', None)
235 attachments = request.files.getlist('attachments')
236 attachment_paths = []
237
@@ -249,6 +249,7 @@ async def handle_message(sync: bool):
249 input_data = request.get_json()
250 text = input_data.get('text', '')
251 ctxid = input_data.get('context', '')
252 + message_id = input_data.get('message_id', None)
253 attachment_paths = []
254
255 # Now process the message
@@ -260,12 +261,21 @@ async def handle_message(sync: bool):
261 # Store attachments in agent data
262 context.agent0.set_data('attachments', attachment_paths)
263
264 + # Prepare attachment filenames for logging
265 + attachment_filenames = [os.path.basename(path) for path in attachment_paths] if attachment_paths else []
266 +
267 # Print to console and log
268 PrintStyle(
269 background_color="#6C3483", font_color="white", bold=True, padding=True
270 ).print(f"User message:")
271 PrintStyle(font_color="white", padding=False).print(f"> {message}")
268 - context.log.log(type="user", heading="User message", content=message)
272 + if attachment_filenames:
273 + PrintStyle(font_color="white", padding=False).print("Attachments:")
274 + for filename in attachment_filenames:
275 + PrintStyle(font_color="white", padding=False).print(f"- {filename}")
276 +
277 + # Log the message with message_id and attachments
278 + context.log.log(type="user", heading="User message", content=message, kvps={'attachments': attachment_filenames}, id=message_id)
279
280 if sync:
281 context.communicate(message)
webui/index.css
+190 -12
@@ -516,6 +516,14 @@ pre {
516 text-align: end;
517 }
518
519 +.message-user > div {
520 +padding-top: var(--spacing-xs);
521 +font-family: 'Roboto Mono', monospace;
522 +font-optical-sizing: auto;
523 +-webkit-font-optical-sizing: auto;
524 +font-size: var(--font-size-small)
525 +}
526 +
527 .message-ai {
528 border-bottom-left-radius: var(--spacing-xs);
529 }
@@ -785,7 +793,6 @@ pre {
793 align-items: center;
794 gap: var(--spacing-xs);
795 }
788 -
796 /* Attachment icon */
797 .attachment-wrapper {
798 position: relative;
@@ -796,7 +803,7 @@ pre {
803 cursor: pointer;
804 color: var(--color-text);
805 opacity: 0.7;
799 - transition: opacity 0.2s;
806 + transition: opacity 0.2s ease;
807 display: flex;
808 align-items: center;
809 }
@@ -809,24 +816,195 @@ pre {
816 opacity: 0.5;
817 }
818
819 +/* Message attachments styles */
820 .attachments-container {
813 - margin-top: 10px;
814 - padding: 10px;
815 - border-radius: 5px;
821 + margin-top: 0.5em;
822 + display: flex;
823 + flex-direction: column;
824 + gap: 0.5em;
825 +}
826 +
827 +.attachment-item {
828 + display: flex;
829 + align-items: center;
830 + gap: 1em;
831 + background: var(--color-background);
832 + padding: 0.5em;
833 + border-radius: 4px;
834 + transition: background-color 0.2s ease;
835 +}
836 +
837 +.attachment-item:hover {
838 + background: var(--color-secondary-dark);
839 +}
840 +
841 +.attachment-item.file-type {
842 + background: var(--color-background);
843 }
844
818 -.message-attachment {
819 - max-width: 100%;
820 - max-height: 400px;
821 - margin: 5px 0;
822 - border-radius: 5px;
845 +.attachment-item:hover {
846 + background: var(--color-secondary-dark);
847 +}
848 +
849 +.attachment-preview {
850 + max-width: 100px;
851 + max-height: 100px;
852 + border-radius: 4px;
853 object-fit: contain;
854 }
855
856 +.attachment-image .attachment-preview {
857 + margin-right: 8px;
858 +}
859 +
860 +.attachment-info,
861 +.file-info {
862 + display: flex;
863 + align-items: center;
864 + gap: 8px;
865 +}
866 +
867 +.file-info {
868 + display: flex;
869 + align-items: center;
870 + gap: 0.5em;
871 +}
872 +
873 +.attachment-name,
874 +.filename,
875 +.file-name {
876 + font-size: 0.9em;
877 + color: var(--color-text);
878 + word-break: break-word;
879 +}
880 +
881 +.attachment-ext,
882 +.extension,
883 +.file-ext {
884 + background: var(--color-primary);
885 + color: var(--color-text);
886 + padding: 2px 6px;
887 + border-radius: 4px;
888 + font-size: 0.8em;
889 + text-transform: uppercase;
890 + white-space: nowrap;
891 +}
892 +
893 +/* Preview section styles */
894 +.preview-section {
895 + display: flex;
896 + flex-wrap: wrap;
897 + gap: 8px;
898 + margin-bottom: 10px;
899 + padding: var(--spacing-xs);
900 +}
901 +
902 +.preview-item {
903 + position: relative;
904 + background: var(--color-secondary);
905 + border-radius: 8px;
906 + padding: 8px;
907 + max-width: 200px;
908 + display: flex;
909 + align-items: center;
910 + gap: 8px;
911 + transition: background-color 0.2s ease;
912 +}
913 +
914 +.preview-item:hover {
915 + background: var(--color-secondary-dark);
916 +}
917 +
918 +.preview-item.image-preview img {
919 + max-height: 100px;
920 + object-fit: cover;
921 + border-radius: 4px;
922 +}
923 +
924 +.image-wrapper {
925 + width: 100px;
926 + height: 100px;
927 + display: flex;
928 + align-items: center;
929 + justify-content: center;
930 +}
931 +
932 +.file-preview {
933 + display: flex;
934 + align-items: center;
935 + gap: 0.5em;
936 +}
937 +
938 +.extension {
939 + background: var(--color-primary);
940 + color: var(--color-text);
941 + padding: 2px 6px;
942 + border-radius: 4px;
943 + font-size: 0.8em;
944 + text-transform: uppercase;
945 +}
946 +
947 +.file-preview:hover {
948 + background: var(--color-secondary-dark);
949 +}
950 +
951 +.remove-attachment {
952 + position: absolute;
953 + top: -6px;
954 + right: -6px;
955 + background-color: var(--color-primary);
956 + /* opacity: 0.6; */
957 + color: white;
958 + border: none;
959 + border-radius: 50%;
960 + width: 20px;
961 + height: 20px;
962 + cursor: pointer;
963 + display: flex;
964 + align-items: center;
965 + justify-content: center;
966 + transition: background-color 0.2s ease, transform 0.1s ease;
967 + z-index: 1;
968 +}
969 +
970 +.remove-attachment:hover {
971 + background-color: var(--color-accent);
972 + transform: scale(1.1);
973 +}
974 +
975 +.remove-attachment:active {
976 + transform: scale(0.9);
977 +}
978 +
979 +/* File type specific styles
980 +.extension[data-type="image"] {
981 + background: #4CAF50;
982 +}
983 +
984 +.extension[data-type="code"] {
985 + background: #2196F3;
986 +}
987 +
988 +.extension[data-type="document"] {
989 + background: #FF9800;
990 +}
991 +
992 +*/
993 +
994 +/* Error handling */
995 .image-error {
827 - border: 1px solid #ff0000;
996 + border: 1px solid var(--color-error);
997 padding: 10px;
829 - color: #ff0000;
998 + color: var(--color-error);
999 + border-radius: 4px;
1000 + font-size: 0.9em;
1001 + display: flex;
1002 + align-items: center;
1003 + gap: 8px;
1004 +}
1005 +
1006 +.image-error::before {
1007 + content: "⚠️";
1008 }
1009
1010 /* Text input */
webui/index.html
+55 -26
@@ -195,34 +195,63 @@
195 alert('Maximum 4 attachments allowed');
196 return;
197 }
198 -
198 +
199 Array.from(files).forEach(file => {
200 - if (file.type.startsWith('image/')) {
201 - const reader = new FileReader();
202 - reader.onload = e => {
200 + const ext = file.name.split('.').pop().toLowerCase();
201 + const allowedExts = new Set(['jpg', 'jpeg', 'png', 'bmp', 'md', 'py', 'js', 'sh',
202 + 'html', 'css', 'pdf', 'txt', 'csv', 'json']);
203 +
204 + if (allowedExts.has(ext)) {
205 + const isImage = ['jpg', 'jpeg', 'png', 'bmp'].includes(ext);
206 +
207 + if (isImage) {
208 + // Handle image preview
209 + const reader = new FileReader();
210 + reader.onload = e => {
211 + this.attachments.push({
212 + file: file,
213 + url: e.target.result,
214 + type: 'image',
215 + name: file.name,
216 + extension: ext
217 + });
218 + this.hasAttachments = true;
219 + };
220 + reader.readAsDataURL(file);
221 + } else {
222 + // Handle other file types
223 this.attachments.push({
224 file: file,
205 - url: e.target.result
225 + type: 'file',
226 + name: file.name,
227 + extension: ext
228 });
229 this.hasAttachments = true;
208 - };
209 - reader.readAsDataURL(file);
230 + }
231 }
232 });
233 }
234 }">
214 -
215 - <!-- Image preview section -->
216 - <div x-show="hasAttachments" class="image-preview-section">
217 - <template x-for="(image, index) in attachments" :key="index">
218 - <div class="preview-item">
219 - <img :src="image.url" alt="Preview">
220 - <button @click="attachments.splice(index, 1); hasAttachments = attachments.length > 0"
221 - class="remove-image">&times;</button>
235 +
236 + <!-- Preview section -->
237 + <div x-show="hasAttachments" class="preview-section">
238 + <template x-for="(attachment, index) in attachments" :key="index">
239 + <div class="preview-item" :class="{'image-preview': attachment.type === 'image'}">
240 + <template x-if="attachment.type === 'image'">
241 + <img :src="attachment.url" :alt="attachment.name">
242 + </template>
243 + <template x-if="attachment.type === 'file'">
244 + <div class="file-preview">
245 + <span class="filename" x-text="attachment.name"></span>
246 + <span class="extension" x-text="attachment.extension.toUpperCase()"></span>
247 + </div>
248 + </template>
249 + <button @click="attachments.splice(index, 1); hasAttachments = attachments.length > 0"
250 + class="remove-attachment">&times;</button>
251 </div>
252 </template>
253 </div>
225 -
254 +
255 <!-- Top row with input and buttons -->
256 <div class="input-row">
257 <!-- Attachment icon with tooltip -->
@@ -235,20 +264,20 @@
264 d="M16.5 6v11.5c0 2.21-1.79 4-4 4s-4-1.79-4-4V5c0-1.38 1.12-2.5 2.5-2.5s2.5 1.12 2.5 2.5v10.5c0 .55-.45 1-1 1s-1-.45-1-1V6H10v9.5c0 1.38 1.12 2.5 2.5 2.5s2.5-1.12 2.5-2.5V5c0-2.21-1.79-4-4-4S7 2.79 7 5v12.5c0 3.04 2.46 5.5 5.5 5.5s5.5-2.46 5.5-5.5V6h-1.5z" />
265 </svg>
266 </label>
238 - <input type="file" id="file-input" accept="png, jpg, jpeg, txt, pdf, csv, html, json, md"
239 - multiple style="display: none" @change="handleFileUpload($event)">
267 + <input type="file" id="file-input" accept=".png, .jpg, .jpeg, .txt, .pdf, .csv, .html, .json, .md, .py, .js, .sh, .css" multiple style="display: none"
268 + @change="handleFileUpload($event)">
269
241 - <div x-show="showTooltip" class="tooltip">
242 - Limit: 4 attachments per message
243 - </div>
270 + <div x-show="showTooltip" class="tooltip">
271 + Limit: 4 attachments per message
272 </div>
273 + </div>
274
246 - <!-- Text input -->
247 - <textarea id="chat-input" placeholder="Type your message here..." rows="1"></textarea>
275 + <!-- Text input -->
276 + <textarea id="chat-input" placeholder="Type your message here..." rows="1"></textarea>
277
249 - <div id="chat-buttons-wrapper">
250 - <!-- Send button -->
251 - <button class="chat-button" id="send-button" aria-label="Send message">
278 + <div id="chat-buttons-wrapper">
279 + <!-- Send button -->
280 + <button class="chat-button" id="send-button" aria-label="Send message">
281 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
282 <path d="M25 20 L75 50 L25 80" fill="none" stroke="currentColor" stroke-width="15">
283 </path>
webui/index.js
+25 -40
@@ -1,4 +1,4 @@
1 -import * as msgs from "./messages.js"
1 +import * as msgs from "./messages.js";
2 import { speech } from "./speech.js";
3
4 const leftPanel = document.getElementById('left-panel');
@@ -10,16 +10,12 @@ const sendButton = document.getElementById('send-button');
10 const inputSection = document.getElementById('input-section');
11 const statusSection = document.getElementById('status-section');
12 const chatsSection = document.getElementById('chats-section');
13 -const scrollbarThumb = document.querySelector('#chat-history::-webkit-scrollbar-thumb');
13 const progressBar = document.getElementById('progress-bar');
14 const autoScrollSwitch = document.getElementById('auto-scroll-switch');
15
17 -
16 let autoScroll = true;
17 let context = "";
18
21 -
22 -
19 // Initialize the toggle button
20 setupSidebarToggle();
21
@@ -42,7 +38,6 @@ function handleResize() {
38 }
39 }
40
45 -// Run on startup and window resize
41 window.addEventListener('load', handleResize);
42 window.addEventListener('resize', handleResize);
43
@@ -57,10 +52,8 @@ function setupSidebarToggle() {
52 setTimeout(setupSidebarToggle, 100);
53 }
54 }
60 -// Make sure to call this function
55 document.addEventListener('DOMContentLoaded', setupSidebarToggle);
56
63 -// index.js
57 export async function sendMessage() {
58 try {
59 const message = chatInput.value.trim();
@@ -72,14 +65,22 @@ export async function sendMessage() {
65 let response;
66 const messageId = generateGUID();
67
75 - // Only render immediately for attachments
68 + // Include attachments in the user message
69 if (hasAttachments) {
77 - const attachmentsWithUrls = attachments.map(attachment => ({
78 - ...attachment,
79 - url: URL.createObjectURL(attachment.file)
80 - }));
70 + const attachmentsWithUrls = attachments.map(attachment => {
71 + if (attachment.type === 'image') {
72 + return {
73 + ...attachment,
74 + url: URL.createObjectURL(attachment.file)
75 + };
76 + } else {
77 + return {
78 + ...attachment
79 + };
80 + }
81 + });
82
82 - // Only render if there's text content or it's an image-only message
83 + // Render user message with attachments
84 setMessage(messageId, 'user', '', message, false, {
85 attachments: attachmentsWithUrls
86 });
@@ -98,7 +99,7 @@ export async function sendMessage() {
99 body: formData
100 });
101 } else {
101 - // For text-only messages, let polling handle the rendering
102 + // For text-only messages
103 const data = {
104 text: message,
105 context,
@@ -186,12 +187,17 @@ function updateUserTime() {
187 updateUserTime();
188 setInterval(updateUserTime, 1000);
189
190 +
191 function setMessage(id, type, heading, content, temp, kvps = null) {
192 // Search for the existing message container by id
193 let messageContainer = document.getElementById(`message-${id}`);
194
195 if (messageContainer) {
194 - // Clear the existing container's content if found
196 + // Don't re-render user messages
197 + if (type === 'user') {
198 + return; // Skip re-rendering
199 + }
200 + // For other types, update the message
201 messageContainer.innerHTML = '';
202 } else {
203 // Create a new container if not found
@@ -199,8 +205,7 @@ function setMessage(id, type, heading, content, temp, kvps = null) {
205 messageContainer = document.createElement('div');
206 messageContainer.id = `message-${id}`;
207 messageContainer.classList.add('message-container', `${sender}-container`);
202 - if (temp) messageContainer.classList.add("message-temp")
203 -
208 + if (temp) messageContainer.classList.add("message-temp");
209 }
210
211 const handler = msgs.getHandler(type);
@@ -215,27 +220,6 @@ function setMessage(id, type, heading, content, temp, kvps = null) {
220 }
221
222
218 -async function handleFileUpload(event) {
219 - const files = event.target.files;
220 - const formData = new FormData();
221 - for (let i = 0; i < files.length; i++) {
222 - formData.append('file', files[i]);
223 - }
224 -
225 - const response = await fetch('/upload', {
226 - method: 'POST',
227 - body: formData,
228 - });
229 -
230 - const data = await response.json();
231 - if (!data.ok) {
232 - toast(data.message, "error");
233 - } else {
234 - toast("Files uploaded: " + data.filenames.join(", "), "success");
235 - }
236 -}
237 -
238 -
223 window.loadKnowledge = async function () {
224 const input = document.createElement('input');
225 input.type = 'file';
@@ -356,7 +340,8 @@ async function poll() {
340 if (lastLogVersion != response.log_version) {
341 updated = true
342 for (const log of response.logs) {
359 - setMessage(log.no, log.type, log.heading, log.content, log.temp, log.kvps);
343 + const messageId = log.id || log.no; // Use log.id if available
344 + setMessage(messageId, log.type, log.heading, log.content, log.temp, log.kvps);
345 }
346 afterMessagesUpdate(response.logs)
347 }
webui/messages.js
+86 -47
@@ -30,7 +30,6 @@ export function getHandler(type) {
30
31 // draw a message with a specific type
32 export function _drawMessage(messageContainer, heading, content, temp, followUp, kvps = null, messageClasses = [], contentClasses = []) {
33 -
33 const messageDiv = document.createElement('div');
34 messageDiv.classList.add('message', ...messageClasses);
35
@@ -42,35 +41,37 @@ export function _drawMessage(messageContainer, heading, content, temp, followUp,
41
42 drawKvps(messageDiv, kvps);
43
45 - const preElement = document.createElement('pre');
46 - preElement.classList.add("msg-content", ...contentClasses);
47 - preElement.style.whiteSpace = 'pre-wrap';
48 - preElement.style.wordBreak = 'break-word';
44 + if (content && content.trim().length > 0) {
45 + const preElement = document.createElement('pre');
46 + preElement.classList.add("msg-content", ...contentClasses);
47 + preElement.style.whiteSpace = 'pre-wrap';
48 + preElement.style.wordBreak = 'break-word';
49 +
50 + const spanElement = document.createElement('span');
51 + spanElement.innerHTML = content;
52 + preElement.appendChild(spanElement);
53 + messageDiv.appendChild(preElement);
54 +
55 + // Render LaTeX math within the span
56 + if (window.renderMathInElement) {
57 + renderMathInElement(spanElement, {
58 + delimiters: [
59 + { left: "$", right: "$", display: true },
60 + { left: "\\$", right: "\\$", display: true },
61 + { left: "$", right: "$", display: false },
62 + { left: "\\$", right: "\\$", display: false }
63 + ],
64 + throwOnError: false
65 + });
66 + }
67 + }
68
50 - // Wrap content in a <span> to allow HTML parsing
51 - const spanElement = document.createElement('span');
52 - spanElement.innerHTML = content; // Use innerHTML instead of textContent
53 - preElement.appendChild(spanElement);
54 - messageDiv.appendChild(preElement);
69 messageContainer.appendChild(messageDiv);
70
71 if (followUp) {
72 messageContainer.classList.add("message-followup");
73 }
74
61 - // Render LaTeX math within the span
62 - if (window.renderMathInElement) {
63 - renderMathInElement(spanElement, {
64 - delimiters: [
65 - { left: "$", right: "$", display: true },
66 - { left: "\\$", right: "\\$", display: true },
67 - { left: "$", right: "$", display: false },
68 - { left: "\\$", right: "\\$", display: false }
69 - ],
70 - throwOnError: false // Prevent KaTeX from throwing errors
71 - });
72 - }
73 -
75 return messageDiv;
76 }
77
@@ -102,41 +103,79 @@ export function drawMessageDelegation(messageContainer, id, type, heading, conte
103 }
104
105 export function drawMessageUser(messageContainer, id, type, heading, content, temp, kvps = null) {
105 - const hasContent = content && content.trim().length > 0;
106 - const hasAttachments = kvps && kvps.attachments && kvps.attachments.length > 0;
106 + const messageDiv = document.createElement('div');
107 + messageDiv.classList.add('message', 'message-user');
108
108 - // Only create message container if there's content or it's the initial message
109 - if (hasContent || !messageContainer.hasChildNodes()) {
110 - // Create the message with user heading and content
111 - _drawMessage(messageContainer, "User message", content, temp, false, null, ['message-user'], []);
109 + const headingElement = document.createElement('h4');
110 + headingElement.textContent = "User message";
111 + messageDiv.appendChild(headingElement);
112 +
113 + if (content && content.trim().length > 0) {
114 + const textDiv = document.createElement('div');
115 + textDiv.classList.add('message-text');
116 + textDiv.textContent = content;
117 + messageDiv.appendChild(textDiv);
118 }
119
114 - // Render image attachments for user messages
115 - if (hasAttachments) {
120 + // Handle attachments
121 + if (kvps && kvps.attachments && kvps.attachments.length > 0) {
122 const attachmentsContainer = document.createElement('div');
123 attachmentsContainer.classList.add('attachments-container');
124
119 - // Add "Attachments" heading if there's text content
120 - if (hasContent) {
121 - const attachmentsHeading = document.createElement('h4');
122 - attachmentsHeading.textContent = "Attachments";
123 - attachmentsContainer.appendChild(attachmentsHeading);
124 - }
125 -
125 kvps.attachments.forEach(attachment => {
127 - const image = document.createElement('img');
128 - if (attachment.url) {
129 - image.src = attachment.url;
130 - } else if (attachment.data) {
131 - image.src = `data:image/jpeg;base64,${attachment.data}`;
126 + const attachmentDiv = document.createElement('div');
127 + attachmentDiv.classList.add('attachment-item');
128 +
129 + if (typeof attachment === 'string') {
130 + // attachment is filename
131 + const filename = attachment;
132 + const extension = filename.split('.').pop().toUpperCase();
133 +
134 + attachmentDiv.classList.add('file-type');
135 + attachmentDiv.innerHTML = `
136 + <div class="file-preview">
137 + <span class="filename">${filename}</span>
138 + <span class="extension">${extension}</span>
139 + </div>
140 + `;
141 + } else if (attachment.type === 'image') {
142 + // Existing logic for images
143 + const imgWrapper = document.createElement('div');
144 + imgWrapper.classList.add('image-wrapper');
145 +
146 + const img = document.createElement('img');
147 + img.src = attachment.url;
148 + img.alt = attachment.name;
149 + img.classList.add('attachment-preview');
150 +
151 + const fileInfo = document.createElement('div');
152 + fileInfo.classList.add('file-info');
153 + fileInfo.innerHTML = `
154 + <span class="filename">${attachment.name}</span>
155 + <span class="extension">${attachment.extension.toUpperCase()}</span>
156 + `;
157 +
158 + imgWrapper.appendChild(img);
159 + attachmentDiv.appendChild(imgWrapper);
160 + attachmentDiv.appendChild(fileInfo);
161 + } else {
162 + // Existing logic for non-image files
163 + attachmentDiv.classList.add('file-type');
164 + attachmentDiv.innerHTML = `
165 + <div class="file-preview">
166 + <span class="filename">${attachment.name}</span>
167 + <span class="extension">${attachment.extension.toUpperCase()}</span>
168 + </div>
169 + `;
170 }
133 - image.alt = 'Attachment';
134 - image.classList.add('message-attachment');
135 - attachmentsContainer.appendChild(image);
171 +
172 + attachmentsContainer.appendChild(attachmentDiv);
173 });
174
138 - messageContainer.appendChild(attachmentsContainer);
175 + messageDiv.appendChild(attachmentsContainer);
176 }
177 +
178 + messageContainer.appendChild(messageDiv);
179 }
180
181 export function drawMessageTool(messageContainer, id, type, heading, content, temp, kvps = null) {