3 files changed
+238
-4
lib/features/chat/data/chat_service.dart
+86
-3
@@ -51,7 +51,11 @@ abstract class ChatService {
51
case ServerType.openAICompatible:
52
return OpenAICompatibleChatService(dio);
53
case ServerType.ollama:
54
- return OllamaChatService(dio);
54
+ return OllamaChatService(
55
+ dio,
56
+ imageCompressionEnabled: imageCompressionEnabled,
57
+ imageCompressionLevel: imageCompressionLevel,
58
+ );
59
case ServerType.openRouter:
60
return OpenRouterChatService(dio);
61
case ServerType.onDevice:
@@ -673,9 +677,15 @@ class OpenAICompatibleChatService implements ChatService {
677
678
class OllamaChatService implements ChatService {
679
final Dio _dio;
680
+ final bool imageCompressionEnabled;
681
+ final ImageCompressionLevel imageCompressionLevel;
682
CancelToken? _cancelToken;
683
678
- OllamaChatService(this._dio);
684
+ OllamaChatService(
685
+ this._dio, {
686
+ this.imageCompressionEnabled = true,
687
+ this.imageCompressionLevel = ImageCompressionLevel.medium,
688
+ });
689
690
@override
691
Stream<ChatResponse> sendMessage({
@@ -691,9 +701,14 @@ class OllamaChatService implements ChatService {
701
_cancelToken = CancelToken();
702
final toolAdapter = OllamaToolAdapter();
703
704
+ final apiMessages = <Map<String, dynamic>>[];
705
+ for (final message in messages) {
706
+ apiMessages.add(await _messageToOllamaMapWithImages(message));
707
+ }
708
+
709
final body = <String, dynamic>{
710
'model': modelId,
696
- 'messages': messages.map(_messageToApiMap).toList(),
711
+ 'messages': apiMessages,
712
'stream': true,
713
'options': {
714
'temperature': params.temperature,
@@ -787,6 +802,74 @@ class OllamaChatService implements ChatService {
802
yield const ChatResponse(type: ChatResponseType.done);
803
}
804
805
+ Future<Map<String, dynamic>> _messageToOllamaMapWithImages(Message m) async {
806
+ var textContent = m.content;
807
+ final images = <String>[];
808
+
809
+ for (final path in m.attachmentPaths ?? const <String>[]) {
810
+ if (AttachmentHelpers.isTextPath(path)) {
811
+ final text = await AttachmentHelpers.readTextFile(path);
812
+ if (text != null) {
813
+ textContent = AttachmentHelpers.appendTextAttachment(
814
+ textContent,
815
+ AttachmentHelpers.fileNameOf(path),
816
+ text,
817
+ );
818
+ }
819
+ continue;
820
+ }
821
+
822
+ if (!AttachmentHelpers.isImagePath(path)) continue;
823
+
824
+ try {
825
+ final file = File(path);
826
+ if (!await file.exists()) continue;
827
+
828
+ final fileBytes = await ImageUploadUtils.prepareImageBytes(
829
+ file,
830
+ enabled: imageCompressionEnabled,
831
+ level: imageCompressionLevel,
832
+ );
833
+ final base64Image = await Isolate.run(() {
834
+ try {
835
+ return base64Encode(fileBytes);
836
+ } catch (_) {
837
+ return null;
838
+ }
839
+ });
840
+
841
+ if (base64Image != null) images.add(base64Image);
842
+ } catch (e) {
843
+ Log.error('Failed to read attachment for Ollama format: $e');
844
+ }
845
+ }
846
+
847
+ final map = <String, dynamic>{
848
+ 'role': _roleToString(m.role),
849
+ 'content': textContent,
850
+ };
851
+ if (images.isNotEmpty) map['images'] = images;
852
+
853
+ if (m.role == MessageRole.tool && m.toolCallId != null) {
854
+ map['tool_call_id'] = m.toolCallId;
855
+ }
856
+ if (m.role == MessageRole.assistant &&
857
+ m.toolCalls != null &&
858
+ m.toolCalls!.isNotEmpty) {
859
+ map['tool_calls'] = m.toolCalls!.map((tc) {
860
+ return {
861
+ 'id': tc.id,
862
+ 'type': 'function',
863
+ 'function': {
864
+ 'name': tc.toolName,
865
+ 'arguments': jsonEncode(tc.arguments),
866
+ },
867
+ };
868
+ }).toList();
869
+ }
870
+ return map;
871
+ }
872
+
873
@override
874
void cancelStream() {
875
_cancelToken?.cancel('User cancelled');
lib/features/chat/utils/image_upload_utils.dart
+1
-1
@@ -11,7 +11,7 @@ class _CompressionConfig {
11
final List<int> dimensions;
12
}
13
14
-/// Resize/compress images so LM Studio requests stay under payload limits.
14
+/// Resize/compress images so vision-model requests stay under payload limits.
15
class ImageUploadUtils {
16
ImageUploadUtils._();
17
test/features/chat/tools/chat_service_adapter_selection_test.dart
+151
@@ -1,5 +1,6 @@
1
import 'dart:async';
2
import 'dart:convert';
3
+import 'dart:io';
4
import 'dart:typed_data';
5
import 'package:dio/dio.dart';
6
import 'package:flutter_test/flutter_test.dart';
@@ -43,6 +44,18 @@ class StreamInterceptor extends Interceptor {
44
}
45
}
46
47
+class CapturingStreamInterceptor extends StreamInterceptor {
48
+ RequestOptions? capturedRequest;
49
+
50
+ CapturingStreamInterceptor(super.lines);
51
+
52
+ @override
53
+ void onRequest(RequestOptions options, RequestInterceptorHandler handler) {
54
+ capturedRequest = options;
55
+ super.onRequest(options, handler);
56
+ }
57
+}
58
+
59
void main() {
60
group('createAdapterForServerType', () {
61
test('uses OpenAI adapter for OpenAI-compatible servers', () {
@@ -171,4 +184,142 @@ void main() {
184
expect(toolCalls.first!.arguments['b'], 6);
185
});
186
});
187
+
188
+ group('Ollama vision message formatting', () {
189
+ late Directory tempDirectory;
190
+
191
+ setUp(() async {
192
+ tempDirectory = await Directory.systemTemp.createTemp(
193
+ 'localmind_ollama_vision_test_',
194
+ );
195
+ });
196
+
197
+ tearDown(() async {
198
+ if (await tempDirectory.exists()) {
199
+ await tempDirectory.delete(recursive: true);
200
+ }
201
+ });
202
+
203
+ test('sends multiple images as ordered raw base64 values', () async {
204
+ final firstBytes = [1, 2, 3];
205
+ final secondBytes = [4, 5, 6];
206
+ final firstImage = await File(
207
+ '${tempDirectory.path}/first.png',
208
+ ).writeAsBytes(firstBytes);
209
+ final secondImage = await File(
210
+ '${tempDirectory.path}/second.jpg',
211
+ ).writeAsBytes(secondBytes);
212
+ final interceptor = CapturingStreamInterceptor(['{"done":true}']);
213
+ final service = OllamaChatService(
214
+ Dio()..interceptors.add(interceptor),
215
+ imageCompressionEnabled: false,
216
+ );
217
+
218
+ final responses = await service
219
+ .sendMessage(
220
+ server: _ollamaTestServer(),
221
+ modelId: 'vision-model',
222
+ messages: [
223
+ Message(
224
+ id: 'vision-message',
225
+ conversationId: 'conversation',
226
+ role: MessageRole.user,
227
+ content: 'What is shown?',
228
+ attachmentPaths: [firstImage.path, secondImage.path],
229
+ createdAt: DateTime.now(),
230
+ ),
231
+ ],
232
+ params: ChatParameters.defaults(),
233
+ )
234
+ .toList();
235
+
236
+ final body = interceptor.capturedRequest!.data as Map<String, dynamic>;
237
+ final message = (body['messages'] as List).single as Map<String, dynamic>;
238
+ expect(message['content'], 'What is shown?');
239
+ expect(message['images'], [
240
+ base64Encode(firstBytes),
241
+ base64Encode(secondBytes),
242
+ ]);
243
+ expect(responses.last.type, ChatResponseType.done);
244
+ });
245
+
246
+ test(
247
+ 'keeps text-only messages unchanged and appends text attachments',
248
+ () async {
249
+ final textFile = await File(
250
+ '${tempDirectory.path}/notes.txt',
251
+ ).writeAsString('attachment text');
252
+ final interceptor = CapturingStreamInterceptor(['{"done":true}']);
253
+ final service = OllamaChatService(Dio()..interceptors.add(interceptor));
254
+
255
+ await service
256
+ .sendMessage(
257
+ server: _ollamaTestServer(),
258
+ modelId: 'text-model',
259
+ messages: [
260
+ Message(
261
+ id: 'text-message',
262
+ conversationId: 'conversation',
263
+ role: MessageRole.user,
264
+ content: 'Summarize this',
265
+ attachmentPaths: [textFile.path],
266
+ createdAt: DateTime.now(),
267
+ ),
268
+ ],
269
+ params: ChatParameters.defaults(),
270
+ )
271
+ .toList();
272
+
273
+ final body = interceptor.capturedRequest!.data as Map<String, dynamic>;
274
+ final message =
275
+ (body['messages'] as List).single as Map<String, dynamic>;
276
+ expect(
277
+ message['content'],
278
+ 'Summarize this\n\n--- notes.txt ---\nattachment text',
279
+ );
280
+ expect(message.containsKey('images'), isFalse);
281
+ },
282
+ );
283
+
284
+ test('skips missing images without aborting the request', () async {
285
+ final interceptor = CapturingStreamInterceptor(['{"done":true}']);
286
+ final service = OllamaChatService(Dio()..interceptors.add(interceptor));
287
+
288
+ final responses = await service
289
+ .sendMessage(
290
+ server: _ollamaTestServer(),
291
+ modelId: 'vision-model',
292
+ messages: [
293
+ Message(
294
+ id: 'missing-image-message',
295
+ conversationId: 'conversation',
296
+ role: MessageRole.user,
297
+ content: 'Continue without the missing image',
298
+ attachmentPaths: ['${tempDirectory.path}/missing.png'],
299
+ createdAt: DateTime.now(),
300
+ ),
301
+ ],
302
+ params: ChatParameters.defaults(),
303
+ )
304
+ .toList();
305
+
306
+ final body = interceptor.capturedRequest!.data as Map<String, dynamic>;
307
+ final message = (body['messages'] as List).single as Map<String, dynamic>;
308
+ expect(message['content'], 'Continue without the missing image');
309
+ expect(message.containsKey('images'), isFalse);
310
+ expect(responses.last.type, ChatResponseType.done);
311
+ });
312
+ });
313
+}
314
+
315
+Server _ollamaTestServer() {
316
+ return Server(
317
+ id: 'test-ollama',
318
+ name: 'Test Ollama',
319
+ type: ServerType.ollama,
320
+ host: 'localhost',
321
+ port: 11434,
322
+ createdAt: DateTime.now(),
323
+ lastConnectedAt: DateTime.now(),
324
+ );
325
}