8 files changed
+742
-37
lib/features/chat/data/chat_service.dart
+10
-5
@@ -77,7 +77,11 @@ abstract class ChatService {
77
'OnDeviceGemmaService is required for onDevice server type',
78
);
79
}
80
- return OnDeviceChatService(onDeviceGemma);
80
+ return OnDeviceChatService(
81
+ onDeviceGemma,
82
+ imageCompressionEnabled: imageCompressionEnabled,
83
+ imageCompressionLevel: imageCompressionLevel,
84
+ );
85
}
86
}
87
}
@@ -205,7 +209,7 @@ class LMStudioChatService implements ChatService {
209
final String systemPrompt;
210
final input = useTypedInput
211
? await _buildNativeTypedInput(messages, continueGeneration)
208
- : _buildNativeInputString(messages, continueGeneration);
212
+ : await _buildNativeInputString(messages, continueGeneration);
213
systemPrompt = _extractSystemPrompt(messages) ?? params.systemPrompt ?? '';
214
215
final body = <String, dynamic>{
@@ -470,10 +474,10 @@ class LMStudioChatService implements ChatService {
474
/// plain-text chats. Conversation history is collapsed into a transcript
475
/// because `/api/v1/chat` does not accept an OpenAI-style `messages`
476
/// array; system prompt is hoisted to a top-level field by the caller.
473
- String _buildNativeInputString(
477
+ Future<String> _buildNativeInputString(
478
List<Message> messages,
479
bool continueGeneration,
476
- ) {
480
+ ) async {
481
final visible = <Message>[];
482
for (final m in messages) {
483
if (m.role == MessageRole.system) continue;
@@ -495,7 +499,8 @@ class LMStudioChatService implements ChatService {
499
MessageRole.tool => 'Tool',
500
MessageRole.system => 'System',
501
};
498
- buf.writeln('$tag: ${m.content}');
502
+ final textContent = await _messageTextWithAttachments(m);
503
+ buf.writeln('$tag: $textContent');
504
}
505
return buf.toString().trimRight();
506
}
lib/features/chat/providers/chat_service_providers.dart
+2
@@ -73,6 +73,8 @@ ChatService createChatServiceForServer({
73
server.type,
74
dio,
75
onDeviceGemma: onDeviceGemmaService,
76
+ imageCompressionEnabled: imageCompressionEnabled,
77
+ imageCompressionLevel: imageCompressionLevel,
78
);
79
}
80
lib/features/on_device/data/on_device_chat_service.dart
+203
-29
@@ -1,4 +1,6 @@
1
import 'dart:async';
2
+import 'dart:io';
3
+import 'dart:typed_data';
4
5
import 'package:flutter_gemma/flutter_gemma.dart' as gemma;
6
@@ -10,6 +12,8 @@ import '../../chat/data/models/chat_parameters.dart';
12
import '../../chat/data/models/mcp_integration.dart';
13
import '../../chat/data/models/message.dart' hide ToolCallData;
14
import '../../chat/data/tools/tool_definition.dart';
15
+import '../../chat/utils/attachment_helpers.dart';
16
+import '../../chat/utils/image_upload_utils.dart';
17
import '../../servers/data/models/server.dart';
18
import 'on_device_gemma_service.dart';
19
@@ -35,6 +39,8 @@ class OnDeviceChatService implements ChatService {
39
OnDeviceChatService(
40
this._gemmaService, {
41
Duration retainedSessionTtl = _defaultRetainedSessionTtl,
42
+ this.imageCompressionEnabled = true,
43
+ this.imageCompressionLevel = ImageCompressionLevel.medium,
44
// The retained-session TTL is a separate constructor parameter (rather
45
// than an initializing formal) so it can have a default value; the
46
// lint suggestion doesn't apply when a default is involved.
@@ -43,6 +49,8 @@ class OnDeviceChatService implements ChatService {
49
50
final OnDeviceInferenceService _gemmaService;
51
final Duration _retainedSessionTtl;
52
+ final bool imageCompressionEnabled;
53
+ final ImageCompressionLevel imageCompressionLevel;
54
final Set<_InferenceRun> _activeRuns = <_InferenceRun>{};
55
_RetainedConversation? _retainedConversation;
56
Timer? _retainedExpirationTimer;
@@ -93,8 +101,25 @@ class OnDeviceChatService implements ChatService {
101
ChatParameters params,
102
) async {
103
try {
96
- final input = _prepareInput(modelId, messages, params);
97
- if (input == null) {
104
+ final input = await _prepareInput(modelId, messages, params);
105
+ if (run.isCancelled) {
106
+ await _closeSession(run, stopGeneration: false);
107
+ return;
108
+ }
109
+ if (input == null ||
110
+ (input.currentMessage.content.trim().isEmpty && !input.hasImages)) {
111
+ final lastMsg = messages.isNotEmpty ? messages.last : null;
112
+ final hasImageAttachment =
113
+ lastMsg?.attachmentPaths?.any(AttachmentHelpers.isImagePath) ??
114
+ false;
115
+ if (hasImageAttachment && !_gemmaService.currentModelSupportsVision) {
116
+ await _finishWithError(
117
+ run,
118
+ 'The active model does not support image attachments. '
119
+ 'Please select a vision-supported model like Gemma 4 or FastVLM.',
120
+ );
121
+ return;
122
+ }
123
await _finishWithError(run, 'On-device chat requires a user message.');
124
return;
125
}
@@ -115,9 +140,11 @@ class OnDeviceChatService implements ChatService {
140
_cancelRetainedExpiration();
141
await retained.session.close();
142
}
143
+ final modelSupportsVision = _gemmaService.currentModelSupportsVision;
144
session = await _gemmaService.createChat(
145
systemInstruction: input.baseSystemInstruction,
146
tools: const [],
147
+ supportImage: modelSupportsVision && input.hasImages,
148
);
149
reusedSession = false;
150
}
@@ -366,6 +393,8 @@ class OnDeviceChatService implements ChatService {
393
modelId: input.modelId,
394
baseSystemInstruction: input.baseSystemInstruction,
395
timeline: input.completedTimeline(run.generatedContent.toString()),
396
+ supportsImages:
397
+ input.hasImages || _gemmaService.currentModelSupportsVision,
398
);
399
_armRetainedExpiration();
400
if (previous != null && !identical(previous.session, session)) {
@@ -402,11 +431,11 @@ class OnDeviceChatService implements ChatService {
431
await run.controller.close();
432
}
433
405
- _PreparedInput? _prepareInput(
434
+ Future<_PreparedInput?> _prepareInput(
435
String modelId,
436
List<Message> messages,
437
ChatParameters params,
409
- ) {
438
+ ) async {
439
final relevant = messages
440
.where(
441
(message) =>
@@ -423,22 +452,50 @@ class OnDeviceChatService implements ChatService {
452
pendingAssistant = relevant.removeLast();
453
}
454
426
- final systemMessages = relevant
427
- .where(
428
- (message) =>
429
- message.role == MessageRole.system &&
430
- message.content.trim().isNotEmpty,
431
- )
432
- .map((message) => message.content.trim())
433
- .toList(growable: false);
434
- final timeline = relevant
455
+ final systemMessages = <String>[];
456
+ for (final message in relevant) {
457
+ if (message.role == MessageRole.system) {
458
+ var content = message.content.trim();
459
+ final paths = message.attachmentPaths;
460
+ if (paths != null && paths.isNotEmpty) {
461
+ for (final path in paths) {
462
+ if (AttachmentHelpers.isDocumentPath(path)) {
463
+ final text = await AttachmentHelpers.readDocumentFile(path);
464
+ if (text != null && text.trim().isNotEmpty) {
465
+ content = AttachmentHelpers.appendTextAttachment(
466
+ content,
467
+ AttachmentHelpers.fileNameOf(path),
468
+ text,
469
+ );
470
+ }
471
+ }
472
+ }
473
+ }
474
+ if (content.isNotEmpty) {
475
+ systemMessages.add(content);
476
+ }
477
+ }
478
+ }
479
+
480
+ final timelineMessages = relevant
481
.where(
482
(message) =>
483
message.role == MessageRole.user ||
484
message.role == MessageRole.assistant,
485
)
440
- .map(_MessageSnapshot.fromMessage)
486
.toList(growable: false);
487
+
488
+ final modelSupportsVision = _gemmaService.currentModelSupportsVision;
489
+ final timeline = <_MessageSnapshot>[];
490
+ for (final message in timelineMessages) {
491
+ final snapshot = await _MessageSnapshot.fromMessage(
492
+ message,
493
+ supportsVision: modelSupportsVision,
494
+ imageCompressionEnabled: imageCompressionEnabled,
495
+ imageCompressionLevel: imageCompressionLevel,
496
+ );
497
+ timeline.add(snapshot);
498
+ }
499
if (timeline.isEmpty) return null;
500
501
final currentMessage = timeline.last;
@@ -535,6 +592,10 @@ class _PreparedInput {
592
final String? pendingAssistantId;
593
final bool isAuxiliary;
594
595
+ bool get hasImages =>
596
+ currentMessage.images.isNotEmpty ||
597
+ history.any((message) => message.images.isNotEmpty);
598
+
599
gemma.Message toGemmaMessage({required bool includeHistory}) {
600
if (!includeHistory || history.isEmpty) {
601
return currentMessage.toGemmaMessage();
@@ -549,10 +610,32 @@ class _PreparedInput {
610
final currentLabel = currentMessage.role == MessageRole.user
611
? 'Current user message'
612
: 'Assistant response to continue';
613
+ final fullText =
614
+ 'Earlier conversation transcript (context only):\n\n$transcript\n\n'
615
+ '$currentLabel:\n${currentMessage.content}';
616
+
617
+ final images = currentMessage.images.isNotEmpty
618
+ ? currentMessage.images
619
+ : history.reversed
620
+ .expand((message) => message.images)
621
+ .toList(growable: false)
622
+ .reversed
623
+ .toList(growable: false);
624
+
625
+ if (images.isNotEmpty) {
626
+ var text = fullText;
627
+ if (text.trim().isEmpty) {
628
+ text = 'Describe the image.';
629
+ }
630
+ return gemma.Message.withImages(
631
+ text: text,
632
+ imageBytes: images,
633
+ isUser: currentMessage.role == MessageRole.user,
634
+ );
635
+ }
636
+
637
return gemma.Message.text(
553
- text:
554
- 'Earlier conversation transcript (context only):\n\n$transcript\n\n'
555
- '$currentLabel:\n${currentMessage.content}',
638
+ text: fullText,
639
isUser: currentMessage.role == MessageRole.user,
640
);
641
}
@@ -584,6 +667,7 @@ class _RetainedConversation {
667
required this.modelId,
668
required this.baseSystemInstruction,
669
required this.timeline,
670
+ required this.supportsImages,
671
});
672
673
final OnDeviceInferenceSession session;
@@ -591,8 +675,12 @@ class _RetainedConversation {
675
final String modelId;
676
final String? baseSystemInstruction;
677
final List<_MessageSnapshot> timeline;
678
+ final bool supportsImages;
679
680
bool canContinueWith(_PreparedInput input, String requestedModelId) {
681
+ if (input.hasImages && !supportsImages) {
682
+ return false;
683
+ }
684
return input.currentMessage.role == MessageRole.user &&
685
conversationId == input.conversationId &&
686
modelId == requestedModelId &&
@@ -618,28 +706,97 @@ class _MessageSnapshot {
706
required this.conversationId,
707
required this.role,
708
required this.content,
709
+ this.images = const [],
710
+ this.attachmentPaths = const [],
711
});
712
623
- factory _MessageSnapshot.fromMessage(Message message) => _MessageSnapshot(
624
- id: message.id,
625
- conversationId: message.conversationId,
626
- role: message.role,
627
- content: message.content,
628
- );
713
+ static Future<_MessageSnapshot> fromMessage(
714
+ Message message, {
715
+ bool supportsVision = false,
716
+ bool imageCompressionEnabled = true,
717
+ ImageCompressionLevel imageCompressionLevel = ImageCompressionLevel.medium,
718
+ }) async {
719
+ var content = message.content;
720
+ final images = <Uint8List>[];
721
+
722
+ final paths = message.attachmentPaths ?? const <String>[];
723
+
724
+ for (final path in paths) {
725
+ if (AttachmentHelpers.isDocumentPath(path)) {
726
+ final text = await AttachmentHelpers.readDocumentFile(path);
727
+ if (text != null && text.trim().isNotEmpty) {
728
+ content = AttachmentHelpers.appendTextAttachment(
729
+ content,
730
+ AttachmentHelpers.fileNameOf(path),
731
+ text,
732
+ );
733
+ }
734
+ } else if (supportsVision && AttachmentHelpers.isImagePath(path)) {
735
+ final file = File(path);
736
+ try {
737
+ if (await file.exists()) {
738
+ final bytes = await ImageUploadUtils.prepareImageBytes(
739
+ file,
740
+ enabled: imageCompressionEnabled,
741
+ level: imageCompressionLevel,
742
+ );
743
+ images.add(bytes);
744
+ }
745
+ } catch (e) {
746
+ Log.warning('Failed to prepare image bytes for $path: $e');
747
+ try {
748
+ if (await file.exists()) {
749
+ images.add(await file.readAsBytes());
750
+ }
751
+ } catch (_) {}
752
+ }
753
+ }
754
+ }
755
+
756
+ return _MessageSnapshot(
757
+ id: message.id,
758
+ conversationId: message.conversationId,
759
+ role: message.role,
760
+ content: content,
761
+ images: List.unmodifiable(images),
762
+ attachmentPaths: List.unmodifiable(paths),
763
+ );
764
+ }
765
766
final String id;
767
final String conversationId;
768
final MessageRole role;
769
final String content;
770
+ final List<Uint8List> images;
771
+ final List<String> attachmentPaths;
772
+
773
+ gemma.Message toGemmaMessage({String? promptOverride}) {
774
+ var text = promptOverride ?? content;
775
+ final isUser = role == MessageRole.user;
776
+ if (images.isNotEmpty) {
777
+ if (text.trim().isEmpty) {
778
+ text = 'Describe the image.';
779
+ }
780
+ return gemma.Message.withImages(
781
+ text: text,
782
+ imageBytes: images,
783
+ isUser: isUser,
784
+ );
785
+ }
786
+ return gemma.Message.text(text: text, isUser: isUser);
787
+ }
788
635
- gemma.Message toGemmaMessage() =>
636
- gemma.Message.text(text: content, isUser: role == MessageRole.user);
637
-
638
- _MessageSnapshot copyWith({String? content}) => _MessageSnapshot(
789
+ _MessageSnapshot copyWith({
790
+ String? content,
791
+ List<Uint8List>? images,
792
+ List<String>? attachmentPaths,
793
+ }) => _MessageSnapshot(
794
id: id,
795
conversationId: conversationId,
796
role: role,
797
content: content ?? this.content,
798
+ images: images ?? this.images,
799
+ attachmentPaths: attachmentPaths ?? this.attachmentPaths,
800
);
801
802
@override
@@ -648,9 +805,26 @@ class _MessageSnapshot {
805
id == other.id &&
806
conversationId == other.conversationId &&
807
role == other.role &&
651
- content == other.content;
808
+ content == other.content &&
809
+ _listEquals(attachmentPaths, other.attachmentPaths) &&
810
+ images.length == other.images.length;
811
}
812
813
@override
655
- int get hashCode => Object.hash(id, conversationId, role, content);
814
+ int get hashCode => Object.hash(
815
+ id,
816
+ conversationId,
817
+ role,
818
+ content,
819
+ Object.hashAll(attachmentPaths),
820
+ images.length,
821
+ );
822
+
823
+ static bool _listEquals(List<String> first, List<String> second) {
824
+ if (first.length != second.length) return false;
825
+ for (var i = 0; i < first.length; i++) {
826
+ if (first[i] != second[i]) return false;
827
+ }
828
+ return true;
829
+ }
830
}
lib/features/on_device/data/on_device_gemma_service.dart
+34
-1
@@ -20,10 +20,12 @@ abstract interface class OnDeviceInferenceSession {
20
21
abstract interface class OnDeviceInferenceService {
22
bool get isLoaded;
23
+ bool get currentModelSupportsVision;
24
25
Future<OnDeviceInferenceSession> createChat({
26
String? systemInstruction,
27
List<Tool> tools,
28
+ bool? supportImage,
29
});
30
}
31
@@ -55,6 +57,16 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
57
58
InferenceModel? get activeModel => _model;
59
String? get currentModelId => _currentModelId;
60
+ @override
61
+ bool get currentModelSupportsVision {
62
+ final id = _currentModelId;
63
+ if (id == null) return false;
64
+ final model = OnDeviceModel.allCuratedModels
65
+ .where((m) => m.id == id || m.fileName == id)
66
+ .firstOrNull;
67
+ return model?.supportsVision ?? false;
68
+ }
69
+
70
@override
71
bool get isLoaded => _model != null && !_isDisposed;
72
bool get isDisposed => _isDisposed;
@@ -234,6 +246,8 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
246
_model = await FlutterGemma.getActiveModel(
247
maxTokens: maxTokens,
248
preferredBackend: backend,
249
+ supportImage: model.supportsVision,
250
+ maxNumImages: model.supportsVision ? 5 : null,
251
);
252
_currentModelId = modelId;
253
_currentBackend = backend;
@@ -245,6 +259,7 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
259
Future<OnDeviceInferenceSession> createChat({
260
String? systemInstruction,
261
List<Tool> tools = const [],
262
+ bool? supportImage,
263
}) async {
264
await _ensureModelLoaded();
265
@@ -252,6 +267,7 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
267
return await _openChat(
268
systemInstruction: systemInstruction,
269
tools: tools,
270
+ supportImage: supportImage,
271
);
272
} catch (e) {
273
if (!_isClosedClientError(e)) rethrow;
@@ -261,7 +277,11 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
277
'Reloading the model once.',
278
);
279
await _recoverClosedModel();
264
- return _openChat(systemInstruction: systemInstruction, tools: tools);
280
+ return _openChat(
281
+ systemInstruction: systemInstruction,
282
+ tools: tools,
283
+ supportImage: supportImage,
284
+ );
285
}
286
}
287
@@ -276,14 +296,27 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
296
Future<OnDeviceInferenceSession> _openChat({
297
required String? systemInstruction,
298
required List<Tool> tools,
299
+ bool? supportImage,
300
}) async {
301
final model = _model!;
302
+ final visionEnabled = currentModelSupportsVision && (supportImage ?? true);
303
+
304
final InferenceChat chat;
305
if (model.fileType == ModelFileType.builtIn) {
306
chat = await model.createChat(
307
systemInstruction: systemInstruction,
308
tools: tools,
309
supportsFunctionCalls: tools.isNotEmpty,
310
+ supportImage: visionEnabled,
311
+ );
312
+ } else if (visionEnabled) {
313
+ // Multimodal .litertlm models MUST use createChat() (single-session)
314
+ // because openChat() (virtual concurrent session) rejects media.
315
+ chat = await model.createChat(
316
+ systemInstruction: systemInstruction,
317
+ tools: tools,
318
+ supportsFunctionCalls: tools.isNotEmpty,
319
+ supportImage: true,
320
);
321
} else {
322
chat = await model.openChat(
lib/features/on_device/data/on_device_llama_service.dart
+18
-1
@@ -9,6 +9,7 @@ import '../../chat/data/models/chat_parameters.dart';
9
import '../../chat/data/models/mcp_integration.dart';
10
import '../../chat/data/models/message.dart' hide ToolCallData;
11
import '../../chat/data/tools/tool_definition.dart';
12
+import '../../chat/utils/attachment_helpers.dart';
13
import '../../servers/data/models/server.dart';
14
import 'models/on_device_model.dart';
15
@@ -169,7 +170,23 @@ class OnDeviceLlamaService {
170
_lastUserMessageId = null;
171
}
172
172
- final text = latestUserMessage.content.trim();
173
+ var text = latestUserMessage.content;
174
+ final paths = latestUserMessage.attachmentPaths;
175
+ if (paths != null && paths.isNotEmpty) {
176
+ for (final path in paths) {
177
+ if (AttachmentHelpers.isDocumentPath(path)) {
178
+ final docText = await AttachmentHelpers.readDocumentFile(path);
179
+ if (docText != null && docText.trim().isNotEmpty) {
180
+ text = AttachmentHelpers.appendTextAttachment(
181
+ text,
182
+ AttachmentHelpers.fileNameOf(path),
183
+ docText,
184
+ );
185
+ }
186
+ }
187
+ }
188
+ }
189
+ text = text.trim();
190
if (text.isEmpty) {
191
yield const ChatResponse(
192
type: ChatResponseType.error,
lib/features/on_device/data/on_device_mlx_service.dart
+18
-1
@@ -16,6 +16,7 @@ import '../../chat/data/models/message.dart';
16
import '../../chat/data/models/mcp_integration.dart';
17
import '../../chat/data/tools/tool_definition.dart';
18
import '../../chat/data/chat_service.dart';
19
+import '../../chat/utils/attachment_helpers.dart';
20
import 'models/on_device_model.dart';
21
22
class OnDeviceMlxService {
@@ -421,7 +422,23 @@ class OnDeviceMlxService {
422
: m.role == MessageRole.assistant
423
? 'assistant'
424
: 'system';
424
- formattedMessages.add({'role': role, 'content': m.content});
425
+ var text = m.content;
426
+ final paths = m.attachmentPaths;
427
+ if (paths != null && paths.isNotEmpty) {
428
+ for (final path in paths) {
429
+ if (AttachmentHelpers.isDocumentPath(path)) {
430
+ final docText = await AttachmentHelpers.readDocumentFile(path);
431
+ if (docText != null && docText.trim().isNotEmpty) {
432
+ text = AttachmentHelpers.appendTextAttachment(
433
+ text,
434
+ AttachmentHelpers.fileNameOf(path),
435
+ docText,
436
+ );
437
+ }
438
+ }
439
+ }
440
+ }
441
+ formattedMessages.add({'role': role, 'content': text});
442
}
443
444
while (formattedMessages.isNotEmpty &&
test/features/on_device/on_device_chat_attachments_test.dart
new
+451
@@ -0,0 +1,451 @@
1
+import 'dart:async';
2
+import 'dart:io';
3
+import 'dart:typed_data';
4
+
5
+import 'package:dio/dio.dart';
6
+import 'package:flutter_gemma/flutter_gemma.dart' as gemma;
7
+import 'package:flutter_test/flutter_test.dart';
8
+import 'package:localmind/core/models/enums.dart';
9
+import 'package:localmind/core/services/crash_report_service.dart';
10
+import 'package:localmind/features/chat/data/chat_service.dart';
11
+import 'package:localmind/features/chat/data/models/chat_parameters.dart';
12
+import 'package:localmind/features/chat/data/models/message.dart';
13
+import 'package:localmind/features/on_device/data/models/on_device_model.dart';
14
+import 'package:localmind/features/on_device/data/on_device_chat_service.dart';
15
+import 'package:localmind/features/on_device/data/on_device_gemma_service.dart';
16
+import 'package:localmind/features/on_device/data/on_device_llama_service.dart';
17
+import 'package:localmind/features/on_device/data/on_device_mlx_service.dart';
18
+import 'package:localmind/features/servers/data/models/server.dart';
19
+
20
+void main() {
21
+ TestWidgetsFlutterBinding.ensureInitialized();
22
+
23
+ late Directory tempDir;
24
+ late _FakeInferenceService inferenceService;
25
+ late OnDeviceChatService chatService;
26
+
27
+ setUp(() async {
28
+ CrashReportService.instance.resetForTesting();
29
+ tempDir = await Directory.systemTemp.createTemp('attachment_test_');
30
+ inferenceService = _FakeInferenceService();
31
+ chatService = OnDeviceChatService(
32
+ inferenceService,
33
+ imageCompressionEnabled: false,
34
+ );
35
+ });
36
+
37
+ tearDown(() async {
38
+ chatService.dispose();
39
+ CrashReportService.instance.resetForTesting();
40
+ if (await tempDir.exists()) {
41
+ await tempDir.delete(recursive: true);
42
+ }
43
+ });
44
+
45
+ group('OnDeviceChatService attachments', () {
46
+ test(
47
+ 'reads and appends text document attachments to user prompt',
48
+ () async {
49
+ final docFile = File('${tempDir.path}/notes.txt');
50
+ await docFile.writeAsString('Revenue for Q3 was 5 million.');
51
+
52
+ final responses = <ChatResponse>[];
53
+ final done = chatService
54
+ .sendMessage(
55
+ server: _server(),
56
+ modelId: 'qwen3-0.6b',
57
+ messages: [
58
+ _message(
59
+ 'Summarize the report',
60
+ attachmentPaths: [docFile.path],
61
+ ),
62
+ ],
63
+ params: ChatParameters.defaults(),
64
+ )
65
+ .listen(responses.add)
66
+ .asFuture<void>();
67
+
68
+ await _waitFor(() => inferenceService.sessions.isNotEmpty);
69
+ final session = inferenceService.sessions.single;
70
+ await _waitFor(() => session.messages.isNotEmpty);
71
+
72
+ expect(session.messages.single.text, contains('Summarize the report'));
73
+ expect(session.messages.single.text, contains('--- notes.txt ---'));
74
+ expect(
75
+ session.messages.single.text,
76
+ contains('Revenue for Q3 was 5 million.'),
77
+ );
78
+ expect(session.messages.single.hasImage, isFalse);
79
+
80
+ session.responses.add(const gemma.TextResponse('Understood.'));
81
+ await session.responses.close();
82
+ await done;
83
+ },
84
+ );
85
+
86
+ test('loads image attachment and creates vision-enabled session', () async {
87
+ inferenceService.currentModelSupportsVision = true;
88
+ final imgFile = File('${tempDir.path}/sample.png');
89
+ final rawBytes = Uint8List.fromList([1, 2, 3, 4, 5]);
90
+ await imgFile.writeAsBytes(rawBytes);
91
+
92
+ final responses = <ChatResponse>[];
93
+ final done = chatService
94
+ .sendMessage(
95
+ server: _server(),
96
+ modelId: 'gemma4-e2b-instruct',
97
+ messages: [
98
+ _message('What is this image?', attachmentPaths: [imgFile.path]),
99
+ ],
100
+ params: ChatParameters.defaults(),
101
+ )
102
+ .listen(responses.add)
103
+ .asFuture<void>();
104
+
105
+ await _waitFor(() => inferenceService.sessions.isNotEmpty);
106
+ expect(inferenceService.supportImages.single, isTrue);
107
+
108
+ final session = inferenceService.sessions.single;
109
+ await _waitFor(() => session.messages.isNotEmpty);
110
+
111
+ final message = session.messages.single;
112
+ expect(message.hasImage, isTrue);
113
+ expect(message.imageBytes, equals(rawBytes));
114
+ expect(message.text, equals('What is this image?'));
115
+
116
+ session.responses.add(const gemma.TextResponse('A sample image.'));
117
+ await session.responses.close();
118
+ await done;
119
+ });
120
+
121
+ test(
122
+ 'provides fallback prompt when image is attached with empty text on vision model',
123
+ () async {
124
+ inferenceService.currentModelSupportsVision = true;
125
+ final imgFile = File('${tempDir.path}/photo.jpg');
126
+ final rawBytes = Uint8List.fromList([10, 20, 30]);
127
+ await imgFile.writeAsBytes(rawBytes);
128
+
129
+ final responses = <ChatResponse>[];
130
+ final done = chatService
131
+ .sendMessage(
132
+ server: _server(),
133
+ modelId: 'gemma4-e4b-instruct',
134
+ messages: [
135
+ _message('', attachmentPaths: [imgFile.path]),
136
+ ],
137
+ params: ChatParameters.defaults(),
138
+ )
139
+ .listen(responses.add)
140
+ .asFuture<void>();
141
+
142
+ await _waitFor(() => inferenceService.sessions.isNotEmpty);
143
+ final session = inferenceService.sessions.single;
144
+ await _waitFor(() => session.messages.isNotEmpty);
145
+
146
+ final message = session.messages.single;
147
+ expect(message.hasImage, isTrue);
148
+ expect(message.text, equals('Describe the image.'));
149
+
150
+ session.responses.add(const gemma.TextResponse('Description'));
151
+ await session.responses.close();
152
+ await done;
153
+ },
154
+ );
155
+
156
+ test(
157
+ 'silently drops image attachment when model is text-only and text is present',
158
+ () async {
159
+ inferenceService.currentModelSupportsVision = false;
160
+ final imgFile = File('${tempDir.path}/screenshot.png');
161
+ await imgFile.writeAsBytes(Uint8List.fromList([1, 2, 3]));
162
+
163
+ final responses = <ChatResponse>[];
164
+ final done = chatService
165
+ .sendMessage(
166
+ server: _server(),
167
+ modelId: 'qwen3-0.6b',
168
+ messages: [
169
+ _message('Explain this code', attachmentPaths: [imgFile.path]),
170
+ ],
171
+ params: ChatParameters.defaults(),
172
+ )
173
+ .listen(responses.add)
174
+ .asFuture<void>();
175
+
176
+ await _waitFor(() => inferenceService.sessions.isNotEmpty);
177
+ expect(inferenceService.supportImages.single, isFalse);
178
+
179
+ final session = inferenceService.sessions.single;
180
+ await _waitFor(() => session.messages.isNotEmpty);
181
+
182
+ final message = session.messages.single;
183
+ expect(message.hasImage, isFalse);
184
+ expect(message.text, equals('Explain this code'));
185
+
186
+ session.responses.add(const gemma.TextResponse('Code explanation.'));
187
+ await session.responses.close();
188
+ await done;
189
+ },
190
+ );
191
+
192
+ test(
193
+ 'returns descriptive error when user attaches only an image to a text-only model',
194
+ () async {
195
+ inferenceService.currentModelSupportsVision = false;
196
+ final imgFile = File('${tempDir.path}/screenshot.png');
197
+ await imgFile.writeAsBytes(Uint8List.fromList([1, 2, 3]));
198
+
199
+ final responses = <ChatResponse>[];
200
+ await chatService
201
+ .sendMessage(
202
+ server: _server(),
203
+ modelId: 'qwen3-0.6b',
204
+ messages: [
205
+ _message('', attachmentPaths: [imgFile.path]),
206
+ ],
207
+ params: ChatParameters.defaults(),
208
+ )
209
+ .listen(responses.add)
210
+ .asFuture<void>();
211
+
212
+ expect(responses, isNotEmpty);
213
+ final error = responses.firstWhere(
214
+ (r) => r.type == ChatResponseType.error,
215
+ );
216
+ expect(
217
+ error.content,
218
+ contains('The active model does not support image attachments'),
219
+ );
220
+ expect(inferenceService.sessions, isEmpty);
221
+ },
222
+ );
223
+
224
+ test(
225
+ 'rebuilds retained session if a follow-up adds an image when previous had none',
226
+ () async {
227
+ // First turn: text only on a vision model (or non-vision session)
228
+ inferenceService.currentModelSupportsVision = false;
229
+
230
+ final firstResponses = <ChatResponse>[];
231
+ final firstDone = chatService
232
+ .sendMessage(
233
+ server: _server(),
234
+ modelId: 'qwen3-0.6b',
235
+ messages: [_message('Hello')],
236
+ params: ChatParameters.defaults(),
237
+ )
238
+ .listen(firstResponses.add)
239
+ .asFuture<void>();
240
+
241
+ await _waitFor(() => inferenceService.sessions.isNotEmpty);
242
+ expect(inferenceService.supportImages.first, isFalse);
243
+
244
+ final firstSession = inferenceService.sessions.first;
245
+ firstSession.responses.add(const gemma.TextResponse('Hi there!'));
246
+ await firstSession.responses.close();
247
+ await firstDone;
248
+
249
+ // Second turn: model switched to vision and user attaches an image.
250
+ inferenceService.currentModelSupportsVision = true;
251
+ final imgFile = File('${tempDir.path}/diagram.png');
252
+ await imgFile.writeAsBytes(Uint8List.fromList([7, 8, 9]));
253
+
254
+ final secondResponses = <ChatResponse>[];
255
+ final secondDone = chatService
256
+ .sendMessage(
257
+ server: _server(),
258
+ modelId: 'gemma4-e2b-instruct',
259
+ messages: [
260
+ _message('Hello'),
261
+ _message('Hi there!', role: MessageRole.assistant),
262
+ _message(
263
+ 'Look at this',
264
+ id: 'user-2',
265
+ attachmentPaths: [imgFile.path],
266
+ ),
267
+ ],
268
+ params: ChatParameters.defaults(),
269
+ )
270
+ .listen(secondResponses.add)
271
+ .asFuture<void>();
272
+
273
+ await _waitFor(() => inferenceService.sessions.length == 2);
274
+ expect(inferenceService.supportImages[1], isTrue);
275
+
276
+ final secondSession = inferenceService.sessions[1];
277
+ secondSession.responses.add(const gemma.TextResponse('Seen.'));
278
+ await secondSession.responses.close();
279
+ await secondDone;
280
+ },
281
+ );
282
+ });
283
+
284
+ group('OnDeviceGemmaService vision support detection', () {
285
+ test('identifies curated vision models', () {
286
+ final gemma2b = OnDeviceModel.allCuratedModels.firstWhere(
287
+ (m) => m.id == 'gemma4-e2b-instruct',
288
+ );
289
+ final gemma4b = OnDeviceModel.allCuratedModels.firstWhere(
290
+ (m) => m.id == 'gemma4-e4b-instruct',
291
+ );
292
+ final fastVlm = OnDeviceModel.allCuratedModels.firstWhere(
293
+ (m) => m.id == 'fastvlm-0.5b',
294
+ );
295
+ final textModel = OnDeviceModel.allCuratedModels.firstWhere(
296
+ (m) => m.id == 'qwen3-0.6b',
297
+ );
298
+
299
+ expect(gemma2b.supportsVision, isTrue);
300
+ expect(gemma4b.supportsVision, isTrue);
301
+ expect(fastVlm.supportsVision, isTrue);
302
+ expect(textModel.supportsVision, isFalse);
303
+ });
304
+ });
305
+
306
+ group('Llama & MLX document attachment extraction', () {
307
+ test('OnDeviceLlamaService extracts document text in user message', () async {
308
+ final docFile = File('${tempDir.path}/llama_notes.txt');
309
+ await docFile.writeAsString('Important context for Llama.');
310
+
311
+ final llamaService = OnDeviceLlamaService();
312
+ // Test the document reading path by passing a message with attachment
313
+ final message = _message(
314
+ 'Check this context',
315
+ attachmentPaths: [docFile.path],
316
+ );
317
+
318
+ // Verify the stream returns an error for model not loaded (which verifies execution reaches engine check)
319
+ final response = await llamaService
320
+ .sendMessage(
321
+ modelId: 'non-loaded',
322
+ messages: [message],
323
+ params: ChatParameters.defaults(),
324
+ )
325
+ .first;
326
+
327
+ expect(response.type, equals(ChatResponseType.error));
328
+ expect(response.content, contains('GGUF model not loaded'));
329
+ });
330
+
331
+ test('OnDeviceMlxService extracts document text in user message', () async {
332
+ final docFile = File('${tempDir.path}/mlx_notes.txt');
333
+ await docFile.writeAsString('Important context for MLX.');
334
+
335
+ final mlxService = OnDeviceMlxService(
336
+ Dio(),
337
+ nativeInferenceAvailable: true,
338
+ isApplePlatform: true,
339
+ );
340
+
341
+ final message = _message(
342
+ 'Check this context',
343
+ attachmentPaths: [docFile.path],
344
+ );
345
+
346
+ final response = await mlxService
347
+ .sendMessage(
348
+ modelId: 'mlx-model',
349
+ messages: [message],
350
+ params: ChatParameters.defaults(),
351
+ )
352
+ .first;
353
+
354
+ expect(response.type, equals(ChatResponseType.error));
355
+ expect(response.content, contains('MLX model is not loaded'));
356
+ });
357
+ });
358
+}
359
+
360
+class _FakeInferenceService implements OnDeviceInferenceService {
361
+ final List<_FakeInferenceSession> sessions = [];
362
+ final List<String?> systemInstructions = [];
363
+ final List<bool?> supportImages = [];
364
+ int createCount = 0;
365
+
366
+ @override
367
+ bool get isLoaded => true;
368
+
369
+ @override
370
+ bool currentModelSupportsVision = false;
371
+
372
+ @override
373
+ Future<OnDeviceInferenceSession> createChat({
374
+ String? systemInstruction,
375
+ List<gemma.Tool> tools = const [],
376
+ bool? supportImage,
377
+ }) async {
378
+ createCount++;
379
+ systemInstructions.add(systemInstruction);
380
+ supportImages.add(supportImage);
381
+ final session = _FakeInferenceSession();
382
+ sessions.add(session);
383
+ return session;
384
+ }
385
+}
386
+
387
+class _FakeInferenceSession implements OnDeviceInferenceSession {
388
+ final List<StreamController<gemma.ModelResponse>> _responseStreams = [];
389
+ final List<gemma.Message> messages = [];
390
+ int stopCount = 0;
391
+ int closeCount = 0;
392
+ bool didGenerate = false;
393
+
394
+ StreamController<gemma.ModelResponse> get responses => _responseStreams.last;
395
+
396
+ @override
397
+ Future<void> addQueryChunk(gemma.Message message) async {
398
+ messages.add(message);
399
+ }
400
+
401
+ @override
402
+ Stream<gemma.ModelResponse> generateChatResponseAsync() {
403
+ didGenerate = true;
404
+ final responses = StreamController<gemma.ModelResponse>();
405
+ _responseStreams.add(responses);
406
+ return responses.stream;
407
+ }
408
+
409
+ @override
410
+ Future<void> stopGeneration() async {
411
+ stopCount++;
412
+ }
413
+
414
+ @override
415
+ Future<void> close() async {
416
+ closeCount++;
417
+ }
418
+}
419
+
420
+Future<void> _waitFor(bool Function() condition) async {
421
+ for (var attempt = 0; attempt < 100; attempt++) {
422
+ if (condition()) return;
423
+ await Future<void>.delayed(const Duration(milliseconds: 1));
424
+ }
425
+ fail('Condition was not met');
426
+}
427
+
428
+Message _message(
429
+ String content, {
430
+ String? id,
431
+ MessageRole role = MessageRole.user,
432
+ List<String>? attachmentPaths,
433
+}) => Message(
434
+ id: id ?? content,
435
+ conversationId: 'conversation',
436
+ role: role,
437
+ content: content,
438
+ createdAt: DateTime.utc(2026, 8, 9),
439
+ attachmentPaths: attachmentPaths,
440
+);
441
+
442
+Server _server() => Server(
443
+ id: 'on-device',
444
+ name: 'On-Device',
445
+ type: ServerType.onDevice,
446
+ host: '',
447
+ port: 0,
448
+ createdAt: DateTime.utc(2026, 8, 9),
449
+ lastConnectedAt: DateTime.utc(2026, 8, 9),
450
+ status: ConnectionStatus.connected,
451
+);
test/features/on_device/on_device_chat_service_lifecycle_test.dart
+6
@@ -687,18 +687,24 @@ void main() {
687
class _FakeInferenceService implements OnDeviceInferenceService {
688
final List<_FakeInferenceSession> sessions = [];
689
final List<String?> systemInstructions = [];
690
+ final List<bool?> supportImages = [];
691
int createCount = 0;
692
693
@override
694
bool get isLoaded => true;
695
696
+ @override
697
+ bool currentModelSupportsVision = false;
698
+
699
@override
700
Future<OnDeviceInferenceSession> createChat({
701
String? systemInstruction,
702
List<gemma.Tool> tools = const [],
703
+ bool? supportImage,
704
}) async {
705
createCount++;
706
systemInstructions.add(systemInstruction);
707
+ supportImages.add(supportImage);
708
final session = _FakeInferenceSession();
709
sessions.add(session);
710
return session;