@tduncan / Cortana / commits / 2d4ccd1

fix(on-device): size Gemma context from settings and recover broken engine #84

LiteRT Gemma models were always loaded with the package default of 2048 tokens, so the retained multi-turn session overflowed after a few turns and the native invoke failed. The broken model instance was then reused, so every later chat failed too. - Load Gemma with the user's context length (also for lazy restore). - Detect native engine failures, drop the retained session, and reload the model so the next message starts on a fresh engine. - Show a localized explanation instead of the raw native error. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

abdulmominsakib committed Sep 28, 2026 at 19:58 UTC 2d4ccd1f39dc51b06a9eb942b6398c02dff3e7ed
38 files changed +348 -30
lib/features/chat/data/chat_api_error.dart
+2
@@ -7,6 +7,8 @@ class ChatApiError {
7 static const onDeviceVisionNotSupportedCode =
8 'on_device_vision_not_supported';
9
10 + static const onDeviceEngineFailedCode = 'on_device_engine_failed';
11 +
12 final String message;
13 final String? type;
14 final String? code;
lib/features/chat/views/components/chat_bubble/chat_error_display.dart
+8 -4
@@ -21,10 +21,14 @@ class ChatErrorDisplay extends StatelessWidget {
21 );
22 }
23
24 - final displayMessage =
25 - parsed.code == ChatApiError.onDeviceVisionNotSupportedCode
26 - ? AppLocalizations.of(context)!.gguf_vision_not_supported_error
27 - : parsed.message;
24 + final l10n = AppLocalizations.of(context)!;
25 + final displayMessage = switch (parsed.code) {
26 + ChatApiError.onDeviceVisionNotSupportedCode =>
27 + l10n.gguf_vision_not_supported_error,
28 + ChatApiError.onDeviceEngineFailedCode =>
29 + l10n.on_device_engine_failed_error,
30 + _ => parsed.message,
31 + };
32
33 final metaParts = <String>[];
34 if (parsed.type != null && parsed.type!.isNotEmpty) {
lib/features/on_device/data/on_device_chat_service.dart
+39 -1
@@ -193,7 +193,7 @@ class OnDeviceChatService implements ChatService {
193 onError: (Object error, StackTrace stackTrace) async {
194 run.subscription = null;
195 Log.error('OnDevice stream error: $error');
196 - await _finishWithError(run, 'Inference error: $error');
196 + await _finishWithInferenceError(run, error);
197 },
198 cancelOnError: true,
199 );
@@ -204,8 +204,46 @@ class OnDeviceChatService implements ChatService {
204 return;
205 }
206 Log.error('OnDevice inference error: $error\n$stackTrace');
207 + await _finishWithInferenceError(run, error);
208 + }
209 + }
210 +
211 + /// Reports a failed run. When the native engine is left unusable, the
212 + /// retained session is dropped and the model is reloaded in the background
213 + /// so the next message (or a new chat) doesn't hit the same broken engine.
214 + Future<void> _finishWithInferenceError(
215 + _InferenceRun run,
216 + Object error,
217 + ) async {
218 + if (!isCorruptedOnDeviceEngineError(error)) {
219 await _finishWithError(run, 'Inference error: $error');
220 + return;
221 + }
222 +
223 + final retained = _retainedConversation;
224 + _retainedConversation = null;
225 + _cancelRetainedExpiration();
226 + await _finishWithError(
227 + run,
228 + ChatApiError(
229 + message: 'Inference error: $error',
230 + code: ChatApiError.onDeviceEngineFailedCode,
231 + ).encode(),
232 + );
233 + if (retained != null) {
234 + try {
235 + await retained.session.close();
236 + } catch (closeError) {
237 + Log.warning('Error closing retained on-device session: $closeError');
238 + }
239 }
240 + unawaited(
241 + _gemmaService.recoverAfterInferenceFailure().catchError((
242 + Object recoveryError,
243 + ) {
244 + Log.error('On-device model recovery failed: $recoveryError');
245 + }),
246 + );
247 }
248
249 void _handleResponse(_InferenceRun run, gemma.ModelResponse response) {
lib/features/on_device/data/on_device_gemma_service.dart
+43 -8
@@ -28,6 +28,20 @@ abstract interface class OnDeviceInferenceService {
28 List<Tool> tools,
29 bool? supportImage,
30 });
31 +
32 + /// Rebuilds the native engine after a failed inference left it unusable.
33 + Future<void> recoverAfterInferenceFailure();
34 +}
35 +
36 +/// Whether [error] indicates the native LiteRT engine is left in a broken
37 +/// state (e.g. after the KV cache overflowed), so later chats on the same
38 +/// model instance would keep failing until it is reloaded.
39 +bool isCorruptedOnDeviceEngineError(Object error) {
40 + final message = error.toString().toLowerCase();
41 + return message.contains('failed to invoke the compiled model') ||
42 + message.contains('per_layer_embedding_lookup') ||
43 + message.contains('failed_precondition') ||
44 + message.contains('internal: error');
45 }
46
47 class _GemmaInferenceSession implements OnDeviceInferenceSession {
@@ -50,9 +64,14 @@ class _GemmaInferenceSession implements OnDeviceInferenceSession {
64 }
65
66 class OnDeviceGemmaService implements OnDeviceInferenceService {
67 + /// Total token budget (prompt + history + output) used when the caller
68 + /// doesn't pass one.
69 + static const defaultMaxTokens = 4096;
70 +
71 InferenceModel? _model;
72 String? _currentModelId;
73 PreferredBackend? _currentBackend;
74 + int _currentMaxTokens = defaultMaxTokens;
75 bool _isDisposed = false;
76 Future<void>? _recoveryFuture;
77
@@ -85,7 +104,7 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
104 /// spec so that [createChat] can lazy-load the model on first use after a
105 /// restart. Sets both [_currentModelId] and [_currentBackend] so the
106 /// lazy-load path in [createChat] can actually trigger.
88 - void syncFromRestoredSpec(PreferredBackend backend) {
107 + void syncFromRestoredSpec(PreferredBackend backend, {int? maxTokens}) {
108 final specName = restoredActiveModelName;
109 if (specName == null) return;
110
@@ -96,6 +115,7 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
115
116 _currentModelId = model.id;
117 _currentBackend = backend;
118 + if (maxTokens != null) _currentMaxTokens = maxTokens;
119 }
120
121 static Future<void> initialize({String? huggingFaceToken}) async {
@@ -196,15 +216,17 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
216 Future<void> loadModel(
217 String modelId,
218 PreferredBackend backend, {
199 - int maxTokens = 2048,
219 + int? maxTokens,
220 }) async {
221 + final effectiveMaxTokens = maxTokens ?? _currentMaxTokens;
222 if (_isDisposed) {
223 throw StateError('OnDeviceGemmaService has been disposed');
224 }
225
226 if (_model != null &&
227 _currentModelId == modelId &&
207 - _currentBackend == backend) {
228 + _currentBackend == backend &&
229 + _currentMaxTokens == effectiveMaxTokens) {
230 return;
231 }
232
@@ -213,7 +235,8 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
235 }
236
237 Log.info(
216 - 'Loading model $modelId with backend=$backend, maxTokens=$maxTokens',
238 + 'Loading model $modelId with backend=$backend, '
239 + 'maxTokens=$effectiveMaxTokens',
240 );
241
242 final model = OnDeviceModel.allCuratedModels.firstWhere(
@@ -249,13 +272,14 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
272 FlutterGemmaPlugin.instance.modelManager.setActiveModel(spec);
273
274 _model = await FlutterGemma.getActiveModel(
252 - maxTokens: maxTokens,
275 + maxTokens: effectiveMaxTokens,
276 preferredBackend: backend,
277 supportImage: model.supportsVision,
278 maxNumImages: model.supportsVision ? 5 : null,
279 );
280 _currentModelId = modelId;
281 _currentBackend = backend;
282 + _currentMaxTokens = effectiveMaxTokens;
283
284 Log.info('Model $modelId loaded successfully');
285 }
@@ -275,10 +299,12 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
299 supportImage: supportImage,
300 );
301 } catch (e) {
278 - if (!_isClosedClientError(e)) rethrow;
302 + if (!_isClosedClientError(e) && !isCorruptedOnDeviceEngineError(e)) {
303 + rethrow;
304 + }
305
306 Log.warning(
281 - 'Gemma FFI model closed while opening an independent chat ($e). '
307 + 'Gemma model unusable while opening a chat ($e). '
308 'Reloading the model once.',
309 );
310 await _recoverClosedModel();
@@ -340,6 +366,15 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
366 message.contains('model was closed');
367 }
368
369 + @override
370 + Future<void> recoverAfterInferenceFailure() {
371 + if (_isDisposed || _currentModelId == null || _currentBackend == null) {
372 + return Future.value();
373 + }
374 + Log.warning('Reloading on-device model after a native inference failure');
375 + return _recoverClosedModel();
376 + }
377 +
378 Future<void> _recoverClosedModel() {
379 final existing = _recoveryFuture;
380 if (existing != null) return existing;
@@ -362,7 +397,7 @@ class OnDeviceGemmaService implements OnDeviceInferenceService {
397 }
398 }
399 _model = null;
365 - await loadModel(modelId, backend);
400 + await loadModel(modelId, backend, maxTokens: _currentMaxTokens);
401 }();
402
403 _recoveryFuture = recovery;
lib/features/on_device/providers/on_device_providers.dart
+12 -2
@@ -372,7 +372,10 @@ class OnDeviceEngineNotifier extends Notifier<OnDeviceEngineState> {
372 : settings.preferredBackend;
373
374 // Sync the service so createChat can lazy-load on first use.
375 - _gemmaService.syncFromRestoredSpec(backend);
375 + _gemmaService.syncFromRestoredSpec(
376 + backend,
377 + maxTokens: settings.contextLength,
378 + );
379
380 Log.info(
381 'Restored active inference model spec: ${model.id} (${model.fileName})',
@@ -482,10 +485,17 @@ class OnDeviceEngineNotifier extends Notifier<OnDeviceEngineState> {
485 return;
486 }
487
488 + final settings = ref.read(settingsProvider);
489 await _llamaService.unloadModel();
490 await _mlxService.unloadModel();
491 if (!ref.mounted) return;
488 - await _gemmaService.loadModel(modelId, effectiveBackend);
492 + // Size the LiteRT KV cache from the user's context setting; the
493 + // package default (2048) overflows after a few turns (#84).
494 + await _gemmaService.loadModel(
495 + modelId,
496 + effectiveBackend,
497 + maxTokens: settings.contextLength,
498 + );
499 if (!ref.mounted) return;
500 }
501
lib/l10n/app_ar.arb
+2 -1
@@ -935,5 +935,6 @@
935 "model_reasoning_default": "الإعداد الافتراضي للنموذج",
936 "model_reasoning_on": "تشغيل",
937 "model_reasoning_help": "يُطبّق على الرد التالي. يؤدي الإيقاف إلى تعطيل التفكير إذا كان قالب المحادثة الخاص بالنموذج يدعم ذلك.",
938 - "model_reasoning_save_error": "تعذّر حفظ وضع التفكير. يُرجى المحاولة مرة أخرى."
938 + "model_reasoning_save_error": "تعذّر حفظ وضع التفكير. يُرجى المحاولة مرة أخرى.",
939 + "on_device_engine_failed_error": "توقف النموذج على الجهاز عن الاستجابة، غالبًا لأن المحادثة تجاوزت نافذة السياق. تتم إعادة تحميل النموذج. حاول مرة أخرى، أو ابدأ محادثة جديدة، أو زد طول السياق من الإعدادات."
940 }
lib/l10n/app_bn.arb
+2 -1
@@ -931,5 +931,6 @@
931 "model_reasoning_default": "মডেলের ডিফল্ট",
932 "model_reasoning_on": "চালু",
933 "model_reasoning_help": "পরবর্তী উত্তরে প্রযোজ্য হবে। মডেলের চ্যাট টেমপ্লেট সমর্থন করলে বন্ধ করলে চিন্তন বন্ধ হবে।",
934 - "model_reasoning_save_error": "চিন্তন মোড সংরক্ষণ করা যায়নি। আবার চেষ্টা করুন।"
934 + "model_reasoning_save_error": "চিন্তন মোড সংরক্ষণ করা যায়নি। আবার চেষ্টা করুন।",
935 + "on_device_engine_failed_error": "অন-ডিভাইস মডেলটি সাড়া দেওয়া বন্ধ করেছে, সাধারণত কথোপকথন কনটেক্সট উইন্ডোর চেয়ে বড় হয়ে যাওয়ায়। মডেলটি আবার লোড হচ্ছে। আবার চেষ্টা করুন, নতুন চ্যাট শুরু করুন, অথবা সেটিংসে কনটেক্সট দৈর্ঘ্য বাড়ান।"
936 }
lib/l10n/app_de.arb
+2 -1
@@ -928,5 +928,6 @@
928 "model_reasoning_default": "Modellstandard",
929 "model_reasoning_on": "Ein",
930 "model_reasoning_help": "Gilt für die nächste Antwort. Aus deaktiviert das Denken, sofern die Chatvorlage des Modells dies unterstützt.",
931 - "model_reasoning_save_error": "Denkmodus konnte nicht gespeichert werden. Bitte erneut versuchen."
931 + "model_reasoning_save_error": "Denkmodus konnte nicht gespeichert werden. Bitte erneut versuchen.",
932 + "on_device_engine_failed_error": "Das On-Device-Modell reagiert nicht mehr, meist weil die Unterhaltung das Kontextfenster überschritten hat. Das Modell wird neu geladen. Versuche es erneut, starte einen neuen Chat oder erhöhe die Kontextlänge in den Einstellungen."
933 }
lib/l10n/app_en.arb
+2 -1
@@ -3451,5 +3451,6 @@
3451 "model_reasoning_default": "Model default",
3452 "model_reasoning_on": "On",
3453 "model_reasoning_help": "Applies to the next response. Off disables thinking when supported by the model’s chat template.",
3454 - "model_reasoning_save_error": "Could not save thinking mode. Please try again."
3454 + "model_reasoning_save_error": "Could not save thinking mode. Please try again.",
3455 + "on_device_engine_failed_error": "The on-device model stopped responding, usually because the conversation grew past its context window. The model is being reloaded. Try again, start a new chat, or increase the context length in settings."
3456 }
lib/l10n/app_es.arb
+2 -1
@@ -931,5 +931,6 @@
931 "model_reasoning_default": "Predeterminado del modelo",
932 "model_reasoning_on": "Activado",
933 "model_reasoning_help": "Se aplica a la siguiente respuesta. Desactivado impide el razonamiento si la plantilla de chat del modelo lo permite.",
934 - "model_reasoning_save_error": "No se pudo guardar el modo de razonamiento. Inténtalo de nuevo."
934 + "model_reasoning_save_error": "No se pudo guardar el modo de razonamiento. Inténtalo de nuevo.",
935 + "on_device_engine_failed_error": "El modelo en el dispositivo dejó de responder, normalmente porque la conversación superó su ventana de contexto. El modelo se está recargando. Inténtalo de nuevo, inicia un chat nuevo o aumenta la longitud de contexto en los ajustes."
936 }
lib/l10n/app_fr.arb
+2 -1
@@ -928,5 +928,6 @@
928 "model_reasoning_default": "Par défaut du modèle",
929 "model_reasoning_on": "Activé",
930 "model_reasoning_help": "S’applique à la prochaine réponse. Désactivé coupe le raisonnement si le modèle de conversation le permet.",
931 - "model_reasoning_save_error": "Impossible d’enregistrer le mode de réflexion. Veuillez réessayer."
931 + "model_reasoning_save_error": "Impossible d’enregistrer le mode de réflexion. Veuillez réessayer.",
932 + "on_device_engine_failed_error": "Le modèle sur l'appareil ne répond plus, généralement parce que la conversation a dépassé sa fenêtre de contexte. Le modèle est en cours de rechargement. Réessayez, démarrez une nouvelle discussion ou augmentez la longueur du contexte dans les paramètres."
933 }
lib/l10n/app_hi.arb
+2 -1
@@ -931,5 +931,6 @@
931 "model_reasoning_default": "मॉडल का डिफ़ॉल्ट",
932 "model_reasoning_on": "चालू",
933 "model_reasoning_help": "अगले उत्तर पर लागू होगा। बंद करने से चिंतन अक्षम होगा, यदि मॉडल का चैट टेम्पलेट इसका समर्थन करता है।",
934 - "model_reasoning_save_error": "चिंतन मोड सहेजा नहीं जा सका। फिर से कोशिश करें।"
934 + "model_reasoning_save_error": "चिंतन मोड सहेजा नहीं जा सका। फिर से कोशिश करें।",
935 + "on_device_engine_failed_error": "ऑन-डिवाइस मॉडल ने जवाब देना बंद कर दिया, आमतौर पर इसलिए कि बातचीत उसकी कॉन्टेक्स्ट विंडो से बड़ी हो गई। मॉडल फिर से लोड हो रहा है। फिर से कोशिश करें, नई चैट शुरू करें, या सेटिंग्स में कॉन्टेक्स्ट लंबाई बढ़ाएँ।"
936 }
lib/l10n/app_it.arb
+2 -1
@@ -931,5 +931,6 @@
931 "model_reasoning_default": "Predefinito del modello",
932 "model_reasoning_on": "Attivo",
933 "model_reasoning_help": "Si applica alla prossima risposta. Disattivato impedisce il ragionamento se il template di chat del modello lo supporta.",
934 - "model_reasoning_save_error": "Impossibile salvare la modalità di ragionamento. Riprova."
934 + "model_reasoning_save_error": "Impossibile salvare la modalità di ragionamento. Riprova.",
935 + "on_device_engine_failed_error": "Il modello sul dispositivo ha smesso di rispondere, di solito perché la conversazione ha superato la finestra di contesto. Il modello viene ricaricato. Riprova, avvia una nuova chat o aumenta la lunghezza del contesto nelle impostazioni."
936 }
lib/l10n/app_ja.arb
+2 -1
@@ -931,5 +931,6 @@
931 "model_reasoning_default": "モデルのデフォルト",
932 "model_reasoning_on": "オン",
933 "model_reasoning_help": "次の応答に適用されます。モデルのチャットテンプレートが対応している場合、オフにすると思考を無効にします。",
934 - "model_reasoning_save_error": "思考モードを保存できませんでした。もう一度お試しください。"
934 + "model_reasoning_save_error": "思考モードを保存できませんでした。もう一度お試しください。",
935 + "on_device_engine_failed_error": "オンデバイスモデルが応答しなくなりました。通常、会話がコンテキストウィンドウを超えたことが原因です。モデルを再読み込みしています。もう一度試すか、新しいチャットを開始するか、設定でコンテキスト長を増やしてください。"
936 }
lib/l10n/app_ko.arb
+2 -1
@@ -942,5 +942,6 @@
942 "model_reasoning_default": "모델 기본값",
943 "model_reasoning_on": "켜짐",
944 "model_reasoning_help": "다음 응답에 적용됩니다. 모델의 채팅 템플릿이 지원하는 경우 끄면 사고가 비활성화됩니다.",
945 - "model_reasoning_save_error": "사고 모드를 저장하지 못했습니다. 다시 시도해 주세요."
945 + "model_reasoning_save_error": "사고 모드를 저장하지 못했습니다. 다시 시도해 주세요.",
946 + "on_device_engine_failed_error": "온디바이스 모델이 응답을 멈췄습니다. 대개 대화가 컨텍스트 창을 초과했기 때문입니다. 모델을 다시 불러오는 중입니다. 다시 시도하거나 새 채팅을 시작하거나 설정에서 컨텍스트 길이를 늘리세요."
947 }
lib/l10n/app_localizations.dart
+6
@@ -5846,6 +5846,12 @@ abstract class AppLocalizations {
5846 /// In en, this message translates to:
5847 /// **'Could not save thinking mode. Please try again.'**
5848 String get model_reasoning_save_error;
5849 +
5850 + /// No description provided for @on_device_engine_failed_error.
5851 + ///
5852 + /// In en, this message translates to:
5853 + /// **'The on-device model stopped responding, usually because the conversation grew past its context window. The model is being reloaded. Try again, start a new chat, or increase the context length in settings.'**
5854 + String get on_device_engine_failed_error;
5855 }
5856
5857 class _AppLocalizationsDelegate
lib/l10n/app_localizations_ar.dart
+4
@@ -3134,4 +3134,8 @@ class AppLocalizationsAr extends AppLocalizations {
3134 @override
3135 String get model_reasoning_save_error =>
3136 'تعذّر حفظ وضع التفكير. يُرجى المحاولة مرة أخرى.';
3137 +
3138 + @override
3139 + String get on_device_engine_failed_error =>
3140 + 'توقف النموذج على الجهاز عن الاستجابة، غالبًا لأن المحادثة تجاوزت نافذة السياق. تتم إعادة تحميل النموذج. حاول مرة أخرى، أو ابدأ محادثة جديدة، أو زد طول السياق من الإعدادات.';
3141 }
lib/l10n/app_localizations_bn.dart
+4
@@ -3160,4 +3160,8 @@ class AppLocalizationsBn extends AppLocalizations {
3160 @override
3161 String get model_reasoning_save_error =>
3162 'চিন্তন মোড সংরক্ষণ করা যায়নি। আবার চেষ্টা করুন।';
3163 +
3164 + @override
3165 + String get on_device_engine_failed_error =>
3166 + 'অন-ডিভাইস মডেলটি সাড়া দেওয়া বন্ধ করেছে, সাধারণত কথোপকথন কনটেক্সট উইন্ডোর চেয়ে বড় হয়ে যাওয়ায়। মডেলটি আবার লোড হচ্ছে। আবার চেষ্টা করুন, নতুন চ্যাট শুরু করুন, অথবা সেটিংসে কনটেক্সট দৈর্ঘ্য বাড়ান।';
3167 }
lib/l10n/app_localizations_de.dart
+4
@@ -3149,4 +3149,8 @@ class AppLocalizationsDe extends AppLocalizations {
3149 @override
3150 String get model_reasoning_save_error =>
3151 'Denkmodus konnte nicht gespeichert werden. Bitte erneut versuchen.';
3152 +
3153 + @override
3154 + String get on_device_engine_failed_error =>
3155 + 'Das On-Device-Modell reagiert nicht mehr, meist weil die Unterhaltung das Kontextfenster überschritten hat. Das Modell wird neu geladen. Versuche es erneut, starte einen neuen Chat oder erhöhe die Kontextlänge in den Einstellungen.';
3156 }
lib/l10n/app_localizations_en.dart
+4
@@ -3139,4 +3139,8 @@ class AppLocalizationsEn extends AppLocalizations {
3139 @override
3140 String get model_reasoning_save_error =>
3141 'Could not save thinking mode. Please try again.';
3142 +
3143 + @override
3144 + String get on_device_engine_failed_error =>
3145 + 'The on-device model stopped responding, usually because the conversation grew past its context window. The model is being reloaded. Try again, start a new chat, or increase the context length in settings.';
3146 }
lib/l10n/app_localizations_es.dart
+4
@@ -3163,4 +3163,8 @@ class AppLocalizationsEs extends AppLocalizations {
3163 @override
3164 String get model_reasoning_save_error =>
3165 'No se pudo guardar el modo de razonamiento. Inténtalo de nuevo.';
3166 +
3167 + @override
3168 + String get on_device_engine_failed_error =>
3169 + 'El modelo en el dispositivo dejó de responder, normalmente porque la conversación superó su ventana de contexto. El modelo se está recargando. Inténtalo de nuevo, inicia un chat nuevo o aumenta la longitud de contexto en los ajustes.';
3170 }
lib/l10n/app_localizations_fr.dart
+4
@@ -3152,4 +3152,8 @@ class AppLocalizationsFr extends AppLocalizations {
3152 @override
3153 String get model_reasoning_save_error =>
3154 'Impossible d’enregistrer le mode de réflexion. Veuillez réessayer.';
3155 +
3156 + @override
3157 + String get on_device_engine_failed_error =>
3158 + 'Le modèle sur l\'appareil ne répond plus, généralement parce que la conversation a dépassé sa fenêtre de contexte. Le modèle est en cours de rechargement. Réessayez, démarrez une nouvelle discussion ou augmentez la longueur du contexte dans les paramètres.';
3159 }
lib/l10n/app_localizations_hi.dart
+4
@@ -3150,4 +3150,8 @@ class AppLocalizationsHi extends AppLocalizations {
3150 @override
3151 String get model_reasoning_save_error =>
3152 'चिंतन मोड सहेजा नहीं जा सका। फिर से कोशिश करें।';
3153 +
3154 + @override
3155 + String get on_device_engine_failed_error =>
3156 + 'ऑन-डिवाइस मॉडल ने जवाब देना बंद कर दिया, आमतौर पर इसलिए कि बातचीत उसकी कॉन्टेक्स्ट विंडो से बड़ी हो गई। मॉडल फिर से लोड हो रहा है। फिर से कोशिश करें, नई चैट शुरू करें, या सेटिंग्स में कॉन्टेक्स्ट लंबाई बढ़ाएँ।';
3157 }
lib/l10n/app_localizations_it.dart
+4
@@ -3166,4 +3166,8 @@ class AppLocalizationsIt extends AppLocalizations {
3166 @override
3167 String get model_reasoning_save_error =>
3168 'Impossibile salvare la modalità di ragionamento. Riprova.';
3169 +
3170 + @override
3171 + String get on_device_engine_failed_error =>
3172 + 'Il modello sul dispositivo ha smesso di rispondere, di solito perché la conversazione ha superato la finestra di contesto. Il modello viene ricaricato. Riprova, avvia una nuova chat o aumenta la lunghezza del contesto nelle impostazioni.';
3173 }
lib/l10n/app_localizations_ja.dart
+4
@@ -3088,4 +3088,8 @@ class AppLocalizationsJa extends AppLocalizations {
3088
3089 @override
3090 String get model_reasoning_save_error => '思考モードを保存できませんでした。もう一度お試しください。';
3091 +
3092 + @override
3093 + String get on_device_engine_failed_error =>
3094 + 'オンデバイスモデルが応答しなくなりました。通常、会話がコンテキストウィンドウを超えたことが原因です。モデルを再読み込みしています。もう一度試すか、新しいチャットを開始するか、設定でコンテキスト長を増やしてください。';
3095 }
lib/l10n/app_localizations_ko.dart
+4
@@ -3071,4 +3071,8 @@ class AppLocalizationsKo extends AppLocalizations {
3071
3072 @override
3073 String get model_reasoning_save_error => '사고 모드를 저장하지 못했습니다. 다시 시도해 주세요.';
3074 +
3075 + @override
3076 + String get on_device_engine_failed_error =>
3077 + '온디바이스 모델이 응답을 멈췄습니다. 대개 대화가 컨텍스트 창을 초과했기 때문입니다. 모델을 다시 불러오는 중입니다. 다시 시도하거나 새 채팅을 시작하거나 설정에서 컨텍스트 길이를 늘리세요.';
3078 }
lib/l10n/app_localizations_pt.dart
+4
@@ -3180,4 +3180,8 @@ class AppLocalizationsPt extends AppLocalizations {
3180 @override
3181 String get model_reasoning_save_error =>
3182 'Não foi possível salvar o modo de raciocínio. Tente novamente.';
3183 +
3184 + @override
3185 + String get on_device_engine_failed_error =>
3186 + 'O modelo no dispositivo parou de responder, geralmente porque a conversa ultrapassou a janela de contexto. O modelo está sendo recarregado. Tente novamente, inicie um novo chat ou aumente o comprimento do contexto nas configurações.';
3187 }
lib/l10n/app_localizations_ru.dart
+4
@@ -3164,4 +3164,8 @@ class AppLocalizationsRu extends AppLocalizations {
3164 @override
3165 String get model_reasoning_save_error =>
3166 'Не удалось сохранить режим рассуждений. Попробуйте ещё раз.';
3167 +
3168 + @override
3169 + String get on_device_engine_failed_error =>
3170 + 'Локальная модель перестала отвечать — обычно потому, что диалог превысил окно контекста. Модель перезагружается. Попробуйте снова, начните новый чат или увеличьте длину контекста в настройках.';
3171 }
lib/l10n/app_localizations_tr.dart
+4
@@ -3170,4 +3170,8 @@ class AppLocalizationsTr extends AppLocalizations {
3170 @override
3171 String get model_reasoning_save_error =>
3172 'Düşünme modu kaydedilemedi. Lütfen tekrar deneyin.';
3173 +
3174 + @override
3175 + String get on_device_engine_failed_error =>
3176 + 'Cihaz üzerindeki model yanıt vermeyi durdurdu; genellikle sohbet bağlam penceresini aştığı için. Model yeniden yükleniyor. Tekrar deneyin, yeni bir sohbet başlatın veya ayarlardan bağlam uzunluğunu artırın.';
3177 }
lib/l10n/app_localizations_zh.dart
+8
@@ -3063,6 +3063,10 @@ class AppLocalizationsZh extends AppLocalizations {
3063
3064 @override
3065 String get model_reasoning_save_error => '无法保存思考模式。请重试。';
3066 +
3067 + @override
3068 + String get on_device_engine_failed_error =>
3069 + '设备端模型已停止响应，通常是因为对话超出了上下文窗口。正在重新加载模型。请重试、开始新对话，或在设置中增加上下文长度。';
3070 }
3071
3072 /// The translations for Chinese, as used in Taiwan (`zh_TW`).
@@ -6030,4 +6034,8 @@ class AppLocalizationsZhTw extends AppLocalizationsZh {
6034
6035 @override
6036 String get model_reasoning_save_error => '無法儲存思考模式。請再試一次。';
6037 +
6038 + @override
6039 + String get on_device_engine_failed_error =>
6040 + '裝置端模型已停止回應，通常是因為對話超出了上下文視窗。正在重新載入模型。請重試、開始新對話，或在設定中增加上下文長度。';
6041 }
lib/l10n/app_pt.arb
+2 -1
@@ -942,5 +942,6 @@
942 "model_reasoning_default": "Padrão do modelo",
943 "model_reasoning_on": "Ativado",
944 "model_reasoning_help": "Aplica-se à próxima resposta. Desativado impede o raciocínio se o modelo de chat permitir.",
945 - "model_reasoning_save_error": "Não foi possível salvar o modo de raciocínio. Tente novamente."
945 + "model_reasoning_save_error": "Não foi possível salvar o modo de raciocínio. Tente novamente.",
946 + "on_device_engine_failed_error": "O modelo no dispositivo parou de responder, geralmente porque a conversa ultrapassou a janela de contexto. O modelo está sendo recarregado. Tente novamente, inicie um novo chat ou aumente o comprimento do contexto nas configurações."
947 }
lib/l10n/app_ru.arb
+2 -1
@@ -928,5 +928,6 @@
928 "model_reasoning_default": "По умолчанию для модели",
929 "model_reasoning_on": "Вкл.",
930 "model_reasoning_help": "Применяется к следующему ответу. Выключение отключает рассуждения, если это поддерживается шаблоном чата модели.",
931 - "model_reasoning_save_error": "Не удалось сохранить режим рассуждений. Попробуйте ещё раз."
931 + "model_reasoning_save_error": "Не удалось сохранить режим рассуждений. Попробуйте ещё раз.",
932 + "on_device_engine_failed_error": "Локальная модель перестала отвечать — обычно потому, что диалог превысил окно контекста. Модель перезагружается. Попробуйте снова, начните новый чат или увеличьте длину контекста в настройках."
933 }
lib/l10n/app_tr.arb
+2 -1
@@ -928,5 +928,6 @@
928 "model_reasoning_default": "Model varsayılanı",
929 "model_reasoning_on": "Açık",
930 "model_reasoning_help": "Sonraki yanıta uygulanır. Modelin sohbet şablonu destekliyorsa Kapalı seçeneği düşünmeyi devre dışı bırakır.",
931 - "model_reasoning_save_error": "Düşünme modu kaydedilemedi. Lütfen tekrar deneyin."
931 + "model_reasoning_save_error": "Düşünme modu kaydedilemedi. Lütfen tekrar deneyin.",
932 + "on_device_engine_failed_error": "Cihaz üzerindeki model yanıt vermeyi durdurdu; genellikle sohbet bağlam penceresini aştığı için. Model yeniden yükleniyor. Tekrar deneyin, yeni bir sohbet başlatın veya ayarlardan bağlam uzunluğunu artırın."
933 }
lib/l10n/app_zh.arb
+2 -1
@@ -931,5 +931,6 @@
931 "model_reasoning_default": "模型默认",
932 "model_reasoning_on": "开启",
933 "model_reasoning_help": "应用于下一条回复。如果模型的聊天模板支持，关闭后将禁用思考。",
934 - "model_reasoning_save_error": "无法保存思考模式。请重试。"
934 + "model_reasoning_save_error": "无法保存思考模式。请重试。",
935 + "on_device_engine_failed_error": "设备端模型已停止响应，通常是因为对话超出了上下文窗口。正在重新加载模型。请重试、开始新对话，或在设置中增加上下文长度。"
936 }
lib/l10n/app_zh_TW.arb
+2 -1
@@ -931,5 +931,6 @@
931 "model_reasoning_default": "模型預設",
932 "model_reasoning_on": "開啟",
933 "model_reasoning_help": "套用至下一則回覆。如果模型的聊天範本支援，關閉後將停用思考。",
934 - "model_reasoning_save_error": "無法儲存思考模式。請再試一次。"
934 + "model_reasoning_save_error": "無法儲存思考模式。請再試一次。",
935 + "on_device_engine_failed_error": "裝置端模型已停止回應，通常是因為對話超出了上下文視窗。正在重新載入模型。請重試、開始新對話，或在設定中增加上下文長度。"
936 }
test/features/on_device/on_device_chat_attachments_test.dart
+3
@@ -541,6 +541,9 @@ class _FakeInferenceService implements OnDeviceInferenceService {
541 sessions.add(session);
542 return session;
543 }
544 +
545 + @override
546 + Future<void> recoverAfterInferenceFailure() async {}
547 }
548
549 class _FakeInferenceSession implements OnDeviceInferenceSession {
test/features/on_device/on_device_chat_service_lifecycle_test.dart
+107
@@ -4,6 +4,7 @@ import 'package:flutter_gemma/flutter_gemma.dart' as gemma;
4 import 'package:flutter_test/flutter_test.dart';
5 import 'package:localmind/core/models/enums.dart';
6 import 'package:localmind/core/services/crash_report_service.dart';
7 +import 'package:localmind/features/chat/data/chat_api_error.dart';
8 import 'package:localmind/features/chat/data/chat_service.dart';
9 import 'package:localmind/features/chat/data/models/chat_parameters.dart';
10 import 'package:localmind/features/chat/data/models/message.dart';
@@ -715,6 +716,106 @@ void main() {
716 expect(CrashReportService.instance.currentCrash.value, isNull);
717 });
718
719 + test(
720 + 'native engine failure drops the retained session and reloads (#84)',
721 + () async {
722 + // First turn succeeds and is retained for reuse.
723 + final firstDone = chatService
724 + .sendMessage(
725 + server: _server(),
726 + modelId: 'gemma4-e4b-instruct',
727 + messages: [
728 + _message('first', id: 'user-1'),
729 + _message('', id: 'assistant-1', role: MessageRole.assistant),
730 + ],
731 + params: ChatParameters.defaults(),
732 + )
733 + .drain<void>();
734 + await _waitFor(
735 + () =>
736 + inferenceService.sessions.isNotEmpty &&
737 + inferenceService.sessions.single.didGenerate,
738 + );
739 + final retained = inferenceService.sessions.single;
740 + retained.responses.add(const gemma.TextResponse('hello'));
741 + await retained.responses.close();
742 + await firstDone;
743 + expect(retained.closeCount, 0);
744 +
745 + // Second turn reuses the session and the native invoke fails.
746 + final responses = <ChatResponse>[];
747 + final secondDone = chatService
748 + .sendMessage(
749 + server: _server(),
750 + modelId: 'gemma4-e4b-instruct',
751 + messages: [
752 + _message('first', id: 'user-1'),
753 + _message('hello', id: 'assistant-1', role: MessageRole.assistant),
754 + _message('second', id: 'user-2'),
755 + _message('', id: 'assistant-2', role: MessageRole.assistant),
756 + ],
757 + params: ChatParameters.defaults(),
758 + )
759 + .listen(responses.add)
760 + .asFuture<void>();
761 + await _waitFor(() => retained.generationCount == 2);
762 + retained.responses.addError(
763 + Exception(
764 + 'Stream error: INTERNAL: ERROR: [runtime/executor/'
765 + 'llm_litert_compiled_model_executor.cc:726] '
766 + 'Failed to invoke the compiled model',
767 + ),
768 + );
769 + await secondDone;
770 +
771 + final error = responses.firstWhere(
772 + (response) => response.type == ChatResponseType.error,
773 + );
774 + expect(
775 + ChatApiError.tryParse(error.content!)?.code,
776 + ChatApiError.onDeviceEngineFailedCode,
777 + );
778 + expect(retained.closeCount, 1);
779 + await _waitFor(() => inferenceService.recoverCount == 1);
780 +
781 + // The next message opens a fresh session instead of the broken one.
782 + final thirdDone = chatService
783 + .sendMessage(
784 + server: _server(),
785 + modelId: 'gemma4-e4b-instruct',
786 + messages: [_message('new chat')],
787 + params: ChatParameters.defaults(),
788 + )
789 + .drain<void>();
790 + await _waitFor(() => inferenceService.sessions.length == 2);
791 + final fresh = inferenceService.sessions.last;
792 + await _waitFor(() => fresh.didGenerate);
793 + await fresh.responses.close();
794 + await thirdDone;
795 + expect(inferenceService.createCount, 2);
796 + },
797 + );
798 +
799 + test('generic inference errors do not reload the model', () async {
800 + final done = chatService
801 + .sendMessage(
802 + server: _server(),
803 + modelId: 'qwen3-0.6b',
804 + messages: [_message('fail')],
805 + params: ChatParameters.defaults(),
806 + )
807 + .drain<void>();
808 + await _waitFor(
809 + () =>
810 + inferenceService.sessions.isNotEmpty &&
811 + inferenceService.sessions.single.didGenerate,
812 + );
813 + inferenceService.sessions.single.responses.addError(StateError('boom'));
814 + await done;
815 + await Future<void>.delayed(const Duration(milliseconds: 5));
816 + expect(inferenceService.recoverCount, 0);
817 + });
818 +
819 test('cancelStream stops and closes every active request once', () async {
820 final subscriptions = List.generate(
821 2,
@@ -754,6 +855,7 @@ class _FakeInferenceService implements OnDeviceInferenceService {
855 final List<String?> systemInstructions = [];
856 final List<bool?> supportImages = [];
857 int createCount = 0;
858 + int recoverCount = 0;
859 bool canReuseChatSession = true;
860
861 @override
@@ -778,6 +880,11 @@ class _FakeInferenceService implements OnDeviceInferenceService {
880 sessions.add(session);
881 return session;
882 }
883 +
884 + @override
885 + Future<void> recoverAfterInferenceFailure() async {
886 + recoverCount++;
887 + }
888 }
889
890 class _FakeInferenceSession implements OnDeviceInferenceSession {
test/features/on_device/on_device_engine_error_test.dart new
+38
@@ -0,0 +1,38 @@
1 +import 'package:flutter_test/flutter_test.dart';
2 +import 'package:localmind/features/on_device/data/on_device_gemma_service.dart';
3 +
4 +void main() {
5 + group('isCorruptedOnDeviceEngineError', () {
6 + test('matches LiteRT failures reported in #84', () {
7 + expect(
8 + isCorruptedOnDeviceEngineError(
9 + Exception(
10 + 'Stream error: INTERNAL: ERROR: [runtime/executor/'
11 + 'llm_litert_compiled_model_executor.cc:726] '
12 + 'Failed to invoke the compiled model',
13 + ),
14 + ),
15 + isTrue,
16 + );
17 + expect(
18 + isCorruptedOnDeviceEngineError(
19 + Exception(
20 + 'Stream error: FAILED_PRECONDITION: Prefill requires '
21 + 'per_layer_embedding_lookup_ when signature has '
22 + 'input_per_layer_embeddings, but per_layer_embedding_lookup_ '
23 + 'is null.',
24 + ),
25 + ),
26 + isTrue,
27 + );
28 + });
29 +
30 + test('ignores unrelated errors', () {
31 + expect(isCorruptedOnDeviceEngineError(StateError('boom')), isFalse);
32 + expect(
33 + isCorruptedOnDeviceEngineError(Exception('Model not found: x')),
34 + isFalse,
35 + );
36 + });
37 + });
38 +}