12 files changed
+228
-671
android/app/build.gradle.kts
+4
@@ -39,6 +39,10 @@ android {
39
versionCode = flutter.versionCode
40
versionName = flutter.versionName
41
multiDexEnabled = true
42
+
43
+ ndk {
44
+ abiFilters += listOf("arm64-v8a")
45
+ }
46
}
47
48
signingConfigs {
lib/core/device/device_memory_service.dart
+4
-1
@@ -10,6 +10,8 @@ class DeviceMemoryInfo {
10
required this.availableMemoryMb,
11
});
12
13
+ bool get isLowRam => totalMemoryMb < 7000; // Threshold for 8GB RAM devices
14
+
15
bool hasEnoughRam(int requiredMb) => availableMemoryMb >= requiredMb;
16
bool isOversized(int minRamMb) => totalMemoryMb < minRamMb;
17
@@ -34,7 +36,8 @@ class DeviceMemoryService {
36
if (Platform.isAndroid) {
37
try {
38
final androidInfo = await _deviceInfo.androidInfo;
37
- totalMb = androidInfo.physicalRamSize; // In MB
39
+ // physicalRamSize is in bytes for device_info_plus 4.0.0+
40
+ totalMb = androidInfo.physicalRamSize ~/ (1024 * 1024);
41
} catch (_) {}
42
43
try {
lib/features/models/screens/model_picker_sheet.dart
+42
-6
@@ -765,6 +765,17 @@ class _OnDeviceModelTile extends ConsumerWidget {
765
766
return InkWell(
767
onTap: () {
768
+ if (deviceMemory?.isLowRam ?? false) {
769
+ ScaffoldMessenger.of(context).showSnackBar(
770
+ const SnackBar(
771
+ content: Text(
772
+ 'On-device models are restricted as your device has less than 8 GB RAM.',
773
+ ),
774
+ backgroundColor: Colors.orange,
775
+ ),
776
+ );
777
+ return;
778
+ }
779
if (isLoaded) {
780
_selectModel(context, ref);
781
} else if (isDownloaded && !isCurrentlyLoading && !isDownloading) {
@@ -933,17 +944,42 @@ class _OnDeviceModelTile extends ConsumerWidget {
944
onPressed: () => ref.read(foregroundDownloadNotifierProvider.notifier).cancelDownload(model.id),
945
),
946
] else ...[
936
- Icon(
937
- Icons.cloud_download_outlined,
938
- size: 18,
939
- color: isDark ? const Color(0xFF555555) : const Color(0xFF999999),
947
+ _IconButton(
948
+ icon: Icon(
949
+ Icons.cloud_download_outlined,
950
+ size: 18,
951
+ color: (deviceMemory?.isLowRam ?? false)
952
+ ? Colors.orange.withValues(alpha: 0.5)
953
+ : (isDark ? const Color(0xFF555555) : const Color(0xFF999999)),
954
+ ),
955
+ tooltip: (deviceMemory?.isLowRam ?? false)
956
+ ? 'Restricted due to low RAM'
957
+ : 'Download model',
958
+ onPressed: (deviceMemory?.isLowRam ?? false)
959
+ ? () {
960
+ ScaffoldMessenger.of(context).showSnackBar(
961
+ const SnackBar(
962
+ content: Text(
963
+ 'Downloading on-device models is restricted on devices with less than 8 GB RAM.',
964
+ ),
965
+ backgroundColor: Colors.orange,
966
+ ),
967
+ );
968
+ }
969
+ : () {
970
+ ref
971
+ .read(foregroundDownloadNotifierProvider.notifier)
972
+ .startDownload(model.id);
973
+ },
974
),
975
const SizedBox(width: 4),
976
Text(
943
- 'Not downloaded',
977
+ (deviceMemory?.isLowRam ?? false) ? 'Restricted' : 'Not downloaded',
978
style: TextStyle(
979
fontSize: 11,
946
- color: isDark ? const Color(0xFF666666) : const Color(0xFF999999),
980
+ color: (deviceMemory?.isLowRam ?? false)
981
+ ? Colors.orange.withValues(alpha: 0.7)
982
+ : (isDark ? const Color(0xFF666666) : const Color(0xFF999999)),
983
),
984
),
985
],
lib/features/on_device/views/model_manager_screen.dart
+46
-2
@@ -98,6 +98,28 @@ class OnDeviceModelManagerScreen extends ConsumerWidget {
98
style: const TextStyle(color: Colors.red),
99
),
100
),
101
+ if (deviceMemoryAsync.value?.isLowRam ?? false)
102
+ Container(
103
+ padding: const EdgeInsets.all(12),
104
+ margin: const EdgeInsets.only(bottom: 16),
105
+ decoration: BoxDecoration(
106
+ color: Colors.orange.withValues(alpha: 0.1),
107
+ borderRadius: BorderRadius.circular(8),
108
+ border: Border.all(color: Colors.orange),
109
+ ),
110
+ child: const Row(
111
+ children: [
112
+ Icon(Icons.warning_amber_rounded, color: Colors.orange, size: 20),
113
+ SizedBox(width: 8),
114
+ Expanded(
115
+ child: Text(
116
+ 'Running or downloading local LLMs is restricted on this device because it has less than 8 GB RAM. This is to ensure device stability.',
117
+ style: TextStyle(color: Colors.orange, fontWeight: FontWeight.bold),
118
+ ),
119
+ ),
120
+ ],
121
+ ),
122
+ ),
123
Text(
124
'Available Models',
125
style: theme.textTheme.titleMedium?.copyWith(
@@ -589,7 +611,18 @@ class _ModelCard extends ConsumerWidget {
611
else
612
ShadButton.outline(
613
size: ShadButtonSize.sm,
592
- onPressed: () => _loadModel(context, ref),
614
+ onPressed: (deviceMemory?.isLowRam ?? false)
615
+ ? () {
616
+ ScaffoldMessenger.of(context).showSnackBar(
617
+ const SnackBar(
618
+ content: Text(
619
+ 'Loading local LLMs is restricted on devices with less than 8 GB RAM.',
620
+ ),
621
+ backgroundColor: Colors.orange,
622
+ ),
623
+ );
624
+ }
625
+ : () => _loadModel(context, ref),
626
child: const Text('Load'),
627
),
628
const SizedBox(width: 8),
@@ -700,7 +733,18 @@ class _ModelCard extends ConsumerWidget {
733
const Spacer(),
734
ShadButton.outline(
735
size: ShadButtonSize.sm,
703
- onPressed: () => _startDownload(context, ref),
736
+ onPressed: (deviceMemory?.isLowRam ?? false)
737
+ ? () {
738
+ ScaffoldMessenger.of(context).showSnackBar(
739
+ const SnackBar(
740
+ content: Text(
741
+ 'Downloading local LLMs is restricted on devices with less than 8 GB RAM.',
742
+ ),
743
+ backgroundColor: Colors.orange,
744
+ ),
745
+ );
746
+ }
747
+ : () => _startDownload(context, ref),
748
child: const Text('Download'),
749
),
750
],
lib/features/onboarding/screens/onboarding_model_download_screen.dart
+12
-1
@@ -414,7 +414,18 @@ class _ModelCard extends ConsumerWidget {
414
else
415
ShadButton.outline(
416
size: ShadButtonSize.sm,
417
- onPressed: () => _startDownload(context, ref),
417
+ onPressed: (deviceMemory?.isLowRam ?? false)
418
+ ? () {
419
+ ScaffoldMessenger.of(context).showSnackBar(
420
+ const SnackBar(
421
+ content: Text(
422
+ 'Downloading local LLMs is restricted on devices with less than 8 GB RAM.',
423
+ ),
424
+ backgroundColor: Colors.orange,
425
+ ),
426
+ );
427
+ }
428
+ : () => _startDownload(context, ref),
429
child: const Text('Download'),
430
),
431
],
lib/features/onboarding/screens/onboarding_server_type_screen.dart
+96
-27
@@ -6,6 +6,7 @@ import 'package:shadcn_ui/shadcn_ui.dart';
6
7
import '../../../core/models/enums.dart';
8
import '../../../core/providers/app_providers.dart';
9
+import '../../../core/providers/device_info_providers.dart';
10
import '../../../core/routes/app_routes.dart';
11
12
class OnboardingServerTypeScreen extends ConsumerStatefulWidget {
@@ -20,6 +21,12 @@ class _OnboardingServerTypeScreenState
21
extends ConsumerState<OnboardingServerTypeScreen> {
22
ServerType? _selectedType;
23
24
+ @override
25
+ void initState() {
26
+ super.initState();
27
+ _selectedType = ServerType.lmStudio;
28
+ }
29
+
30
@override
31
Widget build(BuildContext context) {
32
final theme = Theme.of(context);
@@ -79,17 +86,48 @@ class _OnboardingServerTypeScreenState
86
),
87
),
88
const SizedBox(height: 24),
82
- _buildServerCard(
83
- type: ServerType.onDevice,
84
- title: 'On-Device',
85
- subtitle: 'NO SERVER NEEDED',
86
- iconWidget: Icon(
87
- Icons.phone_android_rounded,
88
- color: _selectedType == ServerType.onDevice
89
- ? theme.colorScheme.primary
90
- : theme.colorScheme.onSurface,
91
- ),
92
- theme: theme,
89
+ Consumer(
90
+ builder: (context, ref, child) {
91
+ final memoryAsync = ref.watch(deviceMemoryProvider);
92
+ return memoryAsync.maybeWhen(
93
+ data: (memory) {
94
+ final isLowRam = memory.isLowRam;
95
+ return _buildServerCard(
96
+ type: ServerType.onDevice,
97
+ title: 'On-Device',
98
+ subtitle: isLowRam
99
+ ? 'RESTRICTED (LOW RAM)'
100
+ : 'NO SERVER NEEDED',
101
+ iconWidget: Icon(
102
+ Icons.phone_android_rounded,
103
+ color: isLowRam
104
+ ? theme.colorScheme.onSurface.withValues(
105
+ alpha: 0.3,
106
+ )
107
+ : (_selectedType == ServerType.onDevice
108
+ ? theme.colorScheme.primary
109
+ : theme.colorScheme.onSurface),
110
+ ),
111
+ theme: theme,
112
+ disabled: isLowRam,
113
+ disabledReason:
114
+ 'Local LLMs require at least 8 GB RAM to run smoothly on your device.',
115
+ );
116
+ },
117
+ orElse: () => _buildServerCard(
118
+ type: ServerType.onDevice,
119
+ title: 'On-Device',
120
+ subtitle: 'NO SERVER NEEDED',
121
+ iconWidget: Icon(
122
+ Icons.phone_android_rounded,
123
+ color: _selectedType == ServerType.onDevice
124
+ ? theme.colorScheme.primary
125
+ : theme.colorScheme.onSurface,
126
+ ),
127
+ theme: theme,
128
+ ),
129
+ );
130
+ },
131
),
132
const SizedBox(height: 16),
133
_buildServerCard(
@@ -208,43 +246,72 @@ class _OnboardingServerTypeScreenState
246
required String subtitle,
247
required Widget iconWidget,
248
required ThemeData theme,
249
+ bool disabled = false,
250
+ String? disabledReason,
251
}) {
252
final isSelected = _selectedType == type;
253
254
return GestureDetector(
215
- onTap: () {
216
- setState(() {
217
- _selectedType = type;
218
- });
219
- },
255
+ onTap: disabled
256
+ ? () {
257
+ if (disabledReason != null) {
258
+ ScaffoldMessenger.of(context).showSnackBar(
259
+ SnackBar(
260
+ content: Text(disabledReason),
261
+ backgroundColor: theme.colorScheme.error,
262
+ ),
263
+ );
264
+ }
265
+ }
266
+ : () {
267
+ setState(() {
268
+ _selectedType = type;
269
+ });
270
+ },
271
child: Container(
272
padding: const EdgeInsets.all(20),
273
decoration: BoxDecoration(
223
- color: theme.colorScheme.surface,
274
+ color: disabled
275
+ ? theme.colorScheme.surface.withValues(alpha: 0.5)
276
+ : theme.colorScheme.surface,
277
borderRadius: BorderRadius.circular(16),
278
border: Border.all(
226
- color: isSelected
279
+ color: isSelected && !disabled
280
? theme.colorScheme.primary
281
: theme.colorScheme.outline.withValues(alpha: 0.3),
229
- width: isSelected ? 2 : 1,
282
+ width: isSelected && !disabled ? 2 : 1,
283
),
284
),
285
child: Column(
286
crossAxisAlignment: CrossAxisAlignment.start,
287
children: [
235
- Container(
236
- padding: const EdgeInsets.all(12),
237
- decoration: BoxDecoration(
238
- color: theme.scaffoldBackgroundColor,
239
- borderRadius: BorderRadius.circular(12),
240
- ),
241
- child: iconWidget,
288
+ Row(
289
+ children: [
290
+ Container(
291
+ padding: const EdgeInsets.all(12),
292
+ decoration: BoxDecoration(
293
+ color: theme.scaffoldBackgroundColor,
294
+ borderRadius: BorderRadius.circular(12),
295
+ ),
296
+ child: iconWidget,
297
+ ),
298
+ const Spacer(),
299
+ if (disabled)
300
+ const Icon(
301
+ Icons.lock_outline_rounded,
302
+ size: 20,
303
+ color: Colors.orange,
304
+ ),
305
+ ],
306
),
307
const SizedBox(height: 16),
308
Text(
309
title,
310
style: theme.textTheme.titleMedium?.copyWith(
311
fontWeight: FontWeight.bold,
312
+ color: disabled
313
+ ? theme.colorScheme.onSurface.withValues(alpha: 0.5)
314
+ : null,
315
),
316
),
317
const SizedBox(height: 4),
@@ -252,7 +319,9 @@ class _OnboardingServerTypeScreenState
319
subtitle,
320
style: theme.textTheme.labelSmall?.copyWith(
321
letterSpacing: 2,
255
- color: theme.colorScheme.onSurface.withValues(alpha: 0.5),
322
+ color: disabled
323
+ ? Colors.orange.withValues(alpha: 0.7)
324
+ : theme.colorScheme.onSurface.withValues(alpha: 0.5),
325
),
326
),
327
],
lib/features/sidebar/sidebar_widget.dart
-1
@@ -8,7 +8,6 @@ import '../../core/providers/app_providers.dart';
8
import '../../core/routes/app_routes.dart';
9
import '../../core/theme/app_theme.dart';
10
import '../chat/providers/chat_providers.dart';
11
-import '../tts/providers/tts_providers.dart' as tts;
11
import '../tts/views/components/tts_player_bar.dart';
12
import 'components/active_server_indicator.dart';
13
import 'components/conversation_drawer_header.dart';
lib/features/tts/providers/tts_providers.dart
+14
-10
@@ -1,21 +1,17 @@
1
import 'dart:async';
2
import 'dart:io';
3
import 'dart:isolate';
4
-import 'dart:typed_data';
4
5
import 'package:archive/archive.dart';
7
-import 'package:flutter/services.dart';
6
import 'package:audioplayers/audioplayers.dart';
7
+import 'package:flutter/services.dart';
8
import 'package:flutter_riverpod/flutter_riverpod.dart';
9
import 'package:flutter_tts/flutter_tts.dart';
11
-import 'package:meomeo/meomeo.dart';
10
import 'package:path_provider/path_provider.dart';
11
12
import '../../../core/logger/app_logger.dart';
13
import '../../../core/models/enums.dart';
14
import '../../../core/providers/app_providers.dart';
17
-import '../data/kitten_tts_model.dart';
18
-import '../data/kokoro_tts_model.dart';
15
import 'tts_model_providers.dart';
16
import 'tts_worker.dart';
17
@@ -148,15 +144,18 @@ class TtsNotifier extends Notifier<TtsState> {
144
if (_workerIsolate != null) return;
145
146
Log.info('Spawning TTS worker isolate...');
151
- _workerIsolate = await Isolate.spawn(ttsWorkerEntry, _mainReceivePort.sendPort);
152
-
147
+ _workerIsolate = await Isolate.spawn(
148
+ ttsWorkerEntry,
149
+ _mainReceivePort.sendPort,
150
+ );
151
+
152
// Wait for worker to send its SendPort
153
int attempts = 0;
154
while (_workerSendPort == null && attempts < 200) {
155
await Future.delayed(const Duration(milliseconds: 50));
156
attempts++;
157
}
159
-
158
+
159
if (_workerSendPort == null) {
160
throw StateError('Failed to establish communication with TTS worker');
161
}
@@ -235,7 +234,9 @@ class TtsNotifier extends Notifier<TtsState> {
234
}
235
}
236
238
- Log.info('espeak-data invalid or missing. (Re)extracting espeak-data.zip to $targetPath...');
237
+ Log.info(
238
+ 'espeak-data invalid or missing. (Re)extracting espeak-data.zip to $targetPath...',
239
+ );
240
if (await dir.exists()) {
241
try {
242
await dir.delete(recursive: true);
@@ -296,7 +297,10 @@ class TtsNotifier extends Notifier<TtsState> {
297
case EngineId.kitten:
298
final variant = ref.read(settingsProvider).kittenTtsModelVariant;
299
final kittenDl = ref.read(kittenTtsDownloaderProvider);
299
- final mPath = await kittenDl.getFilePath(variant, variant.modelFileName);
300
+ final mPath = await kittenDl.getFilePath(
301
+ variant,
302
+ variant.modelFileName,
303
+ );
304
final vPath = await kittenDl.getFilePath(variant, 'voices.npz');
305
if (mPath == null || vPath == null) {
306
throw StateError(
lib/main.dart
+1
-6
@@ -26,10 +26,5 @@ Future<void> main() async {
26
// Initialize services
27
await container.read(downloadNotificationServiceProvider).init();
28
29
- runApp(
30
- UncontrolledProviderScope(
31
- container: container,
32
- child: const App(),
33
- ),
34
- );
29
+ runApp(UncontrolledProviderScope(container: container, child: const App()));
30
}
plan_tts.md
deleted
-608
@@ -1,608 +0,0 @@
1
-# Feature Plan: On-Device Neural TTS for Flutter
2
-
3
-## Overview
4
-
5
-Port the PocketPal AI TTS system to a Flutter app. The system uses **on-device neural TTS engines** (no cloud APIs) orchestrated through a custom native module. All model files are downloaded at runtime from HuggingFace and run locally via ONNX Runtime.
6
-
7
----
8
-
9
-## Architecture
10
-
11
-### High-Level Layers
12
-
13
-```
14
-┌──────────────────────────────────────────────────┐
15
-│ UI Layer (Flutter widgets) │
16
-│ - TTSSetupSheet (engine cards, voice picker) │
17
-│ - PlayButton, VoiceChip │
18
-│ - AutoSpeakRow, HeroRow (quality selector) │
19
-└──────────────────────┬───────────────────────────┘
20
- │ calls
21
-┌──────────────────────▼───────────────────────────┐
22
-│ TTS Store (State Management) │
23
-│ - orchestrates playback, download lifecycle │
24
-│ - per-engine download state machines │
25
-│ - streaming vs replay paths │
26
-│ - memory gate (4 GiB minimum RAM) │
27
-│ - thinking stripper for LLM <think> tags │
28
-└──────────────────────┬───────────────────────────┘
29
- │ calls
30
-┌──────────────────────▼───────────────────────────┐
31
-│ TTS Runtime (singleton coordinator) │
32
-│ - serializes engine swap (only 1 engine at a time)│
33
-│ - acquire/release lifecycle │
34
-│ - stop ordering via mutex chain │
35
-└──────────────────────┬───────────────────────────┘
36
- │ calls
37
-┌──────────────────────▼───────────────────────────┐
38
-│ Engine Implementations │
39
-│ ┌─────────┬──────────┬────────────┬──────────┐ │
40
-│ │ Kitten │ Kokoro │ Supertonic │ System │ │
41
-│ │ (nano) │ (medium) │ (large) │ (OS TTS) │ │
42
-│ └─────────┴──────────┴────────────┴──────────┘ │
43
-└──────────────────────┬───────────────────────────┘
44
- │ calls
45
-┌──────────────────────▼───────────────────────────┐
46
-│ Flutter Native Module (platform channel) │
47
-│ - ONNX Runtime inference │
48
-│ - Phonemizer (grapheme→IPA conversion) │
49
-│ - Audio playback │
50
-└──────────────────────────────────────────────────┘
51
-```
52
-
53
-### Key Design Decisions
54
-
55
-1. **No cloud APIs** — all TTS runs on-device using neural ONNX models
56
-2. **Single active engine** — only one neural engine loaded at a time (saves RAM)
57
-3. **CPU-only execution** — `executionProviders: 'cpu'` for battery/thermal stability
58
-4. **4 GiB RAM minimum gate** — TTS disabled on devices with <4GB total RAM
59
-5. **Phonemization required** for Kokoro and Kitten engines (grapheme→IPA)
60
-
61
----
62
-
63
-## TTS Engines
64
-
65
-### Engine Comparison Table
66
-
67
-| Feature | Kitten | Kokoro | Supertonic | System |
68
-|---------|--------|--------|------------|--------|
69
-| Type | StyleTTS2 nano | Kokoro 82M v1.0 | Supertonic v2 | OS native |
70
-| Format | ONNX FP32 | ONNX FP16 | ONNX (5 files) | Platform API |
71
-| Disk | ~57 MB | ~170 MB | ~265 MB | 0 MB |
72
-| RAM (peak) | ~235 MB | ~510 MB | ~428 MB | Minimal |
73
-| Voices | 8 (4F/4M) | 22 (17F/5M) | 10 (5F/5M) | OS-dependent |
74
-| Languages | EN only | EN, EN-GB | EN, KO, ES, PT, FR | OS-dependent |
75
-| Phonemizer? | Yes (JS `phonemize`) | Yes (JS `phonemize`) | No (grapheme direct) | No |
76
-| Install | One-phase (all-or-nothing) | Two-phase (core + voices) | Two-phase (core + voices) | Always available |
77
-| Voice embeddings | Built-in NPZ | Per-voice `.bin` files | Per-voice `.json` style files | OS built-in |
78
-
79
-### Model Sources (HuggingFace)
80
-
81
-- **IPA dictionary (shared):** `https://huggingface.co/datasets/palshub/phonemizer-dicts/resolve/main/en-us.bin` → saved as `en-us.bin`
82
-- **Kitten:** `https://huggingface.co/palshub/kitten-tts-nano-0.8-fp32`
83
- - Files: `kitten_tts_nano_v0_8.onnx` → `kitten.onnx`, `voices-manifest.json`
84
-- **Kokoro:** `https://huggingface.co/onnx-community/Kokoro-82M-v1.0-ONNX`
85
- - Core: `onnx/model_fp16.onnx` → `model.onnx`, `tokenizer.json`
86
- - Voices: per-voice `.bin` files under `voices/` directory
87
-- **Supertonic:** `https://huggingface.co/Supertone/supertonic-2`
88
- - 5 ONNX files: `duration_predictor.onnx`, `text_encoder.onnx`, `vector_estimator.onnx`, `vocoder.onnx`, `unicode_indexer.json`
89
- - Voices: per-voice `.json` files under `voice_styles/`
90
-
91
----
92
-
93
-## Phonemizer
94
-
95
-### Why It's Needed
96
-
97
-Kokoro and Kitten are **phoneme-based** TTS models: they accept IPA phonemes as input, not raw text. The text must be converted to phonemes before synthesis.
98
-
99
-### Implementation
100
-
101
-- **Library:** MIT `phonemize` JS library (v1.1.0)
102
-- **Input:** Grapheme text (e.g., "Hello world")
103
-- **Output:** IPA phoneme sequence (e.g., "həlˈoʊ wˈɜːld")
104
-- **Dictionary:** Pre-built `en-us.bin` IPA dictionary (~XX MB), downloaded from HuggingFace
105
-- **How it flows:**
106
- 1. User text is received by the engine
107
- 2. Text is chunked (max 200 chars per batch)
108
- 3. Each chunk is passed through `phonemize(text_chunk)`
109
- 4. Phoneme sequence is fed to the ONNX model for synthesis
110
- 5. ONNX output → audio waveform → playback
111
-
112
-### Flutter Port
113
-
114
-In the React Native version, phonemization happens inside the native `@pocketpalai/react-native-speech` module. For Flutter, you have **two options:**
115
-
116
-**Option A: Dart phonemizer (recommended)**
117
-- Port the MIT `phonemize` JS library to Dart
118
-- Bundle the `en-us.bin` IPA dictionary (download at runtime from HuggingFace)
119
-- Run phonemization in Dart before passing phonemes to native
120
-
121
-**Option B: Native phonemizer**
122
-- Implement phonemization in the platform channel (Kotlin/Swift)
123
-- Use the `phonemize` JS library via a JS engine (requires embedding QuickJS/JSC)
124
-- Or use another native phonemizer (e.g., `espeak-ng` bindings — but this adds ~15 MB)
125
-
126
-**Recommendation:** Option A — pure Dart phonemizer keeps the port clean and avoids native complexity.
127
-
128
----
129
-
130
-## Directory Structure (App Documents)
131
-
132
-```
133
-<app_documents>/
134
-└── tts/
135
- ├── en-us.bin # IPA dict (shared)
136
- ├── kitten/
137
- │ ├── kitten.onnx
138
- │ ├── voices-manifest.json
139
- │ └── en-us.bin
140
- ├── kokoro/
141
- │ ├── model.onnx
142
- │ ├── tokenizer.json
143
- │ ├── voices-manifest.json
144
- │ ├── en-us.bin
145
- │ └── voices/
146
- │ ├── af_heart.bin
147
- │ ├── af_bella.bin
148
- │ └── ...
149
- └── supertonic/
150
- ├── duration_predictor.onnx
151
- ├── text_encoder.onnx
152
- ├── vector_estimator.onnx
153
- ├── vocoder.onnx
154
- ├── unicode_indexer.json
155
- ├── voices-manifest.json
156
- ├── F1.json
157
- ├── F2.json
158
- └── ...
159
-```
160
-
161
-**iOS:** Root path is `Library/Application Support/tts/` (excluded from iCloud backup via `NSURLIsExcludedFromBackupKey`)
162
-**Android:** Root path is `files/tts/` (excluded from backup via backup rules XML)
163
-
164
----
165
-
166
-## Voice Catalogs
167
-
168
-### Voice Type Definition
169
-
170
-```dart
171
-enum EngineId { kitten, kokoro, supertonic, system }
172
-
173
-class Voice {
174
- final String id;
175
- final String name;
176
- final EngineId engine;
177
- final String? language;
178
- final String? gender; // 'f' or 'm'
179
-}
180
-```
181
-
182
-### Kitten (8 voices)
183
-
184
-| ID | Name | Gender |
185
-|----|------|--------|
186
-| `expr-voice-2-f` | Bella | f |
187
-| `expr-voice-3-f` | Luna | f |
188
-| `expr-voice-4-f` | Rosie | f |
189
-| `expr-voice-5-f` | Kiki | f |
190
-| `expr-voice-2-m` | Jasper | m |
191
-| `expr-voice-3-m` | Bruno | m |
192
-| `expr-voice-4-m` | Hugo | m |
193
-| `expr-voice-5-m` | Leo | m |
194
-
195
-### Kokoro (22 voices)
196
-
197
-American English female: `af_heart` (Heart), `af_bella` (Bella), `af_nicole` (Nicole), `af_sarah` (Sarah), `af_sky` (Sky), `af_aoede` (Aoede), `af_jessica` (Jessica), `af_kore` (Kore), `af_river` (River)
198
-American English male: `am_adam` (Adam), `am_echo` (Echo), `am_eric` (Eric), `am_fenrir` (Fenrir), `am_liam` (Liam), `am_michael` (Michael), `am_onyx` (Onyx), `am_santa` (Santa)
199
-British English female: `bf_alice` (Alice), `bf_emma` (Emma), `bf_lily` (Lily)
200
-British English male: `bm_george` (George), `bm_lewis` (Lewis)
201
-
202
-### Supertonic (10 voices)
203
-
204
-Female: `F1` (Sarah), `F2` (Lily), `F3` (Jessica), `F4` (Olivia), `F5` (Emily)
205
-Male: `M1` (Alex), `M2` (James), `M3` (Robert), `M4` (Sam), `M5` (Daniel)
206
-
207
----
208
-
209
-## Core Interfaces to Implement
210
-
211
-### Engine Interface
212
-
213
-```dart
214
-abstract class Engine {
215
- EngineId get id;
216
- Future<bool> isInstalled();
217
- Future<List<Voice>> getVoices();
218
- Future<void> loadInto(); // Initialize native engine
219
- Future<void> play(String text, Voice voice);
220
- StreamingHandle playStreaming(Voice voice, [Future<void>? waitFor]);
221
- Future<void> stop();
222
-}
223
-```
224
-
225
-### StreamingHandle
226
-
227
-```dart
228
-abstract class StreamingHandle {
229
- void appendText(String chunk);
230
- Future<void> finalize();
231
- Future<void> cancel();
232
-}
233
-```
234
-
235
-### TTSRuntime (Singleton)
236
-
237
-- Tracks which engine is currently loaded in native land
238
-- Serializes all engine swaps via a promise chain (FIFO mutex)
239
-- `acquire(engine, fn)` — releases previous engine, loads new one, runs fn
240
-- `release()` — frees native resources
241
-- `stop()` — stops playback AND serializes with subsequent acquires
242
-
243
-### TTSStore (State)
244
-
245
-- **Persisted preferences:** `autoSpeakEnabled`, `currentVoice`, `supertonicSteps`
246
-- **Non-persisted state:** per-engine `NeuralDownloadState` (not_installed / downloading / ready / error), download progress, playback state
247
-- **Streaming hooks:** `onAssistantMessageStart`, `onAssistantMessageChunk`, `onAssistantMessageComplete`
248
-- **Replay path:** `play(messageId, text)`
249
-- **Preview path:** `preview(voice)`
250
-- **Download actions:** `downloadSupertonic()`, `downloadKokoro()`, `downloadKitten()`, plus retry/delete variants
251
-
252
----
253
-
254
-## Download Lifecycle
255
-
256
-### Kitten (one-phase, all-or-nothing)
257
-
258
-```
259
-1. mkdir tts/kitten/ (with iOS backup exclusion on parent)
260
-2. Download kitten.onnx (progress tracked)
261
-3. Download voices-manifest.json (progress tracked)
262
-4. Download en-us.bin IPA dict (progress tracked)
263
-5. If ANY step fails: delete entire tts/kitten/ directory
264
-6. On success: set download state to 'ready'
265
-```
266
-
267
-### Kokoro (two-phase)
268
-
269
-```
270
-Phase 1 (all-or-nothing, 60% of progress weight):
271
- 1. mkdir tts/kokoro/voices/
272
- 2. Download model.onnx
273
- 3. Download tokenizer.json
274
- 4. Download en-us.bin
275
- 5. If ANY fails: delete entire tts/kokoro/
276
-
277
-Phase 2 (best-effort, 40% of progress weight):
278
- 6. Download each voice's .bin embedding file
279
- 7. Partial success OK (missing voice files fetched lazily later)
280
- 8. Write voices-manifest.json pointing at voices/ directory
281
-```
282
-
283
-### Supertonic (two-phase)
284
-
285
-```
286
-Phase 1 (all-or-nothing, 70% of progress weight):
287
- 1. mkdir tts/supertonic/
288
- 2. Download duration_predictor.onnx
289
- 3. Download text_encoder.onnx
290
- 4. Download vector_estimator.onnx
291
- 5. Download vocoder.onnx
292
- 6. Download unicode_indexer.json
293
- 7. If ANY fails: delete entire tts/supertonic/
294
-
295
-Phase 2 (best-effort, 30% of progress weight):
296
- 8. Download each voice's .json style file
297
- 9. Write voices-manifest.json with baseUrl for lazy fallback
298
-```
299
-
300
----
301
-
302
-## Playback Paths
303
-
304
-### Streaming Path (during LLM response)
305
-
306
-```
307
-1. onAssistantMessageStart(messageId)
308
- - Stop any prior playback
309
- - Create ThinkingStripper instance
310
- - Get engine for currentVoice
311
- - acquire(engine): release old engine → loadInto() new engine
312
- - Create SpeechStream (native)
313
- - Return StreamingHandle
314
-
315
-2. onAssistantMessageChunk(chunk, reasoningDelta?)
316
- - Feed chunk to ThinkingStripper (strips <think>...</think>)
317
- - If thinking detected → prepend "Hmm, let me think." placeholder
318
- - Append cleaned text to StreamingHandle
319
- - (Native phonemizes and synthesizes incrementally)
320
-
321
-3. onAssistantMessageComplete(text)
322
- - Flush ThinkingStripper leftovers
323
- - StreamingHandle.finalize()
324
-```
325
-
326
-### Replay Path (play button on past message)
327
-
328
-```
329
-1. stop() any in-flight playback
330
-2. Strip <think> tags via ThinkingStripper.stripFinal(text)
331
-3. Prepend thinking placeholder if reasoning occurred
332
-4. play() → acquire(engine) → Speech.speak(phonemized_text, voiceId)
333
-5. On completion → set playbackState to idle
334
-```
335
-
336
----
337
-
338
-## Thinking/Reasoning Stripper
339
-
340
-### Purpose
341
-
342
-LLM responses may contain `<think>...</think>` tags (either from the model or from a separate `reasoning_content` stream). These should be stripped before TTS to avoid speaking internal reasoning aloud.
343
-
344
-### Implementation
345
-
346
-```dart
347
-class ThinkingStripper {
348
- String _buffer = '';
349
- bool _inside = false;
350
- bool _nonEmptyThink = false;
351
-
352
- // Feed a chunk, return clean text to speak
353
- String feed(String chunk);
354
-
355
- // Flush remaining buffer at stream end
356
- String flush();
357
-
358
- // Whether non-empty think content was encountered
359
- bool hadNonEmptyThink();
360
-
361
- // Record out-of-band reasoning delta
362
- void noteReasoning(String delta);
363
-
364
- // Static helper for non-streaming replay
365
- static StripResult stripFinal(String text, {bool? hadReasoning});
366
-}
367
-```
368
-
369
-**Placeholder phrases** (randomly selected): "Hmm, let me think.", "Hmm.", "Let me think.", "Okay, let me think about that.", "One moment.", "Let me consider this."
370
-
371
----
372
-
373
-## Voice Selection Flow (UI)
374
-
375
-1. User opens `TTSSetupSheet` bottom sheet
376
-2. Sees engine cards in order: Kitten → Kokoro → Supertonic
377
-3. Each card shows: engine logo, name, tagline, disk size, RAM, voice count, accent color
378
-4. Uninstalled engine: shows "Install · XX MB" button with progress bar during download
379
-5. Installed engine: expandable to show voice radio buttons grouped by gender
380
-6. Tapping a voice: calls `ttsStore.currentVoice = voice` (persisted)
381
-7. Preview button plays `TTS_PREVIEW_SAMPLE` text through selected voice
382
-8. Quality selector (Supertonic only): diffusion steps `1/2/3/5/10/20`
383
-
384
-### Engine Metadata for UI
385
-
386
-```dart
387
-class EngineMeta {
388
- final int sizeMb;
389
- final int ramMb;
390
- final int voices;
391
- final String accent;
392
- final String gradientFrom;
393
- final String gradientTo;
394
-}
395
-```
396
-
397
-| Engine | Size | RAM | Voices | Accent |
398
-|--------|------|-----|--------|--------|
399
-| Kitten | 57 MB | 235 MB | 8 | #F29547 |
400
-| Kokoro | 170 MB | 510 MB | 22 | #6F5CD6 |
401
-| Supertonic | 265 MB | 428 MB | 10 | #1E4DF6 |
402
-
403
----
404
-
405
-## Native Module Platform Channel API
406
-
407
-The Flutter app needs a native platform channel (MethodChannel) with these methods:
408
-
409
-### Initialization
410
-
411
-```dart
412
-// Kitten
413
-await platform.invokeMethod('tts.initialize', {
414
- 'engine': 'kitten',
415
- 'modelPath': '/path/to/kitten.onnx',
416
- 'voicesPath': '/path/to/voices-manifest.json',
417
- 'dictPath': '/path/to/en-us.bin',
418
- 'executionProviders': 'cpu',
419
- 'maxChunkSize': 200,
420
- 'silentMode': 'obey',
421
- 'ducking': true,
422
-});
423
-
424
-// Kokoro
425
-await platform.invokeMethod('tts.initialize', {
426
- 'engine': 'kokoro',
427
- 'modelPath': '/path/to/model.onnx',
428
- 'tokenizerPath': '/path/to/tokenizer.json',
429
- 'voicesPath': '/path/to/voices-manifest.json',
430
- 'dictPath': '/path/to/en-us.bin',
431
- 'executionProviders': 'cpu',
432
- 'maxChunkSize': 200,
433
- 'silentMode': 'obey',
434
- 'ducking': true,
435
-});
436
-
437
-// Supertonic
438
-await platform.invokeMethod('tts.initialize', {
439
- 'engine': 'supertonic',
440
- 'durationPredictorPath': '/path/to/duration_predictor.onnx',
441
- 'textEncoderPath': '/path/to/text_encoder.onnx',
442
- 'vectorEstimatorPath': '/path/to/vector_estimator.onnx',
443
- 'vocoderPath': '/path/to/vocoder.onnx',
444
- 'unicodeIndexerPath': '/path/to/unicode_indexer.json',
445
- 'voicesPath': '/path/to/voices-manifest.json',
446
- 'executionProviders': 'cpu',
447
- 'maxChunkSize': 200,
448
- 'silentMode': 'obey',
449
- 'ducking': true,
450
-});
451
-
452
-// System (OS native)
453
-await platform.invokeMethod('tts.initialize', {
454
- 'engine': 'os_native',
455
-});
456
-```
457
-
458
-### Playback
459
-
460
-```dart
461
-// Non-streaming speak
462
-await platform.invokeMethod('tts.speak', {
463
- 'text': 'Hello world', // Pre-phonemized text for Kitten/Kokoro
464
- 'voiceId': 'af_bella',
465
- 'language': 'en', // Supertonic only
466
- 'inferenceSteps': 5, // Supertonic only
467
-});
468
-
469
-// Create speech stream
470
-final stream = EventChannel('tts.stream.$voiceId');
471
-// Send chunks via MethodChannel
472
-await platform.invokeMethod('tts.stream.append', {'text': 'chunk'});
473
-await platform.invokeMethod('tts.stream.finalize');
474
-await platform.invokeMethod('tts.stream.cancel');
475
-
476
-// Control
477
-await platform.invokeMethod('tts.stop');
478
-await platform.invokeMethod('tts.release');
479
-
480
-// System voices
481
-final voices = await platform.invokeMethod('tts.getAvailableVoices');
482
-```
483
-
484
-### OS-Specific Notes
485
-
486
-**iOS:**
487
-- Use `AVSpeechSynthesizer` for System engine
488
-- Neural engines: ONNX Runtime with CoreML execution provider (optional, CPU is safer)
489
-- Model storage: `NSLibraryDirectory/Application Support/tts/` with backup exclusion
490
-- Handle audio session interruptions
491
-
492
-**Android:**
493
-- Use `TextToSpeech` API for System engine
494
-- Neural engines: ONNX Runtime with NNAPI (optional, CPU is safer)
495
-- Model storage: `context.filesDir/tts/` with backup rules in `AndroidManifest.xml`
496
-- Handle `TextToSpeech.OnInitListener` for System engine
497
-
498
----
499
-
500
-## Implementation Order
501
-
502
-### Phase 1: Foundation
503
-1. Create Dart `Engine` interface and `Voice` types
504
-2. Implement `TTSRuntime` singleton with serialized acquire/release
505
-3. Create `ThinkingStripper` in Dart
506
-4. Implement platform channel for System engine (OS native TTS)
507
-5. Build basic `TTSStore` with `play()` replay path
508
-6. Verify System engine works on both platforms
509
-
510
-### Phase 2: Neural Engine Support
511
-7. Implement file download infrastructure (resumable, progress tracking)
512
-8. Build native module boilerplate for ONNX Runtime (Kotlin/Swift)
513
-9. Implement Kitten engine (simplest — single phase, 8 voices)
514
-10. Implement Kokoro engine (two-phase, 22 voices, phonemizer needed)
515
-11. Implement Supertonic engine (5 ONNX files, multilingual)
516
-12. Implement per-engine download state machines in `TTSStore`
517
-
518
-### Phase 3: Streaming
519
-13. Implement `StreamingHandle` in Dart
520
-14. Implement `SpeechStream` in native (Kotlin/Swift)
521
-15. Wire streaming path through `TTSStore` (`onAssistantMessageStart/Chunk/Complete`)
522
-16. Integrate with LLM chat session
523
-
524
-### Phase 4: UI
525
-17. Build `TTSSetupSheet` bottom sheet widget
526
-18. Build `EngineCard` widget with install/delete controls
527
-19. Build `VoicePickerView` with gender-grouped radio buttons
528
-20. Build `PlayButton` for per-message replay
529
-21. Build `VoiceChip` header pill
530
-22. Build `AutoSpeakRow` toggle
531
-23. Build `HeroRow` quality selector (Supertonic steps)
532
-
533
-### Phase 5: Polish
534
-24. Implement RAM gate check (`DeviceInfo.getTotalMemory()` → min 4 GiB)
535
-25. iOS backup exclusion on `tts/` directory
536
-26. Handle app backgrounding (release engine resources)
537
-27. Handle audio interruptions (phone calls, etc.)
538
-28. Disk space check before download
539
-29. Handle model deletion cleanup
540
-30. Error states and retry flows
541
-
542
----
543
-
544
-## Constants Reference
545
-
546
-```dart
547
-// Preview sample
548
-const ttsPreviewSample = "Oh, hello there! I've been waiting for you to test me. I sound pretty good!";
549
-
550
-// RAM gate
551
-const ttsMinRamBytes = 4 * 1024 * 1024 * 1024; // 4 GiB
552
-
553
-// Directories
554
-const ttsParentSubdir = 'tts';
555
-const kittenModelSubdir = 'tts/kitten';
556
-const kokoroModelSubdir = 'tts/kokoro';
557
-const supertonicModelSubdir = 'tts/supertonic';
558
-
559
-// Model URLs
560
-const ttsDictUrl = 'https://huggingface.co/datasets/palshub/phonemizer-dicts/resolve/main/en-us.bin';
561
-const kittenModelBaseUrl = 'https://huggingface.co/palshub/kitten-tts-nano-0.8-fp32/resolve/main';
562
-const kokoroModelBaseUrl = 'https://huggingface.co/onnx-community/Kokoro-82M-v1.0-ONNX/resolve/main';
563
-const supertonicModelBaseUrl = 'https://huggingface.co/Supertone/supertonic-2/resolve/main';
564
-
565
-// Engine config
566
-const maxChunkSize = 200;
567
-const streamTargetChars = 300;
568
-const supertonicStepsDefault = 5;
569
-```
570
-
571
----
572
-
573
-## Testing Strategy
574
-
575
-- Unit test `ThinkingStripper` with various `<think>` tag patterns
576
-- Unit test `TTSRuntime` acquire/release/stop ordering
577
-- Unit test `TTSStore` state machine transitions
578
-- Widget test engine cards in various states (not_installed, downloading, ready, error)
579
-- Integration test download → initialize → play flow for each engine
580
-- Integration test streaming path with mock LLM token feed
581
-- Test edge cases: switching engines mid-playback, rapid play/stop cycles, app backgrounding mid-download
582
-- Test on low-RAM devices (verify gate disables TTS)
583
-- Test on iOS and Android with all 4 engines
584
-
585
----
586
-
587
-## Key Source Files (Reference)
588
-
589
-From the original React Native codebase (`src/services/tts/`):
590
-
591
-| File | Purpose |
592
-|------|---------|
593
-| `types.ts` | Core types: `Engine`, `Voice`, `StreamingHandle`, `EngineId` |
594
-| `constants.ts` | Model URLs, file names, sizes, config constants |
595
-| `runtime.ts` | `TTSRuntime` singleton — serialized engine swap |
596
-| `engineRegistry.ts` | Singleton instances of all 4 engines |
597
-| `streamingHandle.ts` | `createEngineStreamingHandle()` — bridges streaming with runtime |
598
-| `thinkingStripper.ts` | `ThinkingStripper` class for `<think>` tag stripping |
599
-| `engines/kitten/index.ts` | Kitten engine implementation |
600
-| `engines/kitten/voices.ts` | Kitten voice catalog |
601
-| `engines/kokoro/index.ts` | Kokoro engine implementation |
602
-| `engines/kokoro/voices.ts` | Kokoro voice catalog |
603
-| `engines/supertonic/index.ts` | Supertonic engine implementation |
604
-| `engines/supertonic/voices.ts` | Supertonic voice catalog |
605
-| `engines/system/index.ts` | System engine implementation |
606
-| `engines/system/voices.ts` | System voice fetcher |
607
-| `store/TTSStore.ts` | Main TTS store with all orchestration |
608
-| `components/TTSSetupSheet/engineMeta.ts` | Per-engine UI metadata |
pubspec.lock
+8
-8
@@ -769,10 +769,10 @@ packages:
769
dependency: transitive
770
description:
771
name: matcher
772
- sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6"
772
+ sha256: dc0b7dc7651697ea4ff3e69ef44b0407ea32c487a39fff6a4004fa585e901861
773
url: "https://pub.dev"
774
source: hosted
775
- version: "0.12.18"
775
+ version: "0.12.19"
776
material_color_utilities:
777
dependency: transitive
778
description:
@@ -1247,26 +1247,26 @@ packages:
1247
dependency: transitive
1248
description:
1249
name: test
1250
- sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a"
1250
+ sha256: "280d6d890011ca966ad08df7e8a4ddfab0fb3aa49f96ed6de56e3521347a9ae7"
1251
url: "https://pub.dev"
1252
source: hosted
1253
- version: "1.29.0"
1253
+ version: "1.30.0"
1254
test_api:
1255
dependency: transitive
1256
description:
1257
name: test_api
1258
- sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636"
1258
+ sha256: "8161c84903fd860b26bfdefb7963b3f0b68fee7adea0f59ef805ecca346f0c7a"
1259
url: "https://pub.dev"
1260
source: hosted
1261
- version: "0.7.9"
1261
+ version: "0.7.10"
1262
test_core:
1263
dependency: transitive
1264
description:
1265
name: test_core
1266
- sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943"
1266
+ sha256: "0381bd1585d1a924763c308100f2138205252fb90c9d4eeaf28489ee65ccde51"
1267
url: "https://pub.dev"
1268
source: hosted
1269
- version: "0.6.15"
1269
+ version: "0.6.16"
1270
theme_extensions_builder_annotation:
1271
dependency: transitive
1272
description:
pubspec.yaml
+1
-1
@@ -16,7 +16,7 @@ publish_to: "none" # Remove this line if you wish to publish to pub.dev
16
# https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
17
# In Windows, build-name is used as the major, minor, and patch parts
18
# of the product and file versions while build-number is used as the build suffix.
19
-version: 1.2.1+20
19
+version: 1.2.3+21
20
21
environment:
22
sdk: ^3.11.0