| 1 | import 'dart:isolate'; |
| 2 | import 'dart:typed_data'; |
| 3 | |
| 4 | import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; |
| 5 | |
| 6 | import '../../../core/logger/app_logger.dart'; |
| 7 | import '../../../core/models/enums.dart'; |
| 8 | |
| 9 | sealed class TtsWorkerMessage {} |
| 10 | |
| 11 | class TtsInitRequest extends TtsWorkerMessage { |
| 12 | final EngineId engineId; |
| 13 | final String modelPath; |
| 14 | final String voicesPath; |
| 15 | final String tokensPath; |
| 16 | final String dataDir; |
| 17 | final String lexicon; |
| 18 | |
| 19 | TtsInitRequest({ |
| 20 | required this.engineId, |
| 21 | required this.modelPath, |
| 22 | required this.voicesPath, |
| 23 | required this.tokensPath, |
| 24 | required this.dataDir, |
| 25 | required this.lexicon, |
| 26 | }); |
| 27 | } |
| 28 | |
| 29 | class TtsSynthesizeRequest extends TtsWorkerMessage { |
| 30 | final String text; |
| 31 | final String voiceId; |
| 32 | final double speed; |
| 33 | final int chunkIndex; |
| 34 | final int sessionId; |
| 35 | |
| 36 | TtsSynthesizeRequest({ |
| 37 | required this.text, |
| 38 | required this.voiceId, |
| 39 | required this.speed, |
| 40 | required this.chunkIndex, |
| 41 | required this.sessionId, |
| 42 | }); |
| 43 | } |
| 44 | |
| 45 | class TtsStopRequest extends TtsWorkerMessage {} |
| 46 | |
| 47 | class TtsWorkerResponse { |
| 48 | final int? chunkIndex; |
| 49 | final int? sessionId; |
| 50 | final Uint8List? wavBytes; |
| 51 | final String? error; |
| 52 | final bool isInitResponse; |
| 53 | |
| 54 | TtsWorkerResponse({ |
| 55 | this.chunkIndex, |
| 56 | this.sessionId, |
| 57 | this.wavBytes, |
| 58 | this.error, |
| 59 | this.isInitResponse = false, |
| 60 | }); |
| 61 | } |
| 62 | |
| 63 | void ttsWorkerEntry(SendPort mainSendPort) { |
| 64 | final receivePort = ReceivePort(); |
| 65 | mainSendPort.send(receivePort.sendPort); |
| 66 | |
| 67 | sherpa_onnx.initBindings(); |
| 68 | |
| 69 | sherpa_onnx.OfflineTts? tts; |
| 70 | EngineId? activeEngineId; |
| 71 | TtsInitRequest? activeInit; |
| 72 | |
| 73 | receivePort.listen((message) async { |
| 74 | try { |
| 75 | if (message is TtsInitRequest) { |
| 76 | tts?.free(); |
| 77 | tts = null; |
| 78 | |
| 79 | activeEngineId = message.engineId; |
| 80 | activeInit = message; |
| 81 | |
| 82 | tts = _createTts(message); |
| 83 | Log.info('Worker initialized engine: ${message.engineId}'); |
| 84 | mainSendPort.send(TtsWorkerResponse(isInitResponse: true)); |
| 85 | } else if (message is TtsSynthesizeRequest) { |
| 86 | if (activeEngineId == null || tts == null || activeInit == null) { |
| 87 | mainSendPort.send( |
| 88 | TtsWorkerResponse( |
| 89 | chunkIndex: message.chunkIndex, |
| 90 | sessionId: message.sessionId, |
| 91 | error: 'No engine initialized in worker', |
| 92 | ), |
| 93 | ); |
| 94 | return; |
| 95 | } |
| 96 | |
| 97 | final speakerId = _resolveSpeakerId(activeEngineId!, message.voiceId); |
| 98 | final genConfig = sherpa_onnx.OfflineTtsGenerationConfig( |
| 99 | sid: speakerId, |
| 100 | speed: message.speed, |
| 101 | silenceScale: 0.2, |
| 102 | ); |
| 103 | |
| 104 | final audio = tts!.generateWithConfig( |
| 105 | text: message.text, |
| 106 | config: genConfig, |
| 107 | ); |
| 108 | if (audio.samples.isEmpty) { |
| 109 | mainSendPort.send( |
| 110 | TtsWorkerResponse( |
| 111 | chunkIndex: message.chunkIndex, |
| 112 | sessionId: message.sessionId, |
| 113 | error: 'Empty PCM generated', |
| 114 | ), |
| 115 | ); |
| 116 | return; |
| 117 | } |
| 118 | |
| 119 | final wavBytes = _createWav(audio.samples, audio.sampleRate); |
| 120 | mainSendPort.send( |
| 121 | TtsWorkerResponse( |
| 122 | chunkIndex: message.chunkIndex, |
| 123 | sessionId: message.sessionId, |
| 124 | wavBytes: wavBytes, |
| 125 | ), |
| 126 | ); |
| 127 | } else if (message is TtsStopRequest) { |
| 128 | // No-op: the main isolate handles queue cancellation. |
| 129 | } |
| 130 | } catch (e) { |
| 131 | mainSendPort.send(TtsWorkerResponse(error: e.toString())); |
| 132 | } |
| 133 | }); |
| 134 | } |
| 135 | |
| 136 | sherpa_onnx.OfflineTts _createTts(TtsInitRequest message) { |
| 137 | final vits = message.engineId == EngineId.piper |
| 138 | ? sherpa_onnx.OfflineTtsVitsModelConfig( |
| 139 | model: message.modelPath, |
| 140 | tokens: message.tokensPath, |
| 141 | dataDir: message.dataDir, |
| 142 | lexicon: message.lexicon, |
| 143 | ) |
| 144 | : sherpa_onnx.OfflineTtsVitsModelConfig(); |
| 145 | |
| 146 | final kitten = message.engineId == EngineId.kitten |
| 147 | ? sherpa_onnx.OfflineTtsKittenModelConfig( |
| 148 | model: message.modelPath, |
| 149 | voices: message.voicesPath, |
| 150 | tokens: message.tokensPath, |
| 151 | dataDir: message.dataDir, |
| 152 | ) |
| 153 | : sherpa_onnx.OfflineTtsKittenModelConfig(); |
| 154 | |
| 155 | final modelConfig = sherpa_onnx.OfflineTtsModelConfig( |
| 156 | vits: vits, |
| 157 | kitten: kitten, |
| 158 | numThreads: 2, |
| 159 | debug: true, |
| 160 | provider: 'cpu', |
| 161 | ); |
| 162 | |
| 163 | final config = sherpa_onnx.OfflineTtsConfig( |
| 164 | model: modelConfig, |
| 165 | maxNumSenetences: 1, |
| 166 | ); |
| 167 | |
| 168 | return sherpa_onnx.OfflineTts(config); |
| 169 | } |
| 170 | |
| 171 | int _resolveSpeakerId(EngineId engine, String voiceId) { |
| 172 | if (engine == EngineId.piper) { |
| 173 | return 0; |
| 174 | } |
| 175 | final voices = voicesForEngine(engine); |
| 176 | final index = voices.indexWhere((voice) => voice.id == voiceId); |
| 177 | return index < 0 ? 0 : index; |
| 178 | } |
| 179 | |
| 180 | Uint8List _createWav(Float32List samples, int sampleRate) { |
| 181 | final int byteCount = samples.length * 2; |
| 182 | final int totalSize = 36 + byteCount; |
| 183 | final Uint8List wav = Uint8List(44 + byteCount); |
| 184 | final ByteData data = ByteData.view(wav.buffer); |
| 185 | |
| 186 | data.setUint8(0, 0x52); |
| 187 | data.setUint8(1, 0x49); |
| 188 | data.setUint8(2, 0x46); |
| 189 | data.setUint8(3, 0x46); |
| 190 | data.setUint32(4, totalSize, Endian.little); |
| 191 | data.setUint8(8, 0x57); |
| 192 | data.setUint8(9, 0x41); |
| 193 | data.setUint8(10, 0x56); |
| 194 | data.setUint8(11, 0x45); |
| 195 | data.setUint8(12, 0x66); |
| 196 | data.setUint8(13, 0x6d); |
| 197 | data.setUint8(14, 0x74); |
| 198 | data.setUint8(15, 0x20); |
| 199 | data.setUint32(16, 16, Endian.little); |
| 200 | data.setUint16(20, 1, Endian.little); |
| 201 | data.setUint16(22, 1, Endian.little); |
| 202 | data.setUint32(24, sampleRate, Endian.little); |
| 203 | data.setUint32(28, sampleRate * 2, Endian.little); |
| 204 | data.setUint16(32, 2, Endian.little); |
| 205 | data.setUint16(34, 16, Endian.little); |
| 206 | data.setUint8(36, 0x64); |
| 207 | data.setUint8(37, 0x61); |
| 208 | data.setUint8(38, 0x74); |
| 209 | data.setUint8(39, 0x61); |
| 210 | data.setUint32(40, byteCount, Endian.little); |
| 211 | |
| 212 | for (var i = 0; i < samples.length; i++) { |
| 213 | final int sample = (samples[i] * 32767).clamp(-32768, 32767).toInt(); |
| 214 | data.setInt16(44 + (i * 2), sample, Endian.little); |
| 215 | } |
| 216 | |
| 217 | return wav; |
| 218 | } |