@tduncan / Cortana
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
dart 121 lines 3.4 KB
Raw
1 import 'dart:async';
2
3 import 'package:localmind/features/servers/data/models/server.dart';
4
5 import '../data/chat_service.dart';
6 import '../data/models/message.dart' show Message;
7
8 /// Snapshot of a reply that is generating, exposed to the UI. Only the
9 /// conversation and whether it occupies the (single) on-device engine matter
10 /// to views.
11 class ActiveGeneration {
12 const ActiveGeneration({
13 required this.conversationId,
14 required this.isOnDevice,
15 });
16
17 final String conversationId;
18 final bool isOnDevice;
19 }
20
21 /// Everything that belongs to one in-flight reply. Remote replies (LM Studio,
22 /// Ollama, OpenRouter, ...) can run one session per conversation at the same
23 /// time; on-device replies share a single engine, so at most one on-device
24 /// session exists at a time.
25 class GenerationSession {
26 GenerationSession({
27 required this.conversationId,
28 required this.server,
29 required this.chatService,
30 required this.persisted,
31 });
32
33 static const _checkpointChunkThreshold = 20;
34 static const _checkpointTimeThreshold = Duration(seconds: 2);
35
36 final String conversationId;
37 final Server server;
38
39 /// Service that owns this stream. Cancelling must target it, never a
40 /// service shared with another reply.
41 final ChatService chatService;
42
43 /// False for temporary chats, whose messages are never written to disk.
44 final bool persisted;
45
46 bool get isOnDevice => server.isOnDevice;
47
48 StreamSubscription<ChatResponse>? subscription;
49 Timer? uiUpdateTimer;
50
51 /// Most recent snapshot of the streaming assistant message. A chat that is
52 /// reopened while this reply still runs reattaches to it.
53 Message? latestMessage;
54
55 /// Set once the reply is cancelled or replaced; late events are ignored.
56 bool cancelled = false;
57
58 int chunkCount = 0;
59 DateTime? lastCheckpointTime;
60 int lastSavedContentLength = 0;
61 int lastSavedReasoningLength = 0;
62
63 ChatStats? stats;
64 DateTime? startTime;
65 DateTime? firstTokenTime;
66
67 void resetStreamMetrics() {
68 stats = null;
69 startTime = DateTime.now();
70 firstTokenTime = null;
71 }
72
73 void noteFirstToken() {
74 firstTokenTime ??= DateTime.now();
75 }
76
77 bool shouldCheckpointSave(Message message) {
78 final now = DateTime.now();
79 final contentLength = message.content.length;
80 final reasoningLength = message.reasoningContent?.length ?? 0;
81
82 final hasNewContent =
83 contentLength > lastSavedContentLength ||
84 reasoningLength > lastSavedReasoningLength;
85 if (!hasNewContent) return false;
86
87 final meetsChunkThreshold = chunkCount >= _checkpointChunkThreshold;
88 final meetsTimeThreshold =
89 lastCheckpointTime != null &&
90 now.difference(lastCheckpointTime!) >= _checkpointTimeThreshold;
91
92 return meetsChunkThreshold || meetsTimeThreshold;
93 }
94
95 void resetCheckpointMetrics() {
96 chunkCount = 0;
97 lastCheckpointTime = DateTime.now();
98 }
99
100 void updateSavedMetrics(Message message) {
101 lastSavedContentLength = message.content.length;
102 lastSavedReasoningLength = message.reasoningContent?.length ?? 0;
103 }
104
105 void cancelUiTimer() {
106 uiUpdateTimer?.cancel();
107 uiUpdateTimer = null;
108 }
109
110 /// Stops delivering events for this reply and cancels the UI timer and the
111 /// stream subscription. Does not cancel the underlying request.
112 Future<void> detach() async {
113 cancelled = true;
114 cancelUiTimer();
115 final sub = subscription;
116 subscription = null;
117 if (sub != null) {
118 await sub.cancel();
119 }
120 }
121 }