fix: improve OpenAI Compatible mode for llama.cpp and other servers
- Fall back to 'name' or 'id' when 'key' is missing in model list response, supporting llama.cpp and other OpenAI-compatible servers - Enrich model metadata from OpenAI-style 'data' array when LM Studio fields are absent (context length, file size, parameter count) - Make model load/unload endpoints non-fatal for OpenAI Compatible since many servers don't implement /v1/models/load or /v1/models/unload - Treat models as running when loaded_instances is absent, since servers like llama.cpp always have their model loaded - Strip trailing empty assistant messages before sending to avoid "prefill incompatible with enable_thinking" errors from servers running thinking models (e.g. Qwen3, DeepSeek-R1)
Nils Jannasch committed
May 22, 2026 at 21:36 UTC
711491853e151977e8028b02595c4b2117318e54
2 files changed
+80
-19
lib/features/chat/data/chat_service.dart
+13
-3
@@ -450,11 +450,21 @@ class OpenAICompatibleChatService implements ChatService {
450
}) async* {
451
_cancelToken = CancelToken();
452
453
+ final apiMessages = messages
454
+ .map((m) => {'role': _roleToString(m.role), 'content': m.content})
455
+ .toList();
456
+ // Strip trailing empty assistant messages to avoid "prefill incompatible
457
+ // with enable_thinking" errors from servers running thinking models
458
+ while (apiMessages.isNotEmpty &&
459
+ apiMessages.last['role'] == 'assistant' &&
460
+ (apiMessages.last['content'] == null ||
461
+ apiMessages.last['content']!.isEmpty)) {
462
+ apiMessages.removeLast();
463
+ }
464
+
465
final body = {
466
'model': modelId,
455
- 'messages': messages
456
- .map((m) => {'role': _roleToString(m.role), 'content': m.content})
457
- .toList(),
467
+ 'messages': apiMessages,
468
'temperature': params.temperature,
469
'top_p': params.topP,
470
'max_tokens': params.maxTokens,
lib/features/servers/data/server_api_service.dart
+67
-16
@@ -97,6 +97,15 @@ class ServerApiService {
97
switch (server.type) {
98
case ServerType.lmStudio:
99
case ServerType.openAICompatible:
100
+ try {
101
+ await _dio.post(
102
+ server.loadModelEndpoint,
103
+ data: {'model': modelId},
104
+ options: Options(headers: _getAuthHeaders(server)),
105
+ );
106
+ } catch (_) {
107
+ // Not all OpenAI-compatible servers support this endpoint
108
+ }
109
case ServerType.ollama:
110
await _dio.post(
111
server.loadModelEndpoint,
@@ -118,12 +127,16 @@ class ServerApiService {
127
switch (server.type) {
128
case ServerType.lmStudio:
129
case ServerType.openAICompatible:
121
- final response = await _dio.post(
122
- server.loadModelEndpoint,
123
- data: {'model': modelId, 'echo_load_config': true},
124
- options: Options(headers: _getAuthHeaders(server)),
125
- );
126
- return response.data['instance_id'] as String?;
130
+ try {
131
+ final response = await _dio.post(
132
+ server.loadModelEndpoint,
133
+ data: {'model': modelId, 'echo_load_config': true},
134
+ options: Options(headers: _getAuthHeaders(server)),
135
+ );
136
+ return response.data['instance_id'] as String?;
137
+ } catch (_) {
138
+ return null;
139
+ }
140
case ServerType.ollama:
141
await _dio.post(
142
server.loadModelEndpoint,
@@ -150,11 +163,15 @@ class ServerApiService {
163
switch (server.type) {
164
case ServerType.lmStudio:
165
case ServerType.openAICompatible:
153
- await _dio.post(
154
- server.unloadModelEndpoint,
155
- data: {'instance_id': instanceId ?? modelId},
156
- options: Options(headers: _getAuthHeaders(server)),
157
- );
166
+ try {
167
+ await _dio.post(
168
+ server.unloadModelEndpoint,
169
+ data: {'instance_id': instanceId ?? modelId},
170
+ options: Options(headers: _getAuthHeaders(server)),
171
+ );
172
+ } catch (_) {
173
+ // Not all OpenAI-compatible servers support this endpoint
174
+ }
175
case ServerType.ollama:
176
await _dio.post(
177
server.unloadModelEndpoint,
@@ -171,9 +188,17 @@ class ServerApiService {
188
final runningModels = <String>{};
189
if (data['models'] != null) {
190
for (final item in data['models']) {
191
+ final id = item['key'] as String? ??
192
+ item['name'] as String? ??
193
+ item['id'] as String? ??
194
+ '';
195
final loadedInstances = item['loaded_instances'] as List?;
196
if (loadedInstances != null && loadedInstances.isNotEmpty) {
176
- runningModels.add(item['key'] as String? ?? '');
197
+ runningModels.add(id);
198
+ } else if (id.isNotEmpty) {
199
+ // Servers without loaded_instances (e.g. llama.cpp) always have
200
+ // their model running if it appears in the model list
201
+ runningModels.add(id);
202
}
203
}
204
}
@@ -199,21 +224,41 @@ class ServerApiService {
224
225
List<ModelInfo> _parseLMStudioModels(dynamic data, Server server) {
226
final List<ModelInfo> models = [];
227
+
228
+ // Build a lookup from the OpenAI-style 'data' array for metadata
229
+ // that may be missing from the 'models' array (e.g. llama.cpp)
230
+ final Map<String, Map<String, dynamic>> metaById = {};
231
+ if (data['data'] != null) {
232
+ for (final item in data['data']) {
233
+ final id = item['id'] as String? ?? '';
234
+ if (id.isNotEmpty) {
235
+ metaById[id] = (item['meta'] as Map<String, dynamic>?) ?? {};
236
+ }
237
+ }
238
+ }
239
+
240
if (data['models'] != null) {
241
for (final item in data['models']) {
204
- final id = item['key'] as String? ?? '';
242
+ final id = item['key'] as String? ??
243
+ item['name'] as String? ??
244
+ item['id'] as String? ??
245
+ '';
246
final displayName = item['display_name'] as String?;
247
final quantization = item['quantization'];
248
final paramsString = item['params_string'] as String?;
249
+ final meta = metaById[id] ?? {};
250
251
models.add(
252
ModelInfo(
253
id: id,
254
name: displayName ?? _formatModelName(id),
255
description: item['description'] as String?,
214
- parameterCount: _parseParameterString(paramsString),
215
- contextLength: item['max_context_length'] as int?,
216
- fileSize: item['size_bytes'] as int?,
256
+ parameterCount: _parseParameterString(paramsString) ??
257
+ _paramCountFromMeta(meta['n_params']),
258
+ contextLength: item['max_context_length'] as int? ??
259
+ meta['n_ctx_train'] as int?,
260
+ fileSize: item['size_bytes'] as int? ??
261
+ meta['size'] as int?,
262
quantization: quantization?['name'] as String?,
263
architecture: item['architecture'] as String?,
264
serverType: server.type,
@@ -225,6 +270,12 @@ class ServerApiService {
270
return models;
271
}
272
273
+ int? _paramCountFromMeta(dynamic nParams) {
274
+ if (nParams == null) return null;
275
+ if (nParams is int) return nParams ~/ 1000000000;
276
+ return null;
277
+ }
278
+
279
List<ModelInfo> _parseOllamaModels(dynamic data, Server server) {
280
final List<ModelInfo> models = [];
281
if (data['models'] != null) {