@ondeinference / onde / commits / 2673cfd

Add support for UQFF format

Seto Elkahfi committed Jul 25, 2026 at 20:51 UTC 2673cfd5f8c18bb701c50a72f99d10ab7eb5c432
23 files changed +1342 -58
README.md
+27
@@ -42,6 +42,33 @@ Onde is already shipping in real apps on the App Store and Google Play. Chat run
42
43 ---
44
45 +## Model formats
46 +
47 +Onde can load GGUF models and UQFF models through the same chat engine. UQFF is mistral.rs' native pre-quantized format: pass the base repository or local model directory for tokenizer/config resolution, plus the first UQFF shard or shorthand.
48 +
49 +```rust
50 +use onde::inference::{ChatEngine, UqffModelConfig};
51 +
52 +let engine = ChatEngine::new();
53 +engine
54 + .load_uqff_model(
55 + UqffModelConfig {
56 + model_id: "google/gemma-4-E4B-it".into(),
57 + files: vec!["q4k-0.uqff".into()],
58 + display_name: "Gemma 4 E4B (UQFF Q4K)".into(),
59 + approx_memory: "~2.5 GB (UQFF Q4K)".into(),
60 + chat_template: None,
61 + },
62 + None,
63 + None,
64 + )
65 + .await?;
66 +```
67 +
68 +For sharded UQFFs, passing the first shard is enough; mistral.rs discovers sibling shards with the same prefix.
69 +
70 +---
71 +
72 ## License
73
74 Onde is dual-licensed under **MIT** and **Apache 2.0**. You may use it under either license at your option.
sdk/dart/README.md
+20
@@ -167,6 +167,26 @@ await engine.loadGgufModel(
167 );
168 ```
169
170 +Pre-quantized UQFF models use the base repository (or local model directory)
171 +for tokenizer/configuration resolution and the first UQFF shard as the file:
172 +
173 +```dart
174 +final uqff = OndeInference.uqffModelConfig(
175 + modelId: 'google/gemma-4-E4B-it',
176 + files: ['q4k-0.uqff'],
177 + displayName: 'Gemma 4 E4B (UQFF Q4K)',
178 + approxMemory: '~2.5 GB (UQFF Q4K)',
179 +);
180 +
181 +await engine.loadUqffModel(
182 + config: uqff,
183 + systemPrompt: 'You are an expert software engineer.',
184 +);
185 +```
186 +
187 +For sharded UQFF models, pass the first shard; mistral.rs discovers sibling
188 +shards with the same prefix.
189 +
190 | Model | Size | Good for |
191 |-------|------|----------|
192 | Qwen 2.5 1.5B Instruct Q4_K_M | ~941 MB | iOS, tvOS, Android |
sdk/dart/lib/onde_inference.dart
+1 -1
@@ -78,7 +78,7 @@
78 library;
79
80 // Re-export the core data types (ChatMessage, SamplingConfig,
81 -// GgufModelConfig, and friends) from the FRB-generated api.dart via types.dart.
81 +// GgufModelConfig, UqffModelConfig, and friends) from the FRB-generated api.dart via types.dart.
82 export 'src/types.dart';
83
84 // Engine API: OndeChatEngineX helpers, OndeInference static helpers,
sdk/dart/lib/src/engine.dart
+19
@@ -235,6 +235,25 @@ abstract final class OndeInference {
235 /// free function so the platform check runs natively.
236 static GgufModelConfig defaultModelConfig() => api.defaultModelConfig();
237
238 + /// Builds a configuration for a pre-quantised UQFF model.
239 + ///
240 + /// [modelId] identifies the base repository or local model directory used
241 + /// for tokenizer and configuration resolution. For sharded models, pass the
242 + /// first shard in [files]; mistral.rs discovers sibling shards.
243 + static UqffModelConfig uqffModelConfig({
244 + required String modelId,
245 + required List<String> files,
246 + required String displayName,
247 + required String approxMemory,
248 + String? chatTemplate,
249 + }) => api.uqffModelConfig(
250 + modelId: modelId,
251 + files: files,
252 + displayName: displayName,
253 + approxMemory: approxMemory,
254 + chatTemplate: chatTemplate,
255 + );
256 +
257 /// Qwen 2.5 1.5B Instruct (GGUF Q4_K_M, ~941 MB).
258 ///
259 /// Suitable for iOS, tvOS, and Android where available memory is limited.
sdk/dart/lib/src/frb_generated.dart/api.dart
+71 -2
@@ -3,14 +3,14 @@
3
4 // ignore_for_file: type=lint, unused_import, unnecessary_import
5
6 -// ignore_for_file: invalid_use_of_internal_member
6 +// ignore_for_file: invalid_use_of_internal_member, unused_import, unnecessary_import
7
8 import 'frb_generated.dart';
9 import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart';
10 import 'package:freezed_annotation/freezed_annotation.dart' hide protected;
11 part 'api.freezed.dart';
12
13 -// These function are ignored because they are on traits that is not defined in current crate (put an empty `#[frb]` on it to unignore): `assert_fields_are_eq`, `assert_fields_are_eq`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `eq`, `eq`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`
13 +// These function are ignored because they are on traits that is not defined in current crate (put an empty `#[frb]` on it to unignore): `assert_fields_are_eq`, `assert_fields_are_eq`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `eq`, `eq`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`
14
15 /// Return the platform-appropriate default `GgufModelConfig`.
16 ///
@@ -19,6 +19,21 @@ part 'api.freezed.dart';
19 GgufModelConfig defaultModelConfig() =>
20 RustLib.instance.api.crateApiDefaultModelConfig();
21
22 +/// Build a generic `UqffModelConfig`.
23 +UqffModelConfig uqffModelConfig({
24 + required String modelId,
25 + required List<String> files,
26 + required String displayName,
27 + required String approxMemory,
28 + String? chatTemplate,
29 +}) => RustLib.instance.api.crateApiUqffModelConfig(
30 + modelId: modelId,
31 + files: files,
32 + displayName: displayName,
33 + approxMemory: approxMemory,
34 + chatTemplate: chatTemplate,
35 +);
36 +
37 /// `GgufModelConfig` for Qwen 2.5 1.5B Instruct Q4_K_M (~941 MB).
38 GgufModelConfig qwen2515BConfig() =>
39 RustLib.instance.api.crateApiQwen2515BConfig();
@@ -152,6 +167,15 @@ abstract class OndeChatEngine implements RustOpaqueInterface {
167 SamplingConfig? sampling,
168 });
169
170 + /// Load a UQFF model into the engine.
171 + ///
172 + /// Returns the load duration in seconds on success.
173 + Future<double> loadUqffModel({
174 + required UqffModelConfig config,
175 + String? systemPrompt,
176 + SamplingConfig? sampling,
177 + });
178 +
179 /// Create a new engine with no model loaded.
180 factory OndeChatEngine() => RustLib.instance.api.crateApiOndeChatEngineNew();
181
@@ -456,3 +480,48 @@ class ToolCallInfo {
480 functionName == other.functionName &&
481 arguments == other.arguments;
482 }
483 +
484 +/// Configuration for loading a pre-quantised UQFF model.
485 +class UqffModelConfig {
486 + /// Base HuggingFace repository ID or local model directory.
487 + final String modelId;
488 +
489 + /// UQFF shard filename(s), first shard, or shorthand such as `q4k` / `4`.
490 + final List<String> files;
491 +
492 + /// Human-friendly display name.
493 + final String displayName;
494 +
495 + /// Approximate memory footprint.
496 + final String approxMemory;
497 +
498 + /// Optional Jinja chat template override.
499 + final String? chatTemplate;
500 +
501 + const UqffModelConfig({
502 + required this.modelId,
503 + required this.files,
504 + required this.displayName,
505 + required this.approxMemory,
506 + this.chatTemplate,
507 + });
508 +
509 + @override
510 + int get hashCode =>
511 + modelId.hashCode ^
512 + files.hashCode ^
513 + displayName.hashCode ^
514 + approxMemory.hashCode ^
515 + chatTemplate.hashCode;
516 +
517 + @override
518 + bool operator ==(Object other) =>
519 + identical(this, other) ||
520 + other is UqffModelConfig &&
521 + runtimeType == other.runtimeType &&
522 + modelId == other.modelId &&
523 + files == other.files &&
524 + displayName == other.displayName &&
525 + approxMemory == other.approxMemory &&
526 + chatTemplate == other.chatTemplate;
527 +}
sdk/dart/lib/src/frb_generated.dart/frb_generated.dart
+206 -26
@@ -66,7 +66,7 @@ class RustLib extends BaseEntrypoint<RustLibApi, RustLibApiImpl, RustLibWire> {
66 String get codegenVersion => '2.12.0';
67
68 @override
69 - int get rustContentHash => -1719313298;
69 + int get rustContentHash => -1633970317;
70
71 static const kDefaultExternalLibraryLoaderConfig =
72 ExternalLibraryLoaderConfig(
@@ -115,6 +115,13 @@ abstract class RustLibApi extends BaseApi {
115 SamplingConfig? sampling,
116 });
117
118 + Future<double> crateApiOndeChatEngineLoadUqffModel({
119 + required OndeChatEngine that,
120 + required UqffModelConfig config,
121 + String? systemPrompt,
122 + SamplingConfig? sampling,
123 + });
124 +
125 OndeChatEngine crateApiOndeChatEngineNew();
126
127 Future<void> crateApiOndeChatEnginePushHistory({
@@ -182,6 +189,14 @@ abstract class RustLibApi extends BaseApi {
189
190 GgufModelConfig crateApiQwen38BConfig();
191
192 + UqffModelConfig crateApiUqffModelConfig({
193 + required String modelId,
194 + required List<String> files,
195 + required String displayName,
196 + required String approxMemory,
197 + String? chatTemplate,
198 + });
199 +
200 RustArcIncrementStrongCountFnType
201 get rust_arc_increment_strong_count_OndeChatEngine;
202
@@ -501,13 +516,55 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
516 argNames: ["that", "config", "systemPrompt", "sampling"],
517 );
518
519 + @override
520 + Future<double> crateApiOndeChatEngineLoadUqffModel({
521 + required OndeChatEngine that,
522 + required UqffModelConfig config,
523 + String? systemPrompt,
524 + SamplingConfig? sampling,
525 + }) {
526 + return handler.executeNormal(
527 + NormalTask(
528 + callFfi: (port_) {
529 + final serializer = SseSerializer(generalizedFrbRustBinding);
530 + sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerOndeChatEngine(
531 + that,
532 + serializer,
533 + );
534 + sse_encode_box_autoadd_uqff_model_config(config, serializer);
535 + sse_encode_opt_String(systemPrompt, serializer);
536 + sse_encode_opt_box_autoadd_sampling_config(sampling, serializer);
537 + pdeCallFfi(
538 + generalizedFrbRustBinding,
539 + serializer,
540 + funcId: 9,
541 + port: port_,
542 + );
543 + },
544 + codec: SseCodec(
545 + decodeSuccessData: sse_decode_f_64,
546 + decodeErrorData: sse_decode_onde_error,
547 + ),
548 + constMeta: kCrateApiOndeChatEngineLoadUqffModelConstMeta,
549 + argValues: [that, config, systemPrompt, sampling],
550 + apiImpl: this,
551 + ),
552 + );
553 + }
554 +
555 + TaskConstMeta get kCrateApiOndeChatEngineLoadUqffModelConstMeta =>
556 + const TaskConstMeta(
557 + debugName: "OndeChatEngine_load_uqff_model",
558 + argNames: ["that", "config", "systemPrompt", "sampling"],
559 + );
560 +
561 @override
562 OndeChatEngine crateApiOndeChatEngineNew() {
563 return handler.executeSync(
564 SyncTask(
565 callFfi: () {
566 final serializer = SseSerializer(generalizedFrbRustBinding);
510 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 9)!;
567 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 10)!;
568 },
569 codec: SseCodec(
570 decodeSuccessData:
@@ -541,7 +598,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
598 pdeCallFfi(
599 generalizedFrbRustBinding,
600 serializer,
544 - funcId: 10,
601 + funcId: 11,
602 port: port_,
603 );
604 },
@@ -579,7 +636,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
636 pdeCallFfi(
637 generalizedFrbRustBinding,
638 serializer,
582 - funcId: 11,
639 + funcId: 12,
640 port: port_,
641 );
642 },
@@ -617,7 +674,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
674 pdeCallFfi(
675 generalizedFrbRustBinding,
676 serializer,
620 - funcId: 12,
677 + funcId: 13,
678 port: port_,
679 );
680 },
@@ -655,7 +712,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
712 pdeCallFfi(
713 generalizedFrbRustBinding,
714 serializer,
658 - funcId: 13,
715 + funcId: 14,
716 port: port_,
717 );
718 },
@@ -696,7 +753,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
753 pdeCallFfi(
754 generalizedFrbRustBinding,
755 serializer,
699 - funcId: 14,
756 + funcId: 15,
757 port: port_,
758 );
759 },
@@ -734,7 +791,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
791 pdeCallFfi(
792 generalizedFrbRustBinding,
793 serializer,
737 - funcId: 15,
794 + funcId: 16,
795 port: port_,
796 );
797 },
@@ -762,7 +819,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
819 callFfi: () {
820 final serializer = SseSerializer(generalizedFrbRustBinding);
821 sse_encode_String(appDataDir, serializer);
765 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 16)!;
822 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 17)!;
823 },
824 codec: SseCodec(
825 decodeSuccessData: sse_decode_unit,
@@ -786,7 +843,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
843 SyncTask(
844 callFfi: () {
845 final serializer = SseSerializer(generalizedFrbRustBinding);
789 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 17)!;
846 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 18)!;
847 },
848 codec: SseCodec(
849 decodeSuccessData: sse_decode_gguf_model_config,
@@ -808,7 +865,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
865 SyncTask(
866 callFfi: () {
867 final serializer = SseSerializer(generalizedFrbRustBinding);
811 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 18)!;
868 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 19)!;
869 },
870 codec: SseCodec(
871 decodeSuccessData: sse_decode_sampling_config,
@@ -830,7 +887,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
887 SyncTask(
888 callFfi: () {
889 final serializer = SseSerializer(generalizedFrbRustBinding);
833 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 19)!;
890 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 20)!;
891 },
892 codec: SseCodec(
893 decodeSuccessData: sse_decode_sampling_config,
@@ -855,7 +912,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
912 SyncTask(
913 callFfi: () {
914 final serializer = SseSerializer(generalizedFrbRustBinding);
858 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 20)!;
915 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 21)!;
916 },
917 codec: SseCodec(
918 decodeSuccessData: sse_decode_sampling_config,
@@ -877,7 +934,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
934 SyncTask(
935 callFfi: () {
936 final serializer = SseSerializer(generalizedFrbRustBinding);
880 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 21)!;
937 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 22)!;
938 },
939 codec: SseCodec(
940 decodeSuccessData: sse_decode_gguf_model_config,
@@ -899,7 +956,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
956 SyncTask(
957 callFfi: () {
958 final serializer = SseSerializer(generalizedFrbRustBinding);
902 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 22)!;
959 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 23)!;
960 },
961 codec: SseCodec(
962 decodeSuccessData: sse_decode_gguf_model_config,
@@ -921,7 +978,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
978 SyncTask(
979 callFfi: () {
980 final serializer = SseSerializer(generalizedFrbRustBinding);
924 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 23)!;
981 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 24)!;
982 },
983 codec: SseCodec(
984 decodeSuccessData: sse_decode_gguf_model_config,
@@ -943,7 +1000,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1000 SyncTask(
1001 callFfi: () {
1002 final serializer = SseSerializer(generalizedFrbRustBinding);
946 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 24)!;
1003 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 25)!;
1004 },
1005 codec: SseCodec(
1006 decodeSuccessData: sse_decode_gguf_model_config,
@@ -965,7 +1022,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1022 SyncTask(
1023 callFfi: () {
1024 final serializer = SseSerializer(generalizedFrbRustBinding);
968 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 25)!;
1025 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 26)!;
1026 },
1027 codec: SseCodec(
1028 decodeSuccessData: sse_decode_gguf_model_config,
@@ -987,7 +1044,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1044 SyncTask(
1045 callFfi: () {
1046 final serializer = SseSerializer(generalizedFrbRustBinding);
990 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 26)!;
1047 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 27)!;
1048 },
1049 codec: SseCodec(
1050 decodeSuccessData: sse_decode_gguf_model_config,
@@ -1009,7 +1066,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1066 SyncTask(
1067 callFfi: () {
1068 final serializer = SseSerializer(generalizedFrbRustBinding);
1012 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 27)!;
1069 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 28)!;
1070 },
1071 codec: SseCodec(
1072 decodeSuccessData: sse_decode_gguf_model_config,
@@ -1031,7 +1088,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1088 SyncTask(
1089 callFfi: () {
1090 final serializer = SseSerializer(generalizedFrbRustBinding);
1034 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 28)!;
1091 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 29)!;
1092 },
1093 codec: SseCodec(
1094 decodeSuccessData: sse_decode_gguf_model_config,
@@ -1056,7 +1113,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1113 SyncTask(
1114 callFfi: () {
1115 final serializer = SseSerializer(generalizedFrbRustBinding);
1059 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 29)!;
1116 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 30)!;
1117 },
1118 codec: SseCodec(
1119 decodeSuccessData: sse_decode_gguf_model_config,
@@ -1078,7 +1135,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1135 SyncTask(
1136 callFfi: () {
1137 final serializer = SseSerializer(generalizedFrbRustBinding);
1081 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 30)!;
1138 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 31)!;
1139 },
1140 codec: SseCodec(
1141 decodeSuccessData: sse_decode_gguf_model_config,
@@ -1100,7 +1157,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1157 SyncTask(
1158 callFfi: () {
1159 final serializer = SseSerializer(generalizedFrbRustBinding);
1103 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 31)!;
1160 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 32)!;
1161 },
1162 codec: SseCodec(
1163 decodeSuccessData: sse_decode_gguf_model_config,
@@ -1125,7 +1182,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1182 SyncTask(
1183 callFfi: () {
1184 final serializer = SseSerializer(generalizedFrbRustBinding);
1128 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 32)!;
1185 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 33)!;
1186 },
1187 codec: SseCodec(
1188 decodeSuccessData: sse_decode_gguf_model_config,
@@ -1150,7 +1207,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1207 SyncTask(
1208 callFfi: () {
1209 final serializer = SseSerializer(generalizedFrbRustBinding);
1153 - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 33)!;
1210 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 34)!;
1211 },
1212 codec: SseCodec(
1213 decodeSuccessData: sse_decode_gguf_model_config,
@@ -1166,6 +1223,47 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1223 TaskConstMeta get kCrateApiQwen38BConfigConstMeta =>
1224 const TaskConstMeta(debugName: "qwen3_8b_config", argNames: []);
1225
1226 + @override
1227 + UqffModelConfig crateApiUqffModelConfig({
1228 + required String modelId,
1229 + required List<String> files,
1230 + required String displayName,
1231 + required String approxMemory,
1232 + String? chatTemplate,
1233 + }) {
1234 + return handler.executeSync(
1235 + SyncTask(
1236 + callFfi: () {
1237 + final serializer = SseSerializer(generalizedFrbRustBinding);
1238 + sse_encode_String(modelId, serializer);
1239 + sse_encode_list_String(files, serializer);
1240 + sse_encode_String(displayName, serializer);
1241 + sse_encode_String(approxMemory, serializer);
1242 + sse_encode_opt_String(chatTemplate, serializer);
1243 + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 35)!;
1244 + },
1245 + codec: SseCodec(
1246 + decodeSuccessData: sse_decode_uqff_model_config,
1247 + decodeErrorData: null,
1248 + ),
1249 + constMeta: kCrateApiUqffModelConfigConstMeta,
1250 + argValues: [modelId, files, displayName, approxMemory, chatTemplate],
1251 + apiImpl: this,
1252 + ),
1253 + );
1254 + }
1255 +
1256 + TaskConstMeta get kCrateApiUqffModelConfigConstMeta => const TaskConstMeta(
1257 + debugName: "uqff_model_config",
1258 + argNames: [
1259 + "modelId",
1260 + "files",
1261 + "displayName",
1262 + "approxMemory",
1263 + "chatTemplate",
1264 + ],
1265 + );
1266 +
1267 RustArcIncrementStrongCountFnType
1268 get rust_arc_increment_strong_count_OndeChatEngine => wire
1269 .rust_arc_increment_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerOndeChatEngine;
@@ -1263,6 +1361,12 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1361 return dco_decode_u_64(raw);
1362 }
1363
1364 + @protected
1365 + UqffModelConfig dco_decode_box_autoadd_uqff_model_config(dynamic raw) {
1366 + // Codec=Dco (DartCObject based), see doc to use other codecs
1367 + return dco_decode_uqff_model_config(raw);
1368 + }
1369 +
1370 @protected
1371 ChatMessage dco_decode_chat_message(dynamic raw) {
1372 // Codec=Dco (DartCObject based), see doc to use other codecs
@@ -1485,6 +1589,21 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1589 return;
1590 }
1591
1592 + @protected
1593 + UqffModelConfig dco_decode_uqff_model_config(dynamic raw) {
1594 + // Codec=Dco (DartCObject based), see doc to use other codecs
1595 + final arr = raw as List<dynamic>;
1596 + if (arr.length != 5)
1597 + throw Exception('unexpected arr length: expect 5 but see ${arr.length}');
1598 + return UqffModelConfig(
1599 + modelId: dco_decode_String(arr[0]),
1600 + files: dco_decode_list_String(arr[1]),
1601 + displayName: dco_decode_String(arr[2]),
1602 + approxMemory: dco_decode_String(arr[3]),
1603 + chatTemplate: dco_decode_opt_String(arr[4]),
1604 + );
1605 + }
1606 +
1607 @protected
1608 BigInt dco_decode_usize(dynamic raw) {
1609 // Codec=Dco (DartCObject based), see doc to use other codecs
@@ -1597,6 +1716,14 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
1716 return (sse_decode_u_64(deserializer));
1717 }
1718
1719 + @protected
1720 + UqffModelConfig sse_decode_box_autoadd_uqff_model_config(
1721 + SseDeserializer deserializer,
1722 + ) {
1723 + // Codec=Sse (Serialization based), see doc to use other codecs
1724 + return (sse_decode_uqff_model_config(deserializer));
1725 + }
1726 +
1727 @protected
1728 ChatMessage sse_decode_chat_message(SseDeserializer deserializer) {
1729 // Codec=Sse (Serialization based), see doc to use other codecs
@@ -1879,6 +2006,23 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
2006 // Codec=Sse (Serialization based), see doc to use other codecs
2007 }
2008
2009 + @protected
2010 + UqffModelConfig sse_decode_uqff_model_config(SseDeserializer deserializer) {
2011 + // Codec=Sse (Serialization based), see doc to use other codecs
2012 + var var_modelId = sse_decode_String(deserializer);
2013 + var var_files = sse_decode_list_String(deserializer);
2014 + var var_displayName = sse_decode_String(deserializer);
2015 + var var_approxMemory = sse_decode_String(deserializer);
2016 + var var_chatTemplate = sse_decode_opt_String(deserializer);
2017 + return UqffModelConfig(
2018 + modelId: var_modelId,
2019 + files: var_files,
2020 + displayName: var_displayName,
2021 + approxMemory: var_approxMemory,
2022 + chatTemplate: var_chatTemplate,
2023 + );
2024 + }
2025 +
2026 @protected
2027 BigInt sse_decode_usize(SseDeserializer deserializer) {
2028 // Codec=Sse (Serialization based), see doc to use other codecs
@@ -2007,6 +2151,15 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
2151 sse_encode_u_64(self, serializer);
2152 }
2153
2154 + @protected
2155 + void sse_encode_box_autoadd_uqff_model_config(
2156 + UqffModelConfig self,
2157 + SseSerializer serializer,
2158 + ) {
2159 + // Codec=Sse (Serialization based), see doc to use other codecs
2160 + sse_encode_uqff_model_config(self, serializer);
2161 + }
2162 +
2163 @protected
2164 void sse_encode_chat_message(ChatMessage self, SseSerializer serializer) {
2165 // Codec=Sse (Serialization based), see doc to use other codecs
@@ -2246,6 +2399,19 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi {
2399 // Codec=Sse (Serialization based), see doc to use other codecs
2400 }
2401
2402 + @protected
2403 + void sse_encode_uqff_model_config(
2404 + UqffModelConfig self,
2405 + SseSerializer serializer,
2406 + ) {
2407 + // Codec=Sse (Serialization based), see doc to use other codecs
2408 + sse_encode_String(self.modelId, serializer);
2409 + sse_encode_list_String(self.files, serializer);
2410 + sse_encode_String(self.displayName, serializer);
2411 + sse_encode_String(self.approxMemory, serializer);
2412 + sse_encode_opt_String(self.chatTemplate, serializer);
2413 + }
2414 +
2415 @protected
2416 void sse_encode_usize(BigInt self, SseSerializer serializer) {
2417 // Codec=Sse (Serialization based), see doc to use other codecs
@@ -2336,6 +2502,20 @@ class OndeChatEngineImpl extends RustOpaque implements OndeChatEngine {
2502 sampling: sampling,
2503 );
2504
2505 + /// Load a UQFF model into the engine.
2506 + ///
2507 + /// Returns the load duration in seconds on success.
2508 + Future<double> loadUqffModel({
2509 + required UqffModelConfig config,
2510 + String? systemPrompt,
2511 + SamplingConfig? sampling,
2512 + }) => RustLib.instance.api.crateApiOndeChatEngineLoadUqffModel(
2513 + that: this,
2514 + config: config,
2515 + systemPrompt: systemPrompt,
2516 + sampling: sampling,
2517 + );
2518 +
2519 /// Append a message to history without running inference.
2520 ///
2521 /// Useful for restoring a saved conversation or injecting context.
sdk/dart/lib/src/frb_generated.dart/frb_generated.io.dart
+26
@@ -74,6 +74,9 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl<RustLibWire> {
74 @protected
75 BigInt dco_decode_box_autoadd_u_64(dynamic raw);
76
77 + @protected
78 + UqffModelConfig dco_decode_box_autoadd_uqff_model_config(dynamic raw);
79 +
80 @protected
81 ChatMessage dco_decode_chat_message(dynamic raw);
82
@@ -149,6 +152,9 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl<RustLibWire> {
152 @protected
153 void dco_decode_unit(dynamic raw);
154
155 + @protected
156 + UqffModelConfig dco_decode_uqff_model_config(dynamic raw);
157 +
158 @protected
159 BigInt dco_decode_usize(dynamic raw);
160
@@ -206,6 +212,11 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl<RustLibWire> {
212 @protected
213 BigInt sse_decode_box_autoadd_u_64(SseDeserializer deserializer);
214
215 + @protected
216 + UqffModelConfig sse_decode_box_autoadd_uqff_model_config(
217 + SseDeserializer deserializer,
218 + );
219 +
220 @protected
221 ChatMessage sse_decode_chat_message(SseDeserializer deserializer);
222
@@ -285,6 +296,9 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl<RustLibWire> {
296 @protected
297 void sse_decode_unit(SseDeserializer deserializer);
298
299 + @protected
300 + UqffModelConfig sse_decode_uqff_model_config(SseDeserializer deserializer);
301 +
302 @protected
303 BigInt sse_decode_usize(SseDeserializer deserializer);
304
@@ -354,6 +368,12 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl<RustLibWire> {
368 @protected
369 void sse_encode_box_autoadd_u_64(BigInt self, SseSerializer serializer);
370
371 + @protected
372 + void sse_encode_box_autoadd_uqff_model_config(
373 + UqffModelConfig self,
374 + SseSerializer serializer,
375 + );
376 +
377 @protected
378 void sse_encode_chat_message(ChatMessage self, SseSerializer serializer);
379
@@ -450,6 +470,12 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl<RustLibWire> {
470 @protected
471 void sse_encode_unit(void self, SseSerializer serializer);
472
473 + @protected
474 + void sse_encode_uqff_model_config(
475 + UqffModelConfig self,
476 + SseSerializer serializer,
477 + );
478 +
479 @protected
480 void sse_encode_usize(BigInt self, SseSerializer serializer);
481 }
sdk/dart/lib/src/types.dart
+2 -1
@@ -4,7 +4,7 @@
4 /// Generated and convenience types for the Onde Inference SDK.
5 ///
6 /// All core data types ([ChatRole], [ChatMessage], [SamplingConfig],
7 -/// [GgufModelConfig], [InferenceResult], [StreamChunk], [EngineStatus],
7 +/// [GgufModelConfig], [UqffModelConfig], [InferenceResult], [StreamChunk], [EngineStatus],
8 /// [EngineInfo], [OndeError]) are generated by `flutter_rust_bridge_codegen`
9 /// and live in `frb_generated.dart/api.dart`. This file re-exports them
10 /// and adds a few Dart-idiomatic convenience extensions.
@@ -19,6 +19,7 @@ export 'frb_generated.dart/api.dart'
19 ChatMessage,
20 SamplingConfig,
21 GgufModelConfig,
22 + UqffModelConfig,
23 InferenceResult,
24 StreamChunk,
25 ToolCallInfo,
sdk/dart/rust/src/api.rs
+78 -1
@@ -17,7 +17,7 @@ use onde::inference::{
17 EngineInfo as OndeEngineInfo, EngineStatus as OndeEngineStatus,
18 GgufModelConfig as OndeGgufModelConfig, InferenceError, InferenceResult as OndeInferenceResult,
19 SamplingConfig as OndeSamplingConfig, StreamChunk as OndeStreamChunk,
20 - ToolCallInfo as OndeToolCallInfo,
20 + ToolCallInfo as OndeToolCallInfo, UqffModelConfig as OndeUqffModelConfig,
21 };
22
23 // ── Mirror: ChatRole ──────────────────────────────────────────────────────────
@@ -162,6 +162,47 @@ impl From<GgufModelConfig> for OndeGgufModelConfig {
162 }
163 }
164
165 +// ── Mirror: UqffModelConfig ──────────────────────────────────────────────────
166 +
167 +/// Configuration for loading a pre-quantised UQFF model.
168 +#[derive(Debug, Clone)]
169 +pub struct UqffModelConfig {
170 + /// Base HuggingFace repository ID or local model directory.
171 + pub model_id: String,
172 + /// UQFF shard filename(s), first shard, or shorthand such as `q4k` / `4`.
173 + pub files: Vec<String>,
174 + /// Human-friendly display name.
175 + pub display_name: String,
176 + /// Approximate memory footprint.
177 + pub approx_memory: String,
178 + /// Optional Jinja chat template override.
179 + pub chat_template: Option<String>,
180 +}
181 +
182 +impl From<OndeUqffModelConfig> for UqffModelConfig {
183 + fn from(c: OndeUqffModelConfig) -> Self {
184 + Self {
185 + model_id: c.model_id,
186 + files: c.files,
187 + display_name: c.display_name,
188 + approx_memory: c.approx_memory,
189 + chat_template: c.chat_template,
190 + }
191 + }
192 +}
193 +
194 +impl From<UqffModelConfig> for OndeUqffModelConfig {
195 + fn from(c: UqffModelConfig) -> Self {
196 + Self {
197 + model_id: c.model_id,
198 + files: c.files,
199 + display_name: c.display_name,
200 + approx_memory: c.approx_memory,
201 + chat_template: c.chat_template,
202 + }
203 + }
204 +}
205 +
206 // ── Mirror: ToolCallInfo ────────────────────────────────────────────────────
207
208 /// Structured tool-call request emitted by the model.
@@ -360,6 +401,23 @@ impl OndeChatEngine {
401 Ok(elapsed.as_secs_f64())
402 }
403
404 + /// Load a UQFF model into the engine.
405 + ///
406 + /// Returns the load duration in seconds on success.
407 + pub async fn load_uqff_model(
408 + &self,
409 + config: UqffModelConfig,
410 + system_prompt: Option<String>,
411 + sampling: Option<SamplingConfig>,
412 + ) -> Result<f64, OndeError> {
413 + let elapsed = self
414 + .inner
415 + .load_uqff_model(config.into(), system_prompt, sampling.map(Into::into))
416 + .await
417 + .map_err(OndeError::from)?;
418 + Ok(elapsed.as_secs_f64())
419 + }
420 +
421 /// Load the model assigned to this Onde app from the dashboard.
422 ///
423 /// Returns the wall-clock seconds taken to load the model.
@@ -521,6 +579,25 @@ pub fn default_model_config() -> GgufModelConfig {
579 OndeGgufModelConfig::platform_default().into()
580 }
581
582 +/// Build a generic `UqffModelConfig`.
583 +#[frb(sync)]
584 +pub fn uqff_model_config(
585 + model_id: String,
586 + files: Vec<String>,
587 + display_name: String,
588 + approx_memory: String,
589 + chat_template: Option<String>,
590 +) -> UqffModelConfig {
591 + OndeUqffModelConfig {
592 + model_id,
593 + files,
594 + display_name,
595 + approx_memory,
596 + chat_template,
597 + }
598 + .into()
599 +}
600 +
601 /// `GgufModelConfig` for Qwen 2.5 1.5B Instruct Q4_K_M (~941 MB).
602 #[frb(sync)]
603 pub fn qwen25_1_5b_config() -> GgufModelConfig {
sdk/dart/rust/src/frb_generated.rs
+184 -26
@@ -39,7 +39,7 @@ flutter_rust_bridge::frb_generated_boilerplate!(
39 default_rust_auto_opaque = RustAutoOpaqueMoi,
40 );
41 pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.12.0";
42 -pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -1719313298;
42 +pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -1633970317;
43
44 // Section: executor
45
@@ -524,6 +524,70 @@ fn wire__crate__api__OndeChatEngine_load_gguf_model_impl(
524 },
525 )
526 }
527 +fn wire__crate__api__OndeChatEngine_load_uqff_model_impl(
528 + port_: flutter_rust_bridge::for_generated::MessagePort,
529 + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
530 + rust_vec_len_: i32,
531 + data_len_: i32,
532 +) {
533 + FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::<flutter_rust_bridge::for_generated::SseCodec, _, _, _>(
534 + flutter_rust_bridge::for_generated::TaskInfo {
535 + debug_name: "OndeChatEngine_load_uqff_model",
536 + port: Some(port_),
537 + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal,
538 + },
539 + move || {
540 + let message = unsafe {
541 + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire(
542 + ptr_,
543 + rust_vec_len_,
544 + data_len_,
545 + )
546 + };
547 + let mut deserializer =
548 + flutter_rust_bridge::for_generated::SseDeserializer::new(message);
549 + let api_that = <RustOpaqueMoi<
550 + flutter_rust_bridge::for_generated::RustAutoOpaqueInner<OndeChatEngine>,
551 + >>::sse_decode(&mut deserializer);
552 + let api_config = <crate::api::UqffModelConfig>::sse_decode(&mut deserializer);
553 + let api_system_prompt = <Option<String>>::sse_decode(&mut deserializer);
554 + let api_sampling = <Option<crate::api::SamplingConfig>>::sse_decode(&mut deserializer);
555 + deserializer.end();
556 + move |context| async move {
557 + transform_result_sse::<_, crate::api::OndeError>(
558 + (move || async move {
559 + let mut api_that_guard = None;
560 + let decode_indices_ =
561 + flutter_rust_bridge::for_generated::lockable_compute_decode_order(
562 + vec![flutter_rust_bridge::for_generated::LockableOrderInfo::new(
563 + &api_that, 0, false,
564 + )],
565 + );
566 + for i in decode_indices_ {
567 + match i {
568 + 0 => {
569 + api_that_guard =
570 + Some(api_that.lockable_decode_async_ref().await)
571 + }
572 + _ => unreachable!(),
573 + }
574 + }
575 + let api_that_guard = api_that_guard.unwrap();
576 + let output_ok = crate::api::OndeChatEngine::load_uqff_model(
577 + &*api_that_guard,
578 + api_config,
579 + api_system_prompt,
580 + api_sampling,
581 + )
582 + .await?;
583 + Ok(output_ok)
584 + })()
585 + .await,
586 + )
587 + }
588 + },
589 + )
590 +}
591 fn wire__crate__api__OndeChatEngine_new_impl(
592 ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
593 rust_vec_len_: i32,
@@ -1440,6 +1504,46 @@ fn wire__crate__api__qwen3_8b_config_impl(
1504 },
1505 )
1506 }
1507 +fn wire__crate__api__uqff_model_config_impl(
1508 + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
1509 + rust_vec_len_: i32,
1510 + data_len_: i32,
1511 +) -> flutter_rust_bridge::for_generated::WireSyncRust2DartSse {
1512 + FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::<flutter_rust_bridge::for_generated::SseCodec, _>(
1513 + flutter_rust_bridge::for_generated::TaskInfo {
1514 + debug_name: "uqff_model_config",
1515 + port: None,
1516 + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync,
1517 + },
1518 + move || {
1519 + let message = unsafe {
1520 + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire(
1521 + ptr_,
1522 + rust_vec_len_,
1523 + data_len_,
1524 + )
1525 + };
1526 + let mut deserializer =
1527 + flutter_rust_bridge::for_generated::SseDeserializer::new(message);
1528 + let api_model_id = <String>::sse_decode(&mut deserializer);
1529 + let api_files = <Vec<String>>::sse_decode(&mut deserializer);
1530 + let api_display_name = <String>::sse_decode(&mut deserializer);
1531 + let api_approx_memory = <String>::sse_decode(&mut deserializer);
1532 + let api_chat_template = <Option<String>>::sse_decode(&mut deserializer);
1533 + deserializer.end();
1534 + transform_result_sse::<_, ()>((move || {
1535 + let output_ok = Result::<_, ()>::Ok(crate::api::uqff_model_config(
1536 + api_model_id,
1537 + api_files,
1538 + api_display_name,
1539 + api_approx_memory,
1540 + api_chat_template,
1541 + ))?;
1542 + Ok(output_ok)
1543 + })())
1544 + },
1545 + )
1546 +}
1547
1548 // Section: related_funcs
1549
@@ -1823,6 +1927,24 @@ impl SseDecode for () {
1927 fn sse_decode(deserializer: &mut flutter_rust_bridge::for_generated::SseDeserializer) -> Self {}
1928 }
1929
1930 +impl SseDecode for crate::api::UqffModelConfig {
1931 + // Codec=Sse (Serialization based), see doc to use other codecs
1932 + fn sse_decode(deserializer: &mut flutter_rust_bridge::for_generated::SseDeserializer) -> Self {
1933 + let mut var_modelId = <String>::sse_decode(deserializer);
1934 + let mut var_files = <Vec<String>>::sse_decode(deserializer);
1935 + let mut var_displayName = <String>::sse_decode(deserializer);
1936 + let mut var_approxMemory = <String>::sse_decode(deserializer);
1937 + let mut var_chatTemplate = <Option<String>>::sse_decode(deserializer);
1938 + return crate::api::UqffModelConfig {
1939 + model_id: var_modelId,
1940 + files: var_files,
1941 + display_name: var_displayName,
1942 + approx_memory: var_approxMemory,
1943 + chat_template: var_chatTemplate,
1944 + };
1945 + }
1946 +}
1947 +
1948 impl SseDecode for usize {
1949 // Codec=Sse (Serialization based), see doc to use other codecs
1950 fn sse_decode(deserializer: &mut flutter_rust_bridge::for_generated::SseDeserializer) -> Self {
@@ -1859,19 +1981,22 @@ fn pde_ffi_dispatcher_primary_impl(
1981 8 => {
1982 wire__crate__api__OndeChatEngine_load_gguf_model_impl(port, ptr, rust_vec_len, data_len)
1983 }
1862 - 10 => wire__crate__api__OndeChatEngine_push_history_impl(port, ptr, rust_vec_len, data_len),
1863 - 11 => wire__crate__api__OndeChatEngine_send_message_impl(port, ptr, rust_vec_len, data_len),
1864 - 12 => wire__crate__api__OndeChatEngine_set_sampling_impl(port, ptr, rust_vec_len, data_len),
1865 - 13 => wire__crate__api__OndeChatEngine_set_system_prompt_impl(
1984 + 9 => {
1985 + wire__crate__api__OndeChatEngine_load_uqff_model_impl(port, ptr, rust_vec_len, data_len)
1986 + }
1987 + 11 => wire__crate__api__OndeChatEngine_push_history_impl(port, ptr, rust_vec_len, data_len),
1988 + 12 => wire__crate__api__OndeChatEngine_send_message_impl(port, ptr, rust_vec_len, data_len),
1989 + 13 => wire__crate__api__OndeChatEngine_set_sampling_impl(port, ptr, rust_vec_len, data_len),
1990 + 14 => wire__crate__api__OndeChatEngine_set_system_prompt_impl(
1991 port,
1992 ptr,
1993 rust_vec_len,
1994 data_len,
1995 ),
1871 - 14 => {
1996 + 15 => {
1997 wire__crate__api__OndeChatEngine_stream_message_impl(port, ptr, rust_vec_len, data_len)
1998 }
1874 - 15 => wire__crate__api__OndeChatEngine_unload_model_impl(port, ptr, rust_vec_len, data_len),
1999 + 16 => wire__crate__api__OndeChatEngine_unload_model_impl(port, ptr, rust_vec_len, data_len),
2000 _ => unreachable!(),
2001 }
2002 }
@@ -1884,27 +2009,28 @@ fn pde_ffi_dispatcher_sync_impl(
2009 ) -> flutter_rust_bridge::for_generated::WireSyncRust2DartSse {
2010 // Codec=Pde (Serialization + dispatch), see doc to use other codecs
2011 match func_id {
1887 - 9 => wire__crate__api__OndeChatEngine_new_impl(ptr, rust_vec_len, data_len),
1888 - 16 => wire__crate__api__configure_cache_dir_impl(ptr, rust_vec_len, data_len),
1889 - 17 => wire__crate__api__default_model_config_impl(ptr, rust_vec_len, data_len),
1890 - 18 => wire__crate__api__default_sampling_config_impl(ptr, rust_vec_len, data_len),
1891 - 19 => wire__crate__api__deterministic_sampling_config_impl(ptr, rust_vec_len, data_len),
1892 - 20 => wire__crate__api__mobile_sampling_config_impl(ptr, rust_vec_len, data_len),
1893 - 21 => wire__crate__api__qwen25_1_5b_config_impl(ptr, rust_vec_len, data_len),
1894 - 22 => wire__crate__api__qwen25_3b_config_impl(ptr, rust_vec_len, data_len),
1895 - 23 => wire__crate__api__qwen25_coder_1_5b_config_impl(ptr, rust_vec_len, data_len),
1896 - 24 => wire__crate__api__qwen25_coder_3b_config_impl(ptr, rust_vec_len, data_len),
1897 - 25 => wire__crate__api__qwen3_0_6b_config_impl(ptr, rust_vec_len, data_len),
1898 - 26 => wire__crate__api__qwen3_14b_config_impl(ptr, rust_vec_len, data_len),
1899 - 27 => wire__crate__api__qwen3_1_7b_config_impl(ptr, rust_vec_len, data_len),
1900 - 28 => {
2012 + 10 => wire__crate__api__OndeChatEngine_new_impl(ptr, rust_vec_len, data_len),
2013 + 17 => wire__crate__api__configure_cache_dir_impl(ptr, rust_vec_len, data_len),
2014 + 18 => wire__crate__api__default_model_config_impl(ptr, rust_vec_len, data_len),
2015 + 19 => wire__crate__api__default_sampling_config_impl(ptr, rust_vec_len, data_len),
2016 + 20 => wire__crate__api__deterministic_sampling_config_impl(ptr, rust_vec_len, data_len),
2017 + 21 => wire__crate__api__mobile_sampling_config_impl(ptr, rust_vec_len, data_len),
2018 + 22 => wire__crate__api__qwen25_1_5b_config_impl(ptr, rust_vec_len, data_len),
2019 + 23 => wire__crate__api__qwen25_3b_config_impl(ptr, rust_vec_len, data_len),
2020 + 24 => wire__crate__api__qwen25_coder_1_5b_config_impl(ptr, rust_vec_len, data_len),
2021 + 25 => wire__crate__api__qwen25_coder_3b_config_impl(ptr, rust_vec_len, data_len),
2022 + 26 => wire__crate__api__qwen3_0_6b_config_impl(ptr, rust_vec_len, data_len),
2023 + 27 => wire__crate__api__qwen3_14b_config_impl(ptr, rust_vec_len, data_len),
2024 + 28 => wire__crate__api__qwen3_1_7b_config_impl(ptr, rust_vec_len, data_len),
2025 + 29 => {
2026 wire__crate__api__qwen3_30b_a3b_instruct_2507_config_impl(ptr, rust_vec_len, data_len)
2027 }
1903 - 29 => wire__crate__api__qwen3_32b_config_impl(ptr, rust_vec_len, data_len),
1904 - 30 => wire__crate__api__qwen3_4b_config_impl(ptr, rust_vec_len, data_len),
1905 - 31 => wire__crate__api__qwen3_4b_instruct_2507_config_impl(ptr, rust_vec_len, data_len),
1906 - 32 => wire__crate__api__qwen3_4b_thinking_2507_config_impl(ptr, rust_vec_len, data_len),
1907 - 33 => wire__crate__api__qwen3_8b_config_impl(ptr, rust_vec_len, data_len),
2028 + 30 => wire__crate__api__qwen3_32b_config_impl(ptr, rust_vec_len, data_len),
2029 + 31 => wire__crate__api__qwen3_4b_config_impl(ptr, rust_vec_len, data_len),
2030 + 32 => wire__crate__api__qwen3_4b_instruct_2507_config_impl(ptr, rust_vec_len, data_len),
2031 + 33 => wire__crate__api__qwen3_4b_thinking_2507_config_impl(ptr, rust_vec_len, data_len),
2032 + 34 => wire__crate__api__qwen3_8b_config_impl(ptr, rust_vec_len, data_len),
2033 + 35 => wire__crate__api__uqff_model_config_impl(ptr, rust_vec_len, data_len),
2034 _ => unreachable!(),
2035 }
2036 }
@@ -2123,6 +2249,27 @@ impl flutter_rust_bridge::IntoIntoDart<crate::api::ToolCallInfo> for crate::api:
2249 self
2250 }
2251 }
2252 +// Codec=Dco (DartCObject based), see doc to use other codecs
2253 +impl flutter_rust_bridge::IntoDart for crate::api::UqffModelConfig {
2254 + fn into_dart(self) -> flutter_rust_bridge::for_generated::DartAbi {
2255 + [
2256 + self.model_id.into_into_dart().into_dart(),
2257 + self.files.into_into_dart().into_dart(),
2258 + self.display_name.into_into_dart().into_dart(),
2259 + self.approx_memory.into_into_dart().into_dart(),
2260 + self.chat_template.into_into_dart().into_dart(),
2261 + ]
2262 + .into_dart()
2263 + }
2264 +}
2265 +impl flutter_rust_bridge::for_generated::IntoDartExceptPrimitive for crate::api::UqffModelConfig {}
2266 +impl flutter_rust_bridge::IntoIntoDart<crate::api::UqffModelConfig>
2267 + for crate::api::UqffModelConfig
2268 +{
2269 + fn into_into_dart(self) -> crate::api::UqffModelConfig {
2270 + self
2271 + }
2272 +}
2273
2274 impl SseEncode for flutter_rust_bridge::for_generated::anyhow::Error {
2275 // Codec=Sse (Serialization based), see doc to use other codecs
@@ -2442,6 +2589,17 @@ impl SseEncode for () {
2589 fn sse_encode(self, serializer: &mut flutter_rust_bridge::for_generated::SseSerializer) {}
2590 }
2591
2592 +impl SseEncode for crate::api::UqffModelConfig {
2593 + // Codec=Sse (Serialization based), see doc to use other codecs
2594 + fn sse_encode(self, serializer: &mut flutter_rust_bridge::for_generated::SseSerializer) {
2595 + <String>::sse_encode(self.model_id, serializer);
2596 + <Vec<String>>::sse_encode(self.files, serializer);
2597 + <String>::sse_encode(self.display_name, serializer);
2598 + <String>::sse_encode(self.approx_memory, serializer);
2599 + <Option<String>>::sse_encode(self.chat_template, serializer);
2600 + }
2601 +}
2602 +
2603 impl SseEncode for usize {
2604 // Codec=Sse (Serialization based), see doc to use other codecs
2605 fn sse_encode(self, serializer: &mut flutter_rust_bridge::for_generated::SseSerializer) {
sdk/kotlin/lib/src/shared/kotlin/com/ondeinference/onde/Convenience.kt
+14
@@ -21,6 +21,20 @@ object OndeSampling {
21 object OndeModels {
22 /** Platform-appropriate default — Qwen 2.5 1.5B on Android, 3B on macOS. */
23 fun default(): GgufModelConfig = uniffi.onde.defaultModelConfig()
24 + /** Generic UQFF model config using a base repo/local directory plus UQFF shard names. */
25 + fun uqff(
26 + modelId: String,
27 + files: List<String>,
28 + displayName: String,
29 + approxMemory: String,
30 + chatTemplate: String? = null,
31 + ): UqffModelConfig = uniffi.onde.uqffModelConfig(
32 + modelId = modelId,
33 + files = files,
34 + displayName = displayName,
35 + approxMemory = approxMemory,
36 + chatTemplate = chatTemplate,
37 + )
38 /** Qwen 2.5 1.5B Instruct GGUF Q4_K_M (~941 MB). */
39 fun qwen25_1_5b(): GgufModelConfig = uniffi.onde.qwen2515bConfig()
40 /** Qwen 2.5 3B Instruct GGUF Q4_K_M (~1.93 GB). */
sdk/kotlin/lib/src/shared/kotlin/com/ondeinference/onde/OndeInference.kt
+22
@@ -21,6 +21,7 @@ typealias ChatMessage = uniffi.onde.ChatMessage
21 typealias ChatRole = uniffi.onde.ChatRole
22 typealias SamplingConfig = uniffi.onde.SamplingConfig
23 typealias GgufModelConfig = uniffi.onde.GgufModelConfig
24 +typealias UqffModelConfig = uniffi.onde.UqffModelConfig
25 typealias InferenceResult = uniffi.onde.InferenceResult
26 typealias StreamChunk = uniffi.onde.StreamChunk
27 typealias EngineInfo = uniffi.onde.EngineInfo
@@ -149,6 +150,27 @@ class OndeInference internal constructor(
150 )
151 }
152
153 + /**
154 + * Load a specific UQFF model.
155 + *
156 + * @param config Base model repository and UQFF shard configuration.
157 + * @param systemPrompt Optional system prompt.
158 + * @param sampling Optional sampling configuration.
159 + * @return Wall-clock model loading time in seconds.
160 + */
161 + suspend fun loadUqffModel(
162 + config: UqffModelConfig,
163 + systemPrompt: String? = null,
164 + sampling: SamplingConfig? = null,
165 + ): Double = withContext(Dispatchers.IO) {
166 + ensureConfigured()
167 + engine.loadUqffModel(
168 + config = config,
169 + systemPrompt = systemPrompt,
170 + sampling = sampling,
171 + )
172 + }
173 +
174 /** Unload the current model and free all associated memory. */
175 suspend fun unload(): String? = withContext(Dispatchers.IO) {
176 engine.unloadModel()
sdk/react-native/README.md
+15
@@ -69,6 +69,7 @@ await OndeChatEngine.unloadModel();
69 |--------|---------|--------------|
70 | `loadDefaultModel(systemPrompt?, sampling?)` | `Promise<number>` | Load the platform default. Returns load time in seconds. |
71 | `loadModel(config, systemPrompt?, sampling?)` | `Promise<number>` | Load a specific GGUF model. |
72 +| `loadUqffModel(config, systemPrompt?, sampling?)` | `Promise<number>` | Load a specific UQFF model. |
73 | `unloadModel()` | `Promise<string \| null>` | Drop the model, free memory. Returns the model name. |
74 | `isLoaded()` | `boolean` | Is anything loaded right now? |
75 | `info()` | `Promise<EngineInfo>` | Status, model name, memory, history length. |
@@ -86,6 +87,7 @@ await OndeChatEngine.unloadModel();
87 ```typescript
88 import {
89 defaultModelConfig, // platform-aware (1.5B on mobile, 3B on desktop)
90 + uqffModelConfig, // base repo/local dir + UQFF shard(s)
91 qwen251_5bConfig, // force 1.5B (~941 MB)
92 qwen253bConfig, // force 3B (~1.93 GB)
93 defaultSamplingConfig, // temp=0.7, top_p=0.95, max_tokens=512
@@ -97,6 +99,19 @@ import {
99 } from "@ondeinference/react-native";
100 ```
101
102 +UQFF example:
103 +
104 +```typescript
105 +const config = uqffModelConfig(
106 + "google/gemma-4-E4B-it",
107 + ["q4k-0.uqff"],
108 + "Gemma 4 E4B (UQFF Q4K)",
109 + "~2.5 GB (UQFF Q4K)",
110 +);
111 +
112 +await OndeChatEngine.loadUqffModel(config);
113 +```
114 +
115 ## Example app
116
117 There is a working chat app in [`example/`](./example):
sdk/react-native/android/src/main/java/com/ondeinference/OndeInferenceModule.kt
+28
@@ -32,6 +32,13 @@ class OndeInferenceModule : Module() {
32 samplingJson: String?
33 ): String
34
35 + private external fun nativeEngineLoadUqffModel(
36 + engine: Long,
37 + configJson: String,
38 + systemPrompt: String?,
39 + samplingJson: String?
40 + ): String
41 +
42 private external fun nativeEngineUnloadModel(engine: Long): String
43 private external fun nativeEngineIsLoaded(engine: Long): Boolean
44 private external fun nativeEngineInfo(engine: Long): String
@@ -49,6 +56,14 @@ class OndeInferenceModule : Module() {
56 ): String
57
58 private external fun nativeDefaultModelConfig(): String
59 + private external fun nativeUqffModelConfig(
60 + modelId: String,
61 + filesJson: String,
62 + displayName: String,
63 + approxMemory: String,
64 + chatTemplate: String?
65 + ): String
66 +
67 private external fun nativeQwen251_5bConfig(): String
68 private external fun nativeQwen253bConfig(): String
69 private external fun nativeQwen3_0_6bConfig(): String
@@ -111,6 +126,14 @@ class OndeInferenceModule : Module() {
126 // -- Config free functions (no engine needed) --
127
128 Function("defaultModelConfig") { nativeDefaultModelConfig() }
129 + Function("uqffModelConfig") {
130 + modelId: String,
131 + filesJson: String,
132 + displayName: String,
133 + approxMemory: String,
134 + chatTemplate: String? ->
135 + nativeUqffModelConfig(modelId, filesJson, displayName, approxMemory, chatTemplate)
136 + }
137 Function("qwen251_5bConfig") { nativeQwen251_5bConfig() }
138 Function("qwen253bConfig") { nativeQwen253bConfig() }
139 Function("qwen3_0_6bConfig") { nativeQwen3_0_6bConfig() }
@@ -138,6 +161,11 @@ class OndeInferenceModule : Module() {
161 nativeEngineLoadModel(enginePtr, configJson, systemPrompt, samplingJson)
162 }
163
164 + AsyncFunction("loadUqffModel") { configJson: String, systemPrompt: String?, samplingJson: String? ->
165 + check(enginePtr != 0L) { "Engine not initialized" }
166 + nativeEngineLoadUqffModel(enginePtr, configJson, systemPrompt, samplingJson)
167 + }
168 +
169 AsyncFunction("unloadModel") {
170 check(enginePtr != 0L) { "Engine not initialized" }
171 nativeEngineUnloadModel(enginePtr)
sdk/react-native/ios/OndeInferenceModule.swift
+87
@@ -31,6 +31,14 @@ func onde_engine_load_model(
31 _ samplingJson: UnsafePointer<CChar>?
32 ) -> UnsafeMutablePointer<CChar>?
33
34 +@_silgen_name("onde_engine_load_uqff_model")
35 +func onde_engine_load_uqff_model(
36 + _ engine: UnsafeMutableRawPointer,
37 + _ configJson: UnsafePointer<CChar>,
38 + _ systemPrompt: UnsafePointer<CChar>?,
39 + _ samplingJson: UnsafePointer<CChar>?
40 +) -> UnsafeMutablePointer<CChar>?
41 +
42 @_silgen_name("onde_engine_unload_model")
43 func onde_engine_unload_model(_ engine: UnsafeMutableRawPointer) -> UnsafeMutablePointer<CChar>?
44
@@ -83,6 +91,15 @@ func onde_engine_generate(
91 @_silgen_name("onde_default_model_config")
92 func onde_default_model_config() -> UnsafeMutablePointer<CChar>?
93
94 +@_silgen_name("onde_uqff_model_config")
95 +func onde_uqff_model_config(
96 + _ modelId: UnsafePointer<CChar>,
97 + _ filesJson: UnsafePointer<CChar>,
98 + _ displayName: UnsafePointer<CChar>,
99 + _ approxMemory: UnsafePointer<CChar>,
100 + _ chatTemplate: UnsafePointer<CChar>?
101 +) -> UnsafeMutablePointer<CChar>?
102 +
103 @_silgen_name("onde_qwen25_1_5b_config")
104 func onde_qwen25_1_5b_config() -> UnsafeMutablePointer<CChar>?
105
@@ -290,6 +307,46 @@ public class OndeInferenceModule: Module {
307 return try self.validateJsonResult(result)
308 }
309
310 + Function("uqffModelConfig") {
311 + (
312 + modelId: String,
313 + filesJson: String,
314 + displayName: String,
315 + approxMemory: String,
316 + chatTemplate: String?
317 + ) -> String in
318 + let result = modelId.withCString { cModelId in
319 + filesJson.withCString { cFilesJson in
320 + displayName.withCString { cDisplayName in
321 + approxMemory.withCString { cApproxMemory in
322 + if let chatTemplate = chatTemplate {
323 + return chatTemplate.withCString { cChatTemplate in
324 + self.consumeRustString(
325 + onde_uqff_model_config(
326 + cModelId,
327 + cFilesJson,
328 + cDisplayName,
329 + cApproxMemory,
330 + cChatTemplate
331 + ))
332 + }
333 + } else {
334 + return self.consumeRustString(
335 + onde_uqff_model_config(
336 + cModelId,
337 + cFilesJson,
338 + cDisplayName,
339 + cApproxMemory,
340 + nil
341 + ))
342 + }
343 + }
344 + }
345 + }
346 + }
347 + return try self.validateJsonResult(result)
348 + }
349 +
350 Function("qwen251_5bConfig") { () -> String in
351 let result = self.consumeRustString(onde_qwen25_1_5b_config())
352 return try self.validateJsonResult(result)
@@ -411,6 +468,36 @@ public class OndeInferenceModule: Module {
468 return try self.validateJsonResult(result)
469 }
470
471 + AsyncFunction("loadUqffModel") {
472 + (configJson: String, systemPrompt: String?, samplingJson: String?) -> String in
473 + try self.configureApplicationFilesystem()
474 + guard let ptr = self.enginePtr else {
475 + throw OndeError.engineNotInitialized
476 + }
477 + let result: String? = configJson.withCString { cConfig in
478 + if let sp = systemPrompt, let sj = samplingJson {
479 + return sp.withCString { cSp in
480 + sj.withCString { cSj in
481 + self.consumeRustString(
482 + onde_engine_load_uqff_model(ptr, cConfig, cSp, cSj))
483 + }
484 + }
485 + } else if let sp = systemPrompt {
486 + return sp.withCString { cSp in
487 + self.consumeRustString(onde_engine_load_uqff_model(ptr, cConfig, cSp, nil))
488 + }
489 + } else if let sj = samplingJson {
490 + return sj.withCString { cSj in
491 + self.consumeRustString(onde_engine_load_uqff_model(ptr, cConfig, nil, cSj))
492 + }
493 + } else {
494 + return self.consumeRustString(
495 + onde_engine_load_uqff_model(ptr, cConfig, nil, nil))
496 + }
497 + }
498 + return try self.validateJsonResult(result)
499 + }
500 +
501 AsyncFunction("unloadModel") { () -> String in
502 guard let ptr = self.enginePtr else {
503 throw OndeError.engineNotInitialized
sdk/react-native/rust/src/lib.rs
+153 -1
@@ -23,7 +23,7 @@ use serde::Serialize;
23 use tokio::runtime::Runtime;
24
25 use onde::inference::engine::ChatEngine;
26 -use onde::inference::types::{ChatMessage, GgufModelConfig, SamplingConfig};
26 +use onde::inference::types::{ChatMessage, GgufModelConfig, SamplingConfig, UqffModelConfig};
27
28 // ── Global Tokio runtime ─────────────────────────────────────────────────────
29
@@ -193,6 +193,47 @@ pub extern "C" fn onde_engine_load_model(
193 })
194 }
195
196 +/// Load a specific UQFF model from a JSON-encoded [`UqffModelConfig`].
197 +///
198 +/// # Parameters
199 +///
200 +/// - `config_json` — JSON-encoded `UqffModelConfig`.
201 +/// - `system_prompt` — optional C string (pass null to omit).
202 +/// - `sampling_json` — optional JSON-encoded `SamplingConfig` (pass null for defaults).
203 +///
204 +/// # Returns
205 +///
206 +/// Same JSON shape as [`onde_engine_load_default_model`].
207 +#[no_mangle]
208 +pub extern "C" fn onde_engine_load_uqff_model(
209 + engine: *mut c_void,
210 + config_json: *const c_char,
211 + system_prompt: *const c_char,
212 + sampling_json: *const c_char,
213 +) -> *mut c_char {
214 + if engine.is_null() {
215 + return error_json("engine pointer is null");
216 + }
217 + let engine_ref = unsafe { &*(engine as *const ChatEngine) };
218 +
219 + let config: UqffModelConfig = match from_json_cstr(config_json) {
220 + Some(c) => c,
221 + None => return error_json("invalid or null config_json"),
222 + };
223 + let prompt = nullable_str(system_prompt);
224 + let sampling: Option<SamplingConfig> = from_json_cstr(sampling_json);
225 +
226 + runtime().block_on(async {
227 + match engine_ref.load_uqff_model(config, prompt, sampling).await {
228 + Ok(duration) => {
229 + let result = serde_json::json!({ "elapsed_secs": duration.as_secs_f64() });
230 + to_json_cstring(&result)
231 + }
232 + Err(err) => error_json(&err.to_string()),
233 + }
234 + })
235 +}
236 +
237 /// Unload the currently loaded model.
238 ///
239 /// # Returns
@@ -461,6 +502,41 @@ pub extern "C" fn onde_default_model_config() -> *mut c_char {
502 to_json_cstring(&GgufModelConfig::platform_default())
503 }
504
505 +/// Build a generic UQFF model config as JSON.
506 +#[no_mangle]
507 +pub extern "C" fn onde_uqff_model_config(
508 + model_id: *const c_char,
509 + files_json: *const c_char,
510 + display_name: *const c_char,
511 + approx_memory: *const c_char,
512 + chat_template: *const c_char,
513 +) -> *mut c_char {
514 + let model_id = match nullable_str(model_id) {
515 + Some(value) => value,
516 + None => return error_json("model_id is required"),
517 + };
518 + let files: Vec<String> = match from_json_cstr(files_json) {
519 + Some(value) => value,
520 + None => return error_json("invalid or null files_json"),
521 + };
522 + let display_name = match nullable_str(display_name) {
523 + Some(value) => value,
524 + None => return error_json("display_name is required"),
525 + };
526 + let approx_memory = match nullable_str(approx_memory) {
527 + Some(value) => value,
528 + None => return error_json("approx_memory is required"),
529 + };
530 +
531 + to_json_cstring(&UqffModelConfig {
532 + model_id,
533 + files,
534 + display_name,
535 + approx_memory,
536 + chat_template: nullable_str(chat_template),
537 + })
538 +}
539 +
540 /// Return the Qwen 2.5 1.5B GGUF model config as JSON.
541 #[no_mangle]
542 pub extern "C" fn onde_qwen25_1_5b_config() -> *mut c_char {
@@ -690,6 +766,41 @@ mod android {
766 cstring_ptr_to_jstring(&mut env, result)
767 }
768
769 + #[no_mangle]
770 + pub extern "system" fn Java_com_ondeinference_OndeInferenceModule_nativeEngineLoadUqffModel(
771 + mut env: JNIEnv,
772 + _class: JClass,
773 + engine: jlong,
774 + config_json: JString,
775 + system_prompt: JString,
776 + sampling_json: JString,
777 + ) -> jstring {
778 + let config_cstr = jstring_to_cstring(&mut env, &config_json);
779 + let prompt_cstr = jstring_to_cstring(&mut env, &system_prompt);
780 + let sampling_cstr = jstring_to_cstring(&mut env, &sampling_json);
781 +
782 + let config_ptr = config_cstr
783 + .as_ref()
784 + .map(|c| c.as_ptr())
785 + .unwrap_or(std::ptr::null());
786 + let prompt_ptr = prompt_cstr
787 + .as_ref()
788 + .map(|c| c.as_ptr())
789 + .unwrap_or(std::ptr::null());
790 + let sampling_ptr = sampling_cstr
791 + .as_ref()
792 + .map(|c| c.as_ptr())
793 + .unwrap_or(std::ptr::null());
794 +
795 + let result = onde_engine_load_uqff_model(
796 + engine as *mut c_void,
797 + config_ptr,
798 + prompt_ptr,
799 + sampling_ptr,
800 + );
801 + cstring_ptr_to_jstring(&mut env, result)
802 + }
803 +
804 #[no_mangle]
805 pub extern "system" fn Java_com_ondeinference_OndeInferenceModule_nativeEngineUnloadModel(
806 mut env: JNIEnv,
@@ -857,6 +968,47 @@ mod android {
968 cstring_ptr_to_jstring(&mut env, result)
969 }
970
971 + #[no_mangle]
972 + pub extern "system" fn Java_com_ondeinference_OndeInferenceModule_nativeUqffModelConfig(
973 + mut env: JNIEnv,
974 + _class: JClass,
975 + model_id: JString,
976 + files_json: JString,
977 + display_name: JString,
978 + approx_memory: JString,
979 + chat_template: JString,
980 + ) -> jstring {
981 + let model_id_cstr = jstring_to_cstring(&mut env, &model_id);
982 + let files_cstr = jstring_to_cstring(&mut env, &files_json);
983 + let display_name_cstr = jstring_to_cstring(&mut env, &display_name);
984 + let approx_memory_cstr = jstring_to_cstring(&mut env, &approx_memory);
985 + let chat_template_cstr = jstring_to_cstring(&mut env, &chat_template);
986 +
987 + let result = onde_uqff_model_config(
988 + model_id_cstr
989 + .as_ref()
990 + .map(|c| c.as_ptr())
991 + .unwrap_or(std::ptr::null()),
992 + files_cstr
993 + .as_ref()
994 + .map(|c| c.as_ptr())
995 + .unwrap_or(std::ptr::null()),
996 + display_name_cstr
997 + .as_ref()
998 + .map(|c| c.as_ptr())
999 + .unwrap_or(std::ptr::null()),
1000 + approx_memory_cstr
1001 + .as_ref()
1002 + .map(|c| c.as_ptr())
1003 + .unwrap_or(std::ptr::null()),
1004 + chat_template_cstr
1005 + .as_ref()
1006 + .map(|c| c.as_ptr())
1007 + .unwrap_or(std::ptr::null()),
1008 + );
1009 + cstring_ptr_to_jstring(&mut env, result)
1010 + }
1011 +
1012 #[no_mangle]
1013 pub extern "system" fn Java_com_ondeinference_OndeInferenceModule_nativeQwen251_15bConfig(
1014 mut env: JNIEnv,
sdk/react-native/src/OndeInferenceModule.ts
+12
@@ -35,6 +35,11 @@ export interface OndeInferenceNativeModule {
35 systemPrompt: string | null,
36 samplingJson: string | null,
37 ): Promise<string>;
38 + loadUqffModel(
39 + configJson: string,
40 + systemPrompt: string | null,
41 + samplingJson: string | null,
42 + ): Promise<string>;
43 unloadModel(): Promise<string>;
44 info(): Promise<string>;
45 history(): Promise<string>;
@@ -43,6 +48,13 @@ export interface OndeInferenceNativeModule {
48
49 // Config functions (sync, return JSON strings)
50 defaultModelConfig(): string;
51 + uqffModelConfig(
52 + modelId: string,
53 + filesJson: string,
54 + displayName: string,
55 + approxMemory: string,
56 + chatTemplate: string | null,
57 + ): string;
58 qwen251_5bConfig(): string;
59 qwen253bConfig(): string;
60 qwen3_0_6bConfig(): string;
sdk/react-native/src/index.ts
+72
@@ -11,6 +11,7 @@ import type {
11 SamplingConfig,
12 StreamChunk,
13 ToolCallInfo,
14 + UqffModelConfig,
15 } from "./types";
16 import { OndeError } from "./types";
17
@@ -25,6 +26,7 @@ export type {
26 SamplingConfig,
27 StreamChunk,
28 ToolCallInfo,
29 + UqffModelConfig,
30 } from "./types";
31 export { OndeError } from "./types";
32
@@ -55,6 +57,17 @@ function serializeModelConfig(config: GgufModelConfig): string {
57 });
58 }
59
60 +// Convert UqffModelConfig from camelCase TS to snake_case Rust
61 +function serializeUqffModelConfig(config: UqffModelConfig): string {
62 + return JSON.stringify({
63 + model_id: config.modelId,
64 + files: config.files,
65 + display_name: config.displayName,
66 + approx_memory: config.approxMemory,
67 + chat_template: config.chatTemplate,
68 + });
69 +}
70 +
71 // Parse a JSON result string, throwing OndeError if it contains an error field
72 function parseResult<T>(json: string): T {
73 const parsed = JSON.parse(json);
@@ -117,6 +130,23 @@ function parseModelConfig(json: string): GgufModelConfig {
130 };
131 }
132
133 +function parseUqffModelConfig(json: string): UqffModelConfig {
134 + const raw = parseResult<{
135 + model_id: string;
136 + files: string[];
137 + display_name: string;
138 + approx_memory: string;
139 + chat_template?: string;
140 + }>(json);
141 + return {
142 + modelId: raw.model_id,
143 + files: raw.files,
144 + displayName: raw.display_name,
145 + approxMemory: raw.approx_memory,
146 + chatTemplate: raw.chat_template,
147 + };
148 +}
149 +
150 function parseSamplingConfig(json: string): SamplingConfig {
151 const raw = JSON.parse(json);
152 return {
@@ -221,6 +251,29 @@ export const OndeChatEngine = {
251 return result.elapsed_secs;
252 },
253
254 + /**
255 + * Load a specific UQFF model.
256 + *
257 + * `config.modelId` is the base repo/local directory used for tokenizer and
258 + * config resolution. `config.files` accepts the first shard, explicit
259 + * shards, or mistral.rs shorthands such as `"q4k"` and `"4"`.
260 + *
261 + * @returns Wall-clock loading time in seconds.
262 + */
263 + async loadUqffModel(
264 + config: UqffModelConfig,
265 + systemPrompt?: string,
266 + sampling?: SamplingConfig,
267 + ): Promise<number> {
268 + const json = await OndeInferenceModule.loadUqffModel(
269 + serializeUqffModelConfig(config),
270 + systemPrompt ?? null,
271 + sampling ? serializeSampling(sampling) : null,
272 + );
273 + const result = parseResult<{ elapsed_secs: number }>(json);
274 + return result.elapsed_secs;
275 + },
276 +
277 /** Unload the current model, freeing all memory. */
278 async unloadModel(): Promise<string | null> {
279 const json = await OndeInferenceModule.unloadModel();
@@ -311,6 +364,25 @@ export function defaultModelConfig(): GgufModelConfig {
364 return parseModelConfig(OndeInferenceModule.defaultModelConfig());
365 }
366
367 +/** Create a generic UQFF model config. */
368 +export function uqffModelConfig(
369 + modelId: string,
370 + files: string[],
371 + displayName: string,
372 + approxMemory: string,
373 + chatTemplate?: string,
374 +): UqffModelConfig {
375 + return parseUqffModelConfig(
376 + OndeInferenceModule.uqffModelConfig(
377 + modelId,
378 + JSON.stringify(files),
379 + displayName,
380 + approxMemory,
381 + chatTemplate ?? null,
382 + ),
383 + );
384 +}
385 +
386 /** Return the Qwen 2.5 1.5B config (~941 MB). */
387 export function qwen251_5bConfig(): GgufModelConfig {
388 return parseModelConfig(OndeInferenceModule.qwen251_5bConfig());
sdk/react-native/src/types.ts
+14
@@ -52,6 +52,20 @@ export interface GgufModelConfig {
52 chatTemplate?: string;
53 }
54
55 +/** Configuration for loading a UQFF model. */
56 +export interface UqffModelConfig {
57 + /** HuggingFace base model repository ID or local model directory. */
58 + modelId: string;
59 + /** UQFF shard filename(s), first shard, or shorthand such as "q4k" / "4". */
60 + files: string[];
61 + /** Human-friendly display name. */
62 + displayName: string;
63 + /** Approximate memory footprint description. */
64 + approxMemory: string;
65 + /** Optional Jinja chat template override. */
66 + chatTemplate?: string;
67 +}
68 +
69 // ── Inference result ─────────────────────────────────────────────────────────
70
71 /** Result of a completed inference request. */
src/inference/engine.rs
+204
@@ -101,6 +101,7 @@ use super::types::*;
101 use mistralrs::{
102 CalledFunction, Function, GgufModelBuilder, Model, RequestBuilder, TextMessageRole,
103 Tool as MistralTool, ToolCallResponse, ToolCallType, ToolChoice, ToolType,
104 + UqffTextModelBuilder,
105 };
106
107 #[cfg(any(
@@ -114,6 +115,17 @@ use mistralrs::{
115 target_os = "android"
116 ))]
117 use std::collections::HashMap;
118 +#[cfg(any(
119 + target_os = "macos",
120 + target_os = "ios",
121 + target_os = "tvos",
122 + target_os = "visionos",
123 + target_os = "watchos",
124 + target_os = "windows",
125 + target_os = "linux",
126 + target_os = "android"
127 +))]
128 +use std::path::PathBuf;
129
130 // ISQ types are only used by load_isq_model, which is macOS-only.
131 #[cfg(target_os = "macos")]
@@ -134,6 +146,7 @@ use mistralrs::{IsqBits, TextModelBuilder};
146 ))]
147 enum LoadedModelConfig {
148 Gguf(GgufModelConfig),
149 + Uqff(UqffModelConfig),
150 #[cfg(target_os = "macos")]
151 Isq(IsqModelConfig),
152 }
@@ -152,6 +165,7 @@ impl LoadedModelConfig {
165 fn display_name(&self) -> &str {
166 match self {
167 LoadedModelConfig::Gguf(c) => &c.display_name,
168 + LoadedModelConfig::Uqff(c) => &c.display_name,
169 #[cfg(target_os = "macos")]
170 LoadedModelConfig::Isq(c) => &c.display_name,
171 }
@@ -160,6 +174,7 @@ impl LoadedModelConfig {
174 fn approx_memory(&self) -> &str {
175 match self {
176 LoadedModelConfig::Gguf(c) => &c.approx_memory,
177 + LoadedModelConfig::Uqff(c) => &c.approx_memory,
178 #[cfg(target_os = "macos")]
179 LoadedModelConfig::Isq(c) => &c.approx_memory,
180 }
@@ -521,6 +536,158 @@ impl ChatEngine {
536 Ok(elapsed)
537 }
538
539 + /// Load a UQFF model into the engine.
540 + ///
541 + /// UQFF stores pre-quantised weights and loads directly
542 + /// without runtime conversion. `config.model_id` is still required for
543 + /// tokenizer and base model config resolution; `config.files` contains the
544 + /// first shard or explicit UQFF shard filenames.
545 + ///
546 + /// If a model is already loaded it will be unloaded first.
547 + ///
548 + /// # Errors
549 + ///
550 + /// Returns [`InferenceError::ModelBuild`] if the model fails to download
551 + /// or load.
552 + pub async fn load_uqff_model(
553 + &self,
554 + config: UqffModelConfig,
555 + system_prompt: Option<String>,
556 + sampling: Option<SamplingConfig>,
557 + ) -> Result<std::time::Duration, InferenceError> {
558 + use log::info;
559 + use std::time::Instant;
560 +
561 + if config.files.is_empty() {
562 + return Err(InferenceError::ModelBuild {
563 + reason: "UQFF model config must include at least one UQFF file or shorthand."
564 + .to_string(),
565 + });
566 + }
567 +
568 + info!(
569 + "ChatEngine: loading UQFF model {} (files: {:?})",
570 + config.model_id, config.files
571 + );
572 +
573 + #[cfg(any(
574 + target_os = "android",
575 + target_os = "ios",
576 + target_os = "tvos",
577 + target_os = "visionos",
578 + target_os = "watchos"
579 + ))]
580 + {
581 + let hf_home = std::env::var("HF_HOME")
582 + .map(std::path::PathBuf::from)
583 + .map_err(|_| InferenceError::ModelBuild {
584 + reason: "HF_HOME is not set — cannot initialise HF cache. \
585 + On iOS/tvOS/Android, call configure_cache_dir() or \
586 + download_model(app_data_dir:) before load_uqff_model()."
587 + .to_string(),
588 + })?;
589 + let hf_hub_cache = hf_home.join("hub");
590 + if let Err(e) = std::fs::create_dir_all(&hf_hub_cache) {
591 + log::warn!(
592 + "ChatEngine: could not create HF hub cache dir {}: {}",
593 + hf_hub_cache.display(),
594 + e
595 + );
596 + }
597 + mistralrs_core::GLOBAL_HF_CACHE
598 + .get_or_init(|| hf_hub::Cache::new(hf_hub_cache.clone()));
599 + std::env::set_var("HF_HUB_CACHE", &hf_hub_cache);
600 + log::debug!(
601 + "ChatEngine: GLOBAL_HF_CACHE seeded at {}",
602 + hf_hub_cache.display()
603 + );
604 + }
605 +
606 + crate::hf_cache::clean_stale_lock_files(&config.model_id);
607 + crate::hf_cache::repair_hf_cache_symlinks(&config.model_id);
608 +
609 + let start = Instant::now();
610 + let uqff_files: Vec<PathBuf> = config.files.iter().map(PathBuf::from).collect();
611 +
612 + let mut builder = UqffTextModelBuilder::new(&config.model_id, uqff_files)
613 + .into_inner()
614 + .with_token_source(super::token::hf_token_source())
615 + .with_logging();
616 +
617 + #[cfg(target_os = "tvos")]
618 + {
619 + use mistralrs::DeviceMapSetting;
620 + use mistralrs_core::AutoDeviceMapParams;
621 +
622 + builder = builder
623 + .with_max_num_seqs(1)
624 + .with_prefix_cache_n(None)
625 + .with_no_kv_cache()
626 + .with_device_mapping(DeviceMapSetting::Auto(AutoDeviceMapParams::Text {
627 + max_seq_len: 512,
628 + max_batch_size: 1,
629 + }));
630 + }
631 +
632 + if let Some(ref template) = config.chat_template {
633 + builder = builder.with_chat_template(template);
634 + }
635 +
636 + let model = builder
637 + .build()
638 + .await
639 + .map_err(|e| InferenceError::ModelBuild {
640 + reason: format!("Failed to build UQFF model {}: {}", config.display_name, e),
641 + })?;
642 +
643 + let elapsed = start.elapsed();
644 +
645 + let sampling = sampling.unwrap_or_else(|| {
646 + if cfg!(any(
647 + target_os = "ios",
648 + target_os = "tvos",
649 + target_os = "visionos",
650 + target_os = "watchos",
651 + target_os = "android"
652 + )) {
653 + SamplingConfig::mobile()
654 + } else {
655 + SamplingConfig::default()
656 + }
657 + });
658 +
659 + info!(
660 + "ChatEngine: UQFF model {} loaded in {} (sampling: temp={:?}, max_tokens={:?})",
661 + config.display_name,
662 + format_duration(elapsed),
663 + sampling.temperature,
664 + sampling.max_tokens,
665 + );
666 +
667 + let pulse_model_id = config.model_id.clone();
668 + let pulse_model_name = config.display_name.clone();
669 +
670 + let old_model = {
671 + let mut guard = self.inner.lock().await;
672 + guard.replace(LoadedModel {
673 + model: Arc::new(model),
674 + config: LoadedModelConfig::Uqff(config),
675 + history: Vec::new(),
676 + system_prompt,
677 + sampling,
678 + })
679 + };
680 + drop(old_model);
681 +
682 + if let Some(pulse) = self.pulse() {
683 + pulse
684 + .record_model_loaded(pulse_model_id, pulse_model_name, elapsed.as_millis() as u64)
685 + .await;
686 + }
687 +
688 + Ok(elapsed)
689 + }
690 +
691 /// Fetch the model assigned to this Onde app from the SDK model-config
692 /// endpoint and load it.
693 ///
@@ -873,6 +1040,7 @@ impl ChatEngine {
1040 let request = self::build_request(loaded, &user_message);
1041 let pulse_model_id = match &loaded.config {
1042 LoadedModelConfig::Gguf(c) => c.model_id.clone(),
1043 + LoadedModelConfig::Uqff(c) => c.model_id.clone(),
1044 #[cfg(target_os = "macos")]
1045 LoadedModelConfig::Isq(c) => c.model_id.clone(),
1046 };
@@ -1824,6 +1992,17 @@ impl ChatEngine {
1992 })
1993 }
1994
1995 + pub async fn load_uqff_model(
1996 + &self,
1997 + _config: UqffModelConfig,
1998 + _system_prompt: Option<String>,
1999 + _sampling: Option<SamplingConfig>,
2000 + ) -> Result<std::time::Duration, InferenceError> {
2001 + Err(InferenceError::Other {
2002 + reason: "LLM inference is not supported on this platform.".into(),
2003 + })
2004 + }
2005 +
2006 pub async fn unload_model(&self) -> Option<String> {
2007 None
2008 }
@@ -2426,6 +2605,31 @@ mod tests {
2605 }
2606 }
2607
2608 + #[tokio::test]
2609 + async fn engine_load_uqff_without_files_errors() {
2610 + let engine = ChatEngine::new();
2611 + let result = engine
2612 + .load_uqff_model(
2613 + UqffModelConfig {
2614 + model_id: "google/gemma-4-E4B-it".into(),
2615 + files: Vec::new(),
2616 + display_name: "Gemma 4 E4B (UQFF Q4K)".into(),
2617 + approx_memory: "~2.5 GB (UQFF Q4K)".into(),
2618 + chat_template: None,
2619 + },
2620 + None,
2621 + None,
2622 + )
2623 + .await;
2624 +
2625 + match result.unwrap_err() {
2626 + InferenceError::ModelBuild { reason } => {
2627 + assert!(reason.contains("at least one UQFF file"));
2628 + }
2629 + other => panic!("Expected ModelBuild, got: {:?}", other),
2630 + }
2631 + }
2632 +
2633 #[tokio::test]
2634 async fn engine_history_empty_when_no_model() {
2635 let engine = ChatEngine::new();
src/inference/ffi.rs
+43
@@ -119,6 +119,27 @@ impl OndeChatEngine {
119 Ok(elapsed.as_secs_f64())
120 }
121
122 + /// Load a UQFF model into the engine.
123 + ///
124 + /// UQFF files are pre-quantised and load directly without runtime
125 + /// conversion. `config.model_id` is required for tokenizer/base model
126 + /// resolution, and `config.files` should contain the first UQFF shard or
127 + /// explicit shard filenames.
128 + ///
129 + /// Returns the wall-clock loading time in seconds.
130 + pub async fn load_uqff_model(
131 + &self,
132 + config: UqffModelConfig,
133 + system_prompt: Option<String>,
134 + sampling: Option<SamplingConfig>,
135 + ) -> Result<f64, InferenceError> {
136 + let elapsed = self
137 + .inner
138 + .load_uqff_model(config, system_prompt, sampling)
139 + .await?;
140 + Ok(elapsed.as_secs_f64())
141 + }
142 +
143 /// Load the platform-appropriate default model.
144 ///
145 /// - tvOS / iOS / Android → Qwen 2.5 1.5B (~941 MB)
@@ -308,6 +329,28 @@ pub fn default_model_config() -> GgufModelConfig {
329 GgufModelConfig::platform_default()
330 }
331
332 +/// Create a generic UQFF model configuration.
333 +///
334 +/// `model_id` is the base HuggingFace repo or local model directory used for
335 +/// tokenizer/config resolution. `files` accepts a first shard (`q4k-0.uqff`),
336 +/// explicit shards, or mistral.rs shorthands such as `q4k` and `4`.
337 +#[uniffi::export]
338 +pub fn uqff_model_config(
339 + model_id: String,
340 + files: Vec<String>,
341 + display_name: String,
342 + approx_memory: String,
343 + chat_template: Option<String>,
344 +) -> UqffModelConfig {
345 + UqffModelConfig {
346 + model_id,
347 + files,
348 + display_name,
349 + approx_memory,
350 + chat_template,
351 + }
352 +}
353 +
354 /// Return the Qwen 2.5 0.5B GGUF model configuration (~380 MB).
355 #[uniffi::export]
356 pub fn qwen25_0_5b_config() -> GgufModelConfig {
src/inference/mod.rs
+1
@@ -83,4 +83,5 @@ pub use types::{
83 ToolCallRequest,
84 ToolDefinition,
85 ToolResult,
86 + UqffModelConfig,
87 };
src/inference/types.rs
+43
@@ -221,6 +221,35 @@ pub struct GgufModelConfig {
221 pub chat_template: Option<String>,
222 }
223
224 +// ── UQFF model configuration ─────────────────────────────────────────────────
225 +
226 +/// Configuration for loading a UQFF model via mistral.rs
227 +/// `UqffTextModelBuilder`.
228 +///
229 +/// UQFF files store pre-quantised weights and load directly
230 +/// without the runtime memory spike of in-situ quantisation. `model_id` is the
231 +/// base HuggingFace repository or local model directory used for tokenizer and
232 +/// config resolution, while `files` identifies the first UQFF shard or explicit
233 +/// UQFF shard files.
234 +#[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
235 +pub struct UqffModelConfig {
236 + /// HuggingFace base model repository ID or local model directory, e.g.
237 + /// `"google/gemma-4-E4B-it"`.
238 + pub model_id: String,
239 + /// UQFF filename(s) within the repository. For sharded UQFFs, passing the
240 + /// first shard, e.g. `["q4k-0.uqff"]`, is enough for mistral.rs to discover
241 + /// siblings. Numeric and ISQ-name shorthands such as `"4"` and `"q4k"` are
242 + /// also accepted by the underlying loader.
243 + pub files: Vec<String>,
244 + /// Human-friendly display name, e.g. `"Gemma 4 E4B (UQFF Q4K)"`.
245 + pub display_name: String,
246 + /// Approximate memory footprint description, e.g. `"~2.5 GB (UQFF Q4K)"`.
247 + pub approx_memory: String,
248 + /// Optional Jinja chat template string. When set, this overrides the
249 + /// template resolved from the base model.
250 + pub chat_template: Option<String>,
251 +}
252 +
253 // ── ISQ model configuration ──────────────────────────────────────────────────
254
255 /// Configuration for loading an ISQ (in-situ quantised) model via mistral.rs
@@ -523,6 +552,20 @@ mod tests {
552 assert_eq!(config.max_tokens, Some(4096));
553 }
554
555 + #[test]
556 + fn uqff_model_config_accepts_first_shard() {
557 + let config = UqffModelConfig {
558 + model_id: "google/gemma-4-E4B-it".into(),
559 + files: vec!["q4k-0.uqff".into()],
560 + display_name: "Gemma 4 E4B (UQFF Q4K)".into(),
561 + approx_memory: "~2.5 GB (UQFF Q4K)".into(),
562 + chat_template: None,
563 + };
564 +
565 + assert_eq!(config.model_id, "google/gemma-4-E4B-it");
566 + assert_eq!(config.files, vec!["q4k-0.uqff"]);
567 + }
568 +
569 #[test]
570 fn engine_status_display() {
571 assert_eq!(EngineStatus::Ready.to_string(), "ready");