|
1
|
// Copyright 2024 Onde Inference. All rights reserved. |
|
2
|
// Use of this source code is governed by the MIT license that can be found in |
|
3
|
// the LICENSE file. |
|
4
|
|
|
5
|
/// On-device LLM inference SDK for Flutter & Dart. |
|
6
|
/// |
|
7
|
/// Runs Qwen 2.5 models locally with Metal (Apple silicon) and CPU |
|
8
|
/// acceleration — no cloud, no data leaving the device. |
|
9
|
/// Powered by the Onde Rust engine and [mistral.rs](https://github.com/EricLBuehler/mistral.rs). |
|
10
|
/// |
|
11
|
/// ## Quick start |
|
12
|
/// |
|
13
|
/// ```dart |
|
14
|
/// import 'package:onde_inference/onde_inference.dart'; |
|
15
|
/// |
|
16
|
/// Future<void> main() async { |
|
17
|
/// // 1. Initialise the Rust library once at startup. |
|
18
|
/// await OndeInference.init(); |
|
19
|
/// |
|
20
|
/// // 2. Create an engine and load the platform-appropriate default model. |
|
21
|
/// final engine = await OndeChatEngine.create(); |
|
22
|
/// await engine.loadDefaultModel( |
|
23
|
/// systemPrompt: 'You are a helpful assistant.', |
|
24
|
/// ); |
|
25
|
/// |
|
26
|
/// // 3. Single-turn completion. |
|
27
|
/// final result = await engine.sendMessage('Hello!'); |
|
28
|
/// print(result.text); |
|
29
|
/// |
|
30
|
/// // 4. Streaming completion. |
|
31
|
/// final buffer = StringBuffer(); |
|
32
|
/// await for (final chunk in engine.streamMessage('Tell me a short story.')) { |
|
33
|
/// buffer.write(chunk.delta); |
|
34
|
/// if (chunk.done) break; |
|
35
|
/// } |
|
36
|
/// print(buffer.toString()); |
|
37
|
/// |
|
38
|
/// // 5. Release device memory when done. |
|
39
|
/// await engine.unloadModel(); |
|
40
|
/// } |
|
41
|
/// ``` |
|
42
|
/// |
|
43
|
/// ## Selecting a model |
|
44
|
/// |
|
45
|
/// ```dart |
|
46
|
/// // Platform-aware default (1.5B on mobile, 3B on desktop). |
|
47
|
/// final config = OndeInference.defaultModelConfig(); |
|
48
|
/// |
|
49
|
/// // Or pick a specific model. |
|
50
|
/// final coderConfig = OndeInference.qwen25Coder3bConfig(); |
|
51
|
/// |
|
52
|
/// await engine.loadGgufModel(coderConfig); |
|
53
|
/// ``` |
|
54
|
/// |
|
55
|
/// ## Customising sampling |
|
56
|
/// |
|
57
|
/// ```dart |
|
58
|
/// // Deterministic output for coding / fact-retrieval tasks. |
|
59
|
/// await engine.setSampling(OndeInference.deterministicSamplingConfig()); |
|
60
|
/// |
|
61
|
/// // Or pass sampling directly when loading a model. |
|
62
|
/// await engine.loadDefaultModel( |
|
63
|
/// sampling: SamplingConfig(temperature: 0.5, maxTokens: 256), |
|
64
|
/// ); |
|
65
|
/// ``` |
|
66
|
/// |
|
67
|
/// ## Error handling |
|
68
|
/// |
|
69
|
/// All methods throw [OndeException] on failure: |
|
70
|
/// |
|
71
|
/// ```dart |
|
72
|
/// try { |
|
73
|
/// await engine.sendMessage('...'); |
|
74
|
/// } on OndeException catch (e) { |
|
75
|
/// debugPrint('Inference error: ${e.message}'); |
|
76
|
/// } |
|
77
|
/// ``` |
|
78
|
/// |
|
79
|
/// ## Code generation (native bridge) |
|
80
|
/// |
|
81
|
/// The package ships with a compilation stub so it can be imported before the |
|
82
|
/// Rust binary is built. To generate the real FFI bridge, run from the package |
|
83
|
/// root: |
|
84
|
/// |
|
85
|
/// ```sh |
|
86
|
/// dart run flutter_rust_bridge_codegen generate |
|
87
|
/// ``` |
|
88
|
/// |
|
89
|
/// See the [README](https://github.com/ondeinference/onde) for full build |
|
90
|
/// instructions. |
|
91
|
library; |
|
92
|
|
|
93
|
// All core data types (ChatMessage, SamplingConfig, GgufModelConfig, etc.) |
|
94
|
// re-exported from the FRB-generated api.dart via types.dart. |
|
95
|
export 'src/types.dart'; |
|
96
|
|
|
97
|
// Engine API — OndeChatEngineX extension, OndeInference static helpers, |
|
98
|
// OndeChatEngine opaque type, OndeError sealed class, and RustLib. |
|
99
|
export 'src/engine.dart'; |