|
1
|
// Copyright 2024 Onde Inference. All rights reserved. |
|
2
|
// Use of this source code is governed by the MIT license that can be found in |
|
3
|
// the LICENSE file. |
|
4
|
|
|
5
|
/// On-device LLM inference SDK for Flutter & Dart. |
|
6
|
/// |
|
7
|
/// Runs Qwen models locally with Metal on Apple platforms and CPU inference on |
|
8
|
/// Android, Linux, and Windows. No cloud hop and no user data leaving the |
|
9
|
/// device. Powered by the Onde Rust engine and |
|
10
|
/// [mistral.rs](https://github.com/EricLBuehler/mistral.rs). |
|
11
|
/// |
|
12
|
/// ## Quick start |
|
13
|
/// |
|
14
|
/// ```dart |
|
15
|
/// import 'package:flutter/widgets.dart'; |
|
16
|
/// import 'package:onde_inference/onde_inference.dart'; |
|
17
|
/// |
|
18
|
/// Future<void> main() async { |
|
19
|
/// WidgetsFlutterBinding.ensureInitialized(); |
|
20
|
/// await OndeInference.init(); |
|
21
|
/// |
|
22
|
/// final engine = OndeChatEngine(); |
|
23
|
/// await engine.loadDefaultModel( |
|
24
|
/// systemPrompt: 'You are a helpful assistant.', |
|
25
|
/// ); |
|
26
|
/// |
|
27
|
/// final result = await engine.sendMessage(message: 'Hello!'); |
|
28
|
/// print(result.text); |
|
29
|
/// |
|
30
|
/// final buffer = StringBuffer(); |
|
31
|
/// await for (final chunk in engine.streamMessage(message: 'Tell me a short story.')) { |
|
32
|
/// buffer.write(chunk.delta); |
|
33
|
/// if (chunk.done) break; |
|
34
|
/// } |
|
35
|
/// print(buffer.toString()); |
|
36
|
/// |
|
37
|
/// await engine.unloadModel(); |
|
38
|
/// } |
|
39
|
/// ``` |
|
40
|
/// |
|
41
|
/// ## Selecting a model |
|
42
|
/// |
|
43
|
/// ```dart |
|
44
|
/// final config = OndeInference.defaultModelConfig(); |
|
45
|
/// final coderConfig = OndeInference.qwen25Coder3bConfig(); |
|
46
|
/// |
|
47
|
/// await engine.loadGgufModel(config: coderConfig); |
|
48
|
/// ``` |
|
49
|
/// |
|
50
|
/// ## Customising sampling |
|
51
|
/// |
|
52
|
/// ```dart |
|
53
|
/// await engine.setSampling( |
|
54
|
/// sampling: OndeInference.deterministicSamplingConfig(), |
|
55
|
/// ); |
|
56
|
/// |
|
57
|
/// await engine.loadDefaultModel( |
|
58
|
/// sampling: SamplingConfig( |
|
59
|
/// temperature: 0.5, |
|
60
|
/// maxTokens: BigInt.from(256), |
|
61
|
/// ), |
|
62
|
/// ); |
|
63
|
/// ``` |
|
64
|
/// |
|
65
|
/// ## Error handling |
|
66
|
/// |
|
67
|
/// The generated bridge throws [OndeError] values directly: |
|
68
|
/// |
|
69
|
/// ```dart |
|
70
|
/// try { |
|
71
|
/// await engine.sendMessage(message: '...'); |
|
72
|
/// } on OndeError catch (e) { |
|
73
|
/// debugPrint('Inference error: $e'); |
|
74
|
/// } |
|
75
|
/// ``` |
|
76
|
/// |
|
77
|
/// See the package README and the example app for platform-specific setup, |
|
78
|
/// cache configuration, and end-to-end Flutter integration. |
|
79
|
library; |
|
80
|
|
|
81
|
// Re-export the core data types (ChatMessage, SamplingConfig, |
|
82
|
// GgufModelConfig, and friends) from the FRB-generated api.dart via types.dart. |
|
83
|
export 'src/types.dart'; |
|
84
|
|
|
85
|
// Engine API: OndeChatEngineX helpers, OndeInference static helpers, |
|
86
|
// the OndeChatEngine opaque type, the OndeError sealed class, and RustLib. |
|
87
|
export 'src/engine.dart'; |