@ondeinference / onde / commits / 7751bf3

Move tvos_chkstk.s to scripts/, update references, and clean up docs

- Move tvos_chkstk.s into scripts/ and update all references in build.rs, .agents/AGENTS.md, and the repo layout - Update Cargo.toml description for clarity - Revise and simplify README.md for improved clarity and branding - Remove Mintlify docs/ directory and all Markdown docs (now hosted externally) - Fix conditional mistralrs imports for ISQ types (macOS only) in engine.rs - Add missing cfg to make_progress in hf_cache.rs

Seto Elkahfi committed Apr 2, 2026 at 17:17 UTC 7751bf397baf0d22f30be1175fd6b68c5ac4b5e6

14 files changed +36 -1655

.agents/AGENTS.md

+3 -2

@@ -45,7 +45,8 @@ onde/

45 ├── .cargo/config.toml # Target-specific rustflags (fp16, linker overrides)

46 ├── Cargo.toml # Platform-conditional mistralrs deps

47 ├── build.rs # tvOS ___chkstk_darwin assembly stub

48 -├── tvos_chkstk.s # No-op arm64 stub for missing tvOS symbol

48 +├── scripts/

49 +│ └── tvos_chkstk.s # No-op arm64 stub for missing tvOS symbol

50 ├── uniffi.toml # UniFFI binding config

51 └── docs/

52 ├── dev.md # Developer guide (build, architecture, platform table)

@@ -428,7 +429,7 @@ Tests that require model downloads are integration tests and not run by default

429

430 ### `___chkstk_darwin` linker error on tvOS

431

431 -`aws-lc-sys` (transitive via `reqwest → rustls → aws-lc-rs`) references `___chkstk_darwin`, a stack probing symbol that tvOS does not export. `build.rs` compiles `tvos_chkstk.s` (a no-op `ret` stub) via the `cc` crate to satisfy the linker. **Do not delete `tvos_chkstk.s`.**

432 +`aws-lc-sys` (transitive via `reqwest → rustls → aws-lc-rs`) references `___chkstk_darwin`, a stack probing symbol that tvOS does not export. `build.rs` compiles `scripts/tvos_chkstk.s` (a no-op `ret` stub) via the `cc` crate to satisfy the linker. **Do not delete `scripts/tvos_chkstk.s`.**

433

434 Affects: tvOS only. macOS, iOS, Android, Windows, Linux are unaffected.

435

Cargo.toml

+1 -1

@@ -2,7 +2,7 @@

2 name = "onde"

3 version = "0.1.0"

4 edition = "2021"

5 -description = "On-device chat inference for cross-platform apps with automatic model management."

5 +description = "On-device inference engine for Apple silicon."

6 license = "MIT"

7 repository = "https://github.com/ondeinference/onde"

8 homepage = "https://ondeinference.com"

README.md

+21 -29

@@ -1,45 +1,37 @@

1 -

2 -<h1>

1

4 - <img src="./assets/onde-inference-logo.svg" alt="Onde Inference Logo" width="320">

5 - Onde Inference

6 -</h1>

7 -

8 - On-device inference engine optimized for <a href="https://en.wikipedia.org/wiki/Apple_silicon" target="_blank">Apple silicon</a>.

9 -

10 -

11 - <a href="#in-production">In Production</a>

12 - ·

13 - <a href="https://ondeinference.com">Website</a>

14 - ·

15 - <a href="https://apps.apple.com/se/developer/splitfire-ab/id1831430993" target="_blank">App Store</a>

16 -

2 + <img src="./assets/onde-inference-logo.svg" alt="Onde Inference" width="96">

3

19 -## In production

5 +<h1 align="center">Onde Inference</h1>

21 -Onde already powers real apps on the App Store with on-device chat experiences.

7 +

8 + On-device LLM inference — optimized for <a href="https://en.wikipedia.org/wiki/Apple_silicon">Apple silicon</a>.

9 +

23 -### Chat completion

24 -- [Karokowe](https://apps.apple.com/se/app/karaoke-karokowe/id6756801861)

25 -- [Reimagen](https://apps.apple.com/se/app/reimagen-ai-musik-till-konst/id6757844075)

26 -- [GT8](https://apps.apple.com/se/app/gt8-kord-gitar-tutorial/id6756177644)

27 -- [Splitfire](https://apps.apple.com/se/app/splitfire-bassndrums/id6751143237)

28 -- [Rumi – Lär dig persiska](https://apps.apple.com/se/app/rumi-l%C3%A4r-dig-persiska/id6753832408)

11 +

12 + <a href="https://crates.io/crates/onde"><img src="https://img.shields.io/crates/v/onde?style=flat-square&color=235843&labelColor=17211D&label=crates.io" alt="crates.io"></a>

13 + <a href="https://ondeinference.com"><img src="https://img.shields.io/badge/ondeinference.com-235843?style=flat-square&labelColor=17211D" alt="Website"></a>

14 + <a href="https://apps.apple.com/se/developer/splitfire-ab/id1831430993"><img src="https://img.shields.io/badge/App%20Store-live-235843?style=flat-square&labelColor=17211D" alt="App Store"></a>

15 +

30 -### AI coding agent (WIP)

31 -- [Sigit](https://apps.apple.com/se/app/sigit-git-client-code-editor/id6753018849)

17 +

18 + <a href="#in-production">In Production</a> · <a href="https://ondeinference.com/sdk">SDK</a> · <a href="https://ondeinference.com">Website</a>

19 +

20 +

21 +---

22 +

23 +## In production

24 +

25 +Onde powers live App Store apps with fully on-device chat — no server, no latency, no data leaving the device.

27

28 <a href="https://apps.apple.com/se/developer/splitfire-ab/id1831430993" target="_blank">

35 - <img src="https://developer.apple.com/assets/elements/badges/download-on-the-app-store.svg" alt="Download on the App Store" height="48">

29 + <img src="https://developer.apple.com/assets/elements/badges/download-on-the-app-store.svg" alt="Download on the App Store" height="44">

30 </a>

31

33 ---

41 -## Copyright

42 -

35

44 - 2026 <a href="https://ondeinference.com">Onde Inference</a>

37

build.rs

+2 -2

@@ -19,7 +19,7 @@ fn main() {

20 if target_os == "tvos" {

21 let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set");

22 - let asm_path = std::path::Path::new(&manifest_dir).join("tvos_chkstk.s");

22 + let asm_path = std::path::Path::new(&manifest_dir).join("scripts/tvos_chkstk.s");

24 if asm_path.exists() {

25 // Use the `cc` crate to assemble the stub into a static library

@@ -31,7 +31,7 @@ fn main() {

31 // aws-lc-sys, ring, etc.), we use it directly.

32 cc::Build::new().file(&asm_path).compile("tvos_chkstk");

34 - println!("cargo:rerun-if-changed=tvos_chkstk.s");

34 + println!("cargo:rerun-if-changed=scripts/tvos_chkstk.s");

35 } else {

36 println!(

37 "cargo:warning=tvos_chkstk.s not found at {} — \

docs/README.md deleted

-72

@@ -1,72 +0,0 @@

1 -# Onde Docs

2 -

3 -Source for the [Onde documentation site](https://ondeinference.com/docs), built with [Mintlify](https://mintlify.com).

4 -

5 -## Structure

6 -

7 -```

8 -docs/

9 -├── mint.json # Mintlify site config — navigation, colors, logo

10 -├── index.md # Landing page → /docs

11 -├── dev.md # Developer guide → /docs/dev

12 -├── swift-package.md # Swift Package guide → /docs/swift-package

13 -├── ruby-gem.md # Ruby Gem guide → /docs/ruby-gem

14 -└── distribution.md # Publishing guide → /docs/distribution

15 -```

16 -

17 -## Local Preview

18 -

19 -Install the Mintlify CLI:

20 -

21 -```bash

22 -npm install -g mintlify

23 -```

24 -

25 -Start the dev server from the `docs/` directory:

26 -

27 -```bash

28 -cd docs

29 -mintlify dev

30 -```

31 -

32 -The site is served at `http://localhost:3000`.

33 -

34 -## Deployment

35 -

36 -The docs deploy automatically via the [Mintlify GitHub App](https://github.com/apps/mintlify).

37 -

38 -1. Install the Mintlify GitHub App on the `ondeinference/onde` repository.

39 -2. In the Mintlify dashboard, set the docs directory to `docs/` and the base path to `/docs`.

40 -3. Every push to `main` triggers a redeploy.

41 -

42 -The custom domain is configured in the Mintlify dashboard:

43 -

44 -- Domain: `ondeinference.com`

45 -- Path: `/docs`

46 -

47 -## Adding a Page

48 -

49 -1. Create a new `.md` file in `docs/`.

50 -2. Add a YAML frontmatter block at the top:

51 - ```yaml

52 - ---

53 - title: "Page Title"

54 - description: "One-sentence description shown in search and link previews."

55 - ---

56 - ```

57 -3. Add the filename (without `.md`) to the appropriate group in `mint.json` under `navigation`.

58 -

59 -## Frontmatter Fields

60 -

61 -| Field | Required | Description |

62 -| ----- | -------- | ----------- |

63 -| `title` | Yes | Page title shown in the browser tab and sidebar |

64 -| `description` | Yes | Meta description for search and link previews |

65 -| `sidebarTitle` | No | Override the sidebar label (defaults to `title`) |

66 -

67 -## Content Guidelines

68 -

69 -- Write for developers who are evaluating or integrating Onde — assume Rust familiarity, not deep familiarity with the project.

70 -- Code blocks must specify a language for syntax highlighting (` ```swift `, ` ```rust `, ` ```bash `, etc.).

71 -- Cross-links between pages omit the `.md` extension: `[Swift Package](swift-package)`, not `[Swift Package](swift-package.md)`.

72 -- Keep internal/contributor notes (build internals, pitfalls, architecture deep-dives) in the `.agent-skills/` directory, not in `docs/`.

\ No newline at end of file

docs/dev.md deleted

-209

@@ -1,209 +0,0 @@

1 ----

2 -title: "Developer Guide"

3 -description: "Build Onde from source, understand the architecture, and integrate it as a Rust dependency."

4 -sidebarTitle: "Developer Guide"

5 ----

6 -

7 -# Onde — Developer Guide

8 -

9 -> **See also:** [Swift Package](swift-package) · [Ruby Gem](ruby-gem) · [Distribution](distribution)

10 -

11 -## Prerequisites

12 -

13 -```bash

14 -# Rust stable

15 -rustup toolchain install stable

16 -

17 -# Rust nightly (only needed for tvOS tier-3 targets with -Z build-std)

18 -rustup toolchain install nightly

19 -rustup component add rust-src --toolchain nightly

20 -

21 -# iOS targets (stable)

22 -rustup target add aarch64-apple-ios aarch64-apple-ios-sim

23 -```

24 -

25 -## Building

26 -

27 -### Native (macOS — Metal accelerated)

28 -

29 -```bash

30 -# Check compilation

31 -cargo check

32 -

33 -# Run all tests

34 -cargo test

35 -

36 -# Clippy lint

37 -cargo clippy

38 -```

39 -

40 -### Ruby Gem (magnus)

41 -

42 -Build the Ruby native extension that exposes Onde's HuggingFace cache management and model metadata to Ruby via [magnus](https://github.com/matsadler/magnus):

43 -

44 -```bash

45 -cd sdk/gem

46 -

47 -# Install Ruby dependencies + compile the Rust native extension

48 -bin/setup

49 -

50 -# Or step by step:

51 -bundle install

52 -bundle exec rake compile

53 -

54 -# Interactive console

55 -bin/console

56 -```

57 -

58 -Verify it works:

59 -

60 -```bash

61 -bundle exec ruby -e "require 'onde'; puts Onde::VERSION; puts Onde::SUPPORTED_MODELS"

62 -```

63 -

64 -For full API reference, usage examples, and architecture details, see [ruby-gem](ruby-gem).

65 -

66 -## Architecture

67 -

68 -```

69 -onde/

70 -├── src/

71 -│ ├── lib.rs # Crate root — uniffi::setup_scaffolding!()

72 -│ ├── hf_cache.rs # HuggingFace Hub cache management

73 -│ └── inference/

74 -│ ├── mod.rs # Module exports + re-exports

75 -│ ├── types.rs # UniFFI-annotated shared types

76 -│ ├── engine.rs # ChatEngine (Rust-native API)

77 -│ ├── ffi.rs # OndeChatEngine (UniFFI Object for Swift/Kotlin)

78 -│ ├── models.rs # Model ID constants + metadata

79 -│ └── token.rs # HF token resolution (build-time / cache)

80 -├── sdk/

81 -│ ├── Onde/ # Generated Swift package (git-ignored)

82 -│ ├── gem/ # Ruby gem (magnus-based native extension)

83 -│ ├── kotlin/ # Kotlin bindings

84 -│ └── python/ # Python bindings

85 -├── generated/ # UniFFI-generated headers and sources

86 -├── .cargo/

87 -│ └── config.toml # Target-specific rustflags (fp16)

88 -├── Cargo.toml # Platform-conditional mistralrs deps

89 -├── build.rs # UniFFI scaffolding + tvOS chkstk stub

90 -├── uniffi.toml # UniFFI binding config

91 -└── docs/

92 - ├── dev.md # This file

93 - ├── swift-package.md # Swift package building & API guide

94 - └── ruby-gem.md # Ruby gem API reference & guide

95 -```

96 -

97 -### Layer Diagram

98 -

99 -```

100 -┌──────────────────────────────────────────────────────┐

101 -│ Swift (tvOS / iOS) Rust app / CLI / server │

102 -│ import Onde use onde::inference::* │

103 -└──────────┬───────────────────────────┬───────────────┘

104 - │ UniFFI bindings │ Direct Rust API

105 - ▼ ▼

106 -┌──────────────────────┐ ┌────────────────────────────┐

107 -│ OndeChatEngine │ │ ChatEngine │

108 -│ (ffi.rs — Object) │──│ (engine.rs — native) │

109 -│ FFI-safe methods │ │ impl Into<String>, mpsc │

110 -└──────────┬───────────┘ └────────────┬───────────────┘

111 - │ │

112 - └───────────┬───────────────┘

113 - ▼

114 - ┌───────────────────────┐

115 - │ mistralrs::Model │

116 - │ (GgufModelBuilder) │

117 - │ Metal / CUDA / CPU │

118 - └───────────────────────┘

119 -```

120 -

121 -## Platform Support

122 -

124 -| -------- | ----------- | ----------------- | -------------- | -------------------------------- |

126 -| iOS | `ios` | Metal | Qwen 2.5 1.5B | mistralrs `features = ["metal"]` |

131 -

132 -## Testing

133 -

134 -```bash

135 -# All tests

136 -cargo test

137 -

138 -# Just inference module tests

139 -cargo test inference::

140 -

141 -# Just FFI wrapper tests

142 -cargo test inference::ffi::

143 -

144 -# Just type tests

145 -cargo test inference::types::

146 -

147 -# Clippy

148 -cargo clippy

149 -```

150 -

151 -## Using Onde as a Rust Dependency

152 -

153 -Add onde as a dependency in your `Cargo.toml`:

154 -

155 -```toml

156 -[dependencies]

157 -onde = { path = "../../onde" } # adjust relative path to your layout

158 -# or from git:

159 -# onde = { git = "https://github.com/ondeinference/onde" }

160 -```

161 -

162 -The Rust API is accessed directly — no UniFFI needed:

163 -

164 -```rust

165 -use onde::mistralrs::{GgufModelBuilder, Model, RequestBuilder, TextMessageRole};

166 -use onde::inference::token::hf_token_source;

167 -

168 -// Load model

169 -let model = GgufModelBuilder::new("bartowski/Qwen2.5-1.5B-Instruct-GGUF", vec!["Qwen2.5-1.5B-Instruct-Q4_K_M.gguf"])

170 - .with_token_source(hf_token_source())

171 - .with_logging()

172 - .build()

173 - .await?;

174 -

175 -// Chat

176 -let request = RequestBuilder::new()

177 - .add_message(TextMessageRole::System, "You are a helpful assistant.")

178 - .add_message(TextMessageRole::User, "Hello!")

179 - .set_sampler_temperature(0.7);

180 -

181 -let response = model.send_chat_request(request).await?;

182 -```

183 -

184 -### Important Setup for Sandboxed Apps (iOS / macOS App Store)

185 -

186 -Your app's setup function must configure `HF_HOME` and `TMPDIR` before any inference code runs. See the [pepakbasajawa](https://github.com/pfrfrfr/pepakbasajawa) or [fatapp](https://github.com/setoelkahfi/fatapp) repos for working examples of the `setup_hf_home()` and `setup_tmpdir()` pattern.

187 -

188 -## Known Issues

189 -

190 -### `___chkstk_darwin` linker error on tvOS

191 -

192 -When building for `aarch64-apple-tvos`, `aws-lc-sys` (a transitive dependency via `rustls` → `reqwest` → `hf-hub`) references `___chkstk_darwin`, a stack probing symbol not available on tvOS. The onde `build.rs` provides a no-op assembly stub (`tvos_chkstk.s`) that resolves this automatically.

193 -

194 -This does not affect macOS, iOS, Windows, Linux, or Android builds.

195 -

196 -### Metal Toolchain must be installed (Xcode 26+)

197 -

198 -On Xcode 26+, the Metal compiler is a separate downloadable component. If you see empty `.metallib` files (92 bytes) or `"Error while loading function: fused_glu_float"` at runtime, run:

199 -

200 -```bash

201 -xcodebuild -downloadComponent MetalToolchain

202 -```

203 -

204 -Then clean and rebuild:

205 -

206 -```bash

207 -cargo clean -p mistralrs-quant

208 -cargo check

209 -```

docs/distribution.md deleted

-205

@@ -1,205 +0,0 @@

1 ----

2 -title: "Distribution"

3 -description: "How Onde is distributed across package registries — crates.io, Swift Package Index, PyPI, RubyGems, and Maven Central."

4 ----

5 -

6 -# Distribution

7 -

8 -Onde is published to every major package registry under a consistent naming convention:

9 -

10 -| Registry | Package name | Namespace |

11 -| -------- | ------------ | --------- |

12 -| crates.io | `onde` | — |

13 -| Swift Package Index | `onde-swift` | `ondeinference` |

14 -| PyPI | `onde-inference` | — |

15 -| RubyGems | `onde-inference` | — |

16 -| Maven Central | `onde-inference` | `com.ondeinference` |

17 -

18 -**Naming rules:**

19 -- Registries with a namespace → `ondeinference` / `onde`

20 -- Registries without a namespace → `onde-inference` (hyphenated) to avoid conflicts

21 -- Swift follows Apple conventions → PascalCase package name (`Onde`), kebab-case repo slug (`onde-swift`)

22 -- The Rust crate uses the bare `onde` name on crates.io

23 -

24 ----

25 -

26 -## Rust — crates.io

27 -

28 -The core Rust crate is published as [`onde`](https://crates.io/crates/onde).

29 -

30 -```toml

31 -[dependencies]

32 -onde = "0.1"

33 -```

34 -

35 -Publish:

36 -

37 -```bash

38 -cargo publish

39 -```

40 -

41 ----

42 -

43 -## Swift — Swift Package Index

44 -

45 -The Swift package is distributed from a dedicated public repository [`ondeinference/onde-swift`](https://github.com/ondeinference/onde-swift) as a remote binary package. The `OndeFramework.xcframework` is attached as a GitHub Release asset; `Package.swift` references it by URL and SHA-256 checksum.

46 -

47 -Swift Package Index indexes the `onde-swift` repository automatically on every new Git tag.

48 -

49 -### Installation

50 -

51 -In Xcode: **File → Add Package Dependencies** and enter:

52 -

53 -```

54 -https://github.com/ondeinference/onde-swift

55 -```

56 -

57 -Or in `Package.swift`:

58 -

59 -```swift

60 -dependencies: [

61 - .package(url: "https://github.com/ondeinference/onde-swift", from: "0.1.0")

62 -]

63 -```

64 -

65 -### Release process

66 -

67 -1. Build `OndeFramework.xcframework` from Rust source (see [Swift Package](swift-package#building-the-xcframework-locally)).

68 -2. Zip it and compute the checksum:

69 - ```bash

70 - zip -r OndeFramework.xcframework.zip OndeFramework.xcframework

71 - shasum -a 256 OndeFramework.xcframework.zip

72 - ```

73 -3. Create a GitHub Release on the Rust source repo and attach the zip as a release asset.

74 -4. In `onde-swift`, update `Package.swift` with the new `url:` and `checksum:`.

75 -5. Commit, tag the version (`0.1.0`, `0.2.0`, etc.), and push — SPI picks it up automatically.

76 -

77 ----

78 -

79 -## Python — PyPI

80 -

81 -The Python package is published as [`onde-inference`](https://pypi.org/project/onde-inference) and imported as `onde_inference`. It uses [maturin](https://github.com/PyO3/maturin) with `bindings = "uniffi"` to generate Python bindings from the same UniFFI scaffolding used for Swift and Kotlin.

82 -

83 -### Installation

84 -

85 -```bash

86 -pip install onde-inference

87 -```

88 -

89 -### Usage

90 -

91 -```python

92 -import asyncio

93 -from onde_inference import OndeChatEngine, default_model_config

94 -

95 -async def main():

96 - engine = OndeChatEngine()

97 - await engine.load_default_model(

98 - system_prompt="You are a helpful assistant.",

99 - sampling=None,

100 - )

101 - result = await engine.send_message("Hello!")

102 - print(result.text)

103 -

104 -asyncio.run(main())

105 -```

106 -

107 -### Prerequisites

108 -

109 -- Python 3.9+

110 -- `maturin >= 1.7` — `uv tool install maturin` or `pip install maturin`

111 -- `uniffi-bindgen == 0.31.0` — must match the `uniffi` version pinned in `Cargo.toml`

112 -

113 - ```bash

114 - pip install uniffi-bindgen==0.31.0

115 - ```

116 -

117 -### Build

118 -

119 -```bash

120 -cd sdk/python

121 -

122 -# Build a release wheel for the current platform

123 -maturin build --release --out dist

124 -

125 -# Development install into the current venv

126 -maturin develop

127 -```

128 -

129 -### Publish

130 -

131 -```bash

132 -cd sdk/python

133 -maturin publish

134 -```

135 -

136 -To publish wheels for all platforms, run `maturin publish` on each platform in CI and pass `--skip-existing` so subsequent uploads don't fail. Platforms to cover:

137 -

138 -- macOS: `aarch64-apple-darwin`, `x86_64-apple-darwin`

139 -- Linux: `x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu`

140 -- Windows: `x86_64-pc-windows-msvc`

141 -

142 ----

143 -

144 -## Ruby — RubyGems

145 -

146 -The Ruby gem is published as [`onde-inference`](https://rubygems.org/gems/onde-inference) and required as `onde`. It is a [Magnus](https://github.com/matsadler/magnus)-based native extension exposing Onde's HuggingFace cache management and model metadata to Ruby.

147 -

148 -See the [Ruby Gem guide](ruby-gem) for full API reference and Rails integration examples.

149 -

150 -### Installation

151 -

152 -```bash

153 -gem install onde-inference

154 -```

155 -

156 -Or in your `Gemfile`:

157 -

158 -```ruby

159 -gem "onde-inference"

160 -```

161 -

162 -### Publish

163 -

164 -```bash

165 -cd sdk/gem

166 -bundle exec rake compile # build the native extension

167 -gem build onde-inference.gemspec

168 -gem push onde-inference-*.gem

169 -```

170 -

171 ----

172 -

173 -## Kotlin / Android — Maven Central

174 -

175 -The Kotlin Android library is published as `com.ondeinference:onde-inference` on Maven Central.

176 -

177 -### Installation

178 -

179 -In `build.gradle.kts`:

180 -

181 -```kotlin

182 -dependencies {

183 - implementation("com.ondeinference:onde-inference:0.1.0")

184 -}

185 -```

186 -

187 -### Build

188 -

189 -The Android library is built with `build-kotlin.sh` at the crate root. It cross-compiles `libonde.so` for all four Android ABIs and generates Kotlin bindings via `uniffi-bindgen`:

190 -

191 -```bash

192 -# Full release build for all Android ABIs

193 -./build-kotlin.sh

194 -

195 -# arm64 only (faster for development)

196 -./build-kotlin.sh --target aarch64-linux-android

197 -```

198 -

199 -Then assemble and publish the AAR from the Android project:

200 -

201 -```bash

202 -cd sdk/kotlin

203 -./gradlew :onde:assembleRelease

204 -./gradlew publishReleasePublicationToMavenCentralRepository

205 -```

docs/index.md deleted

-95

@@ -1,95 +0,0 @@

1 ----

2 -title: "Onde"

3 -description: "On-device chat inference for every platform. Run LLMs locally with no cloud, no latency, and no data leaving the device."

4 ----

5 -

6 -# Onde

7 -

8 -**On-device chat inference for every platform.**

9 -

10 -Run LLM chat locally — no cloud, no latency, no data leaving the device. Onde wraps [mistral.rs](https://github.com/EricLBuehler/mistral.rs) with a unified API that handles model discovery, HuggingFace Hub downloads, cache management, and GPU acceleration across every platform.

11 -

12 -## Platforms

13 -

14 -| Platform | GPU Backend | Default Model |

15 -| -------- | ----------- | ------------- |

16 -| macOS | Metal | Qwen 2.5 3B |

17 -| iOS | Metal | Qwen 2.5 1.5B |

18 -| tvOS | Metal | Qwen 2.5 1.5B |

19 -| Android | CPU (candle) | Qwen 2.5 1.5B |

20 -| Windows | CPU / CUDA | Qwen 2.5 3B |

21 -| Linux | CPU / CUDA | Qwen 2.5 3B |

22 -

23 -## SDKs

24 -

25 -| Language | Package | Install |

26 -| -------- | ------- | ------- |

27 -| Rust | [`onde`](https://crates.io/crates/onde) | `onde = "0.1"` in Cargo.toml |

28 -| Swift | [`onde-swift`](https://github.com/ondeinference/onde-swift) | Swift Package Manager |

29 -| Kotlin | [`onde-inference`](https://central.sonatype.com/) | Gradle |

30 -| Python | [`onde-inference`](https://pypi.org/project/onde-inference) | `pip install onde-inference` |

31 -| Ruby | [`onde-inference`](https://rubygems.org/gems/onde-inference) | `gem install onde-inference` |

32 -

33 -## Quick Start

34 -

35 -### Swift (iOS / tvOS / macOS)

36 -

37 -```swift

38 -import Onde

39 -

40 -let engine = OndeChatEngine()

41 -try await engine.loadDefaultModel(systemPrompt: "You are a helpful assistant.", sampling: nil)

42 -

43 -let result = try await engine.sendMessage(message: "Hello!")

44 -print(result.text)

45 -```

46 -

47 -### Rust

48 -

49 -```rust

50 -use onde::inference::{ChatEngine, GgufModelConfig};

51 -

52 -let engine = ChatEngine::new();

53 -engine.load_gguf_model(

54 - GgufModelConfig::platform_default(),

55 - Some("You are a helpful assistant.".into()),

56 - None,

57 -).await?;

58 -

59 -let reply = engine.send_message("Hello!").await?;

60 -println!("{}", reply.text);

61 -```

62 -

63 -### Python

64 -

65 -```python

66 -import asyncio

67 -from onde_inference import OndeChatEngine

68 -

69 -async def main():

70 - engine = OndeChatEngine()

71 - await engine.load_default_model(system_prompt="You are a helpful assistant.", sampling=None)

72 - result = await engine.send_message("Hello!")

73 - print(result.text)

74 -

75 -asyncio.run(main())

76 -```

77 -

78 -## Guides

79 -

80 -- [Developer Guide](dev) — build from source, architecture overview, platform support

81 -- [Swift Package](swift-package) — XCFramework distribution, Swift API reference

82 -- [Ruby Gem](ruby-gem) — Magnus-based native extension, Rails integration

83 -- [Distribution](distribution) — publishing to crates.io, Swift Package Index, PyPI, RubyGems, Maven Central

84 -

85 -## Design Principles

86 -

87 -**Zero framework coupling.** Onde knows nothing about any app framework. Progress updates use plain callbacks — wire them to whatever event system you use.

88 -

89 -**Platform-aware by default.** GPU backend, model size, cache directory — Onde picks the right defaults so you don't configure per-platform.

90 -

91 -**One dependency.** Add `onde` and get on-device chat inference, HuggingFace cache management, and HF token resolution in a single crate.

92 -

93 -## License

94 -

95 -Licensed under either of [MIT](https://opensource.org/licenses/MIT) or [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) at your option.

\ No newline at end of file

docs/mint.json deleted

-77

@@ -1,77 +0,0 @@

1 -{

2 - "$schema": "https://mintlify.com/schema.json",

3 - "name": "Onde",

4 - "logo": {

5 - "light": "/logo/light.svg",

6 - "dark": "/logo/dark.svg",

7 - "href": "https://ondeinference.com"

8 - },

9 - "favicon": "/favicon.svg",

10 - "colors": {

11 - "primary": "#0F172A",

12 - "light": "#38BDF8",

13 - "dark": "#0F172A"

14 - },

15 - "topbarLinks": [

16 - {

17 - "name": "GitHub",

18 - "url": "https://github.com/ondeinference/onde"

19 - }

20 - ],

21 - "topbarCtaButton": {

22 - "name": "Get Started",

23 - "url": "/docs"

24 - },

25 - "anchors": [

26 - {

27 - "name": "crates.io",

28 - "icon": "box",

29 - "url": "https://crates.io/crates/onde"

30 - },

31 - {

32 - "name": "docs.rs",

33 - "icon": "book",

34 - "url": "https://docs.rs/onde"

35 - },

36 - {

37 - "name": "GitHub",

38 - "icon": "github",

39 - "url": "https://github.com/ondeinference/onde"

40 - }

41 - ],

42 - "navigation": [

43 - {

44 - "group": "Getting Started",

45 - "pages": [

46 - "index",

47 - "dev"

48 - ]

49 - },

50 - {

51 - "group": "SDKs",

52 - "pages": [

53 - "swift-package",

54 - "ruby-gem"

55 - ]

56 - },

57 - {

58 - "group": "Publishing",

59 - "pages": [

60 - "distribution"

61 - ]

62 - }

63 - ],

64 - "footerSocials": {

65 - "github": "https://github.com/ondeinference/onde",

66 - "website": "https://ondeinference.com"

67 - },

68 - "feedback": {

69 - "thumbsRating": true,

70 - "suggestEdit": true,

71 - "raiseIssue": true

72 - },

73 - "modeToggle": {

74 - "default": "dark"

75 - },

76 - "tabs": []

77 -}

docs/ruby-gem.md deleted

-632

@@ -1,632 +0,0 @@

1 ----

2 -title: "Ruby Gem"

3 -description: "Build and use the onde-inference Ruby gem — a Magnus-based native extension exposing HuggingFace cache management and model metadata to Ruby."

4 ----

5 -

6 -# Onde Ruby Gem

7 -

8 -Ruby bindings for the [Onde](https://ondeinference.com) on-device inference engine, powered by Rust via [magnus](https://github.com/matsadler/magnus).

9 -

10 -Manage HuggingFace model caches, inspect supported models, and access inference configuration — all from Ruby with native Rust performance.

11 -

12 -## Prerequisites

13 -

14 -- **Ruby** >= 3.0

15 -- **Rust** >= 1.92 (installed via [rustup](https://rustup.rs))

16 -- **Bundler** (`gem install bundler`)

17 -- macOS, Linux, or Windows

18 -

19 -The gem compiles a native Rust extension at install time. Rust must be available on your `$PATH`.

20 -

21 -## Setup

22 -

23 -```bash

24 -cd lib/crates/onde/gem

25 -

26 -# Install Ruby dependencies and compile the Rust native extension

27 -bin/setup

28 -

29 -# Or step by step:

30 -bundle install

31 -bundle exec rake compile

32 -```

33 -

34 -### Verify the installation

35 -

36 -```bash

37 -bundle exec ruby -e "require 'onde'; puts Onde::VERSION"

38 -# => 0.1.0

39 -```

40 -

41 -### Interactive console

42 -

43 -```bash

44 -bin/console

45 -```

46 -

47 -Opens an IRB session with the gem pre-loaded. Useful for exploring the API:

48 -

49 -```ruby

50 -irb> Onde::SUPPORTED_MODELS

51 -=> ["bartowski/Qwen2.5-1.5B-Instruct-GGUF", "bartowski/Qwen2.5-3B-Instruct-GGUF"]

52 -

53 -irb> Onde.cache_path

54 -=> "/Users/you/.cache/huggingface/hub"

55 -```

56 -

57 -## Build Commands

58 -

59 -| Command | Description |

60 -|---------|-------------|

61 -| `bundle exec rake compile` | Compile the Rust native extension (release profile) |

62 -| `bundle exec rake compile:dev` | Compile with debug profile (faster builds, slower runtime) |

63 -| `bundle exec rake build` | Build the `.gem` package |

64 -| `bundle exec rake` | Default task — compiles the extension |

65 -| `bin/setup` | One-step: `bundle install` + `rake compile` |

66 -| `bin/console` | IRB with the gem loaded |

67 -

68 -## API Reference

69 -

70 -### Cache Management

71 -

72 -#### `Onde.list_local_models` → Hash

73 -

74 -Scans the local HuggingFace hub cache (`~/.cache/huggingface/hub/`) and returns all downloaded models that the inference engine supports.

75 -

76 -```ruby

77 -response = Onde.list_local_models

78 -

79 -response["cache_path"] # => "/Users/you/.cache/huggingface/hub"

80 -response["total_size_bytes"] # => 24768454656

81 -response["total_size_display"] # => "23.07 GB"

82 -

83 -response["models"].each do |model|

84 - puts "#{model["model_id"]} — #{model["size_display"]}"

85 - puts " Path: #{model["path"]}"

86 - puts " Revisions: #{model["revisions"].join(", ")}"

87 -end

88 -```

89 -

90 -**Return value keys:**

91 -

92 -| Key | Type | Description |

93 -|-----|------|-------------|

94 -| `cache_path` | String | Absolute path that was scanned |

95 -| `total_size_bytes` | Integer | Total size of all cached models |

96 -| `total_size_display` | String | Human-readable total size (e.g. `"23.07 GB"`) |

97 -| `models` | Array | Array of model Hashes (see below) |

98 -

99 -**Each model Hash:**

100 -

101 -| Key | Type | Description |

102 -|-----|------|-------------|

103 -| `model_id` | String | Full HuggingFace ID (e.g. `"bartowski/Qwen2.5-1.5B-Instruct-GGUF"`) |

104 -| `org` | String | Organisation or publisher |

105 -| `name` | String | Model name without org prefix |

106 -| `size_bytes` | Integer | Total size on disk in bytes |

107 -| `size_display` | String | Human-readable size (e.g. `"1.84 GB"`) |

108 -| `path` | String | Absolute path to the model cache directory |

109 -| `revisions` | Array | List of snapshot revision strings |

110 -

111 ----

112 -

113 -#### `Onde.list_supported_models` → Hash

114 -

115 -Returns all models the engine supports, together with flags indicating whether each one is fully downloaded, partially downloaded, or absent.

116 -

117 -```ruby

118 -Onde.list_supported_models["models"].each do |m|

119 - status = if m["is_downloaded"]

120 - "✓ downloaded"

121 - elsif m["is_incomplete"]

122 - "⏳ incomplete (#{m["local_size_display"]} / #{m["expected_size_display"]})"

123 - else

124 - "✗ not downloaded"

125 - end

126 - puts "[#{status}] #{m["name"]} (#{m["org"]})"

127 - puts " #{m["description"]}"

128 -end

129 -```

130 -

131 -**Each model Hash:**

132 -

133 -| Key | Type | Description |

134 -|-----|------|-------------|

135 -| `model_id` | String | Full HuggingFace ID |

136 -| `name` | String | Human-friendly display name |

137 -| `org` | String | Organisation or publisher |

138 -| `description` | String | Short description of the model |

139 -| `is_downloaded` | Boolean | Fully downloaded locally |

140 -| `is_incomplete` | Boolean | Partial download exists on disk |

141 -| `local_size_bytes` | Integer | Bytes currently on disk (0 if absent) |

142 -| `local_size_display` | String | Human-readable local size |

143 -| `expected_size_bytes` | Integer | Approximate total size when fully downloaded |

144 -| `expected_size_display` | String | Human-readable expected size |

145 -

146 ----

147 -

148 -#### `Onde.delete_model(model_id)` → nil

149 -

150 -Delete a locally cached HuggingFace model by removing its directory from the hub cache.

151 -

152 -```ruby

153 -Onde.delete_model("bartowski/Qwen2.5-1.5B-Instruct-GGUF")

154 -```

155 -

156 -**Parameters:**

157 -

158 -| Name | Type | Description |

159 -|------|------|-------------|

160 -| `model_id` | String | Full model identifier (e.g. `"bartowski/Qwen2.5-1.5B-Instruct-GGUF"`) |

161 -

162 -**Raises:** `RuntimeError` if the model is not found or deletion fails.

163 -

164 ----

165 -

166 -#### `Onde.cache_path` → String or nil

167 -

168 -Returns the resolved HuggingFace cache directory path, or `nil` if it cannot be determined (e.g. `$HOME` is unset).

169 -

170 -```ruby

171 -Onde.cache_path

172 -# => "/Users/you/.cache/huggingface/hub"

173 -```

174 -

175 ----

176 -

177 -### Model Metadata

178 -

179 -#### `Onde.model_info(model_id)` → Hash or nil

180 -

181 -Look up rich metadata for a single supported model. Returns `nil` if the model ID is not in the supported list.

182 -

183 -```ruby

184 -info = Onde.model_info("bartowski/Qwen2.5-1.5B-Instruct-GGUF")

185 -# => {

186 -# "id" => "bartowski/Qwen2.5-1.5B-Instruct-GGUF",

187 -# "name" => "Qwen 2.5 1.5B (GGUF)",

188 -# "org" => "Qwen / Alibaba",

189 -# "description" => "Lightest pre-quantized chat model — ...",

190 -# "expected_size_bytes" => 986048768

191 -# }

192 -

193 -Onde.model_info("nonexistent/model")

194 -# => nil

195 -```

196 -

197 -**Return value keys:**

198 -

199 -| Key | Type | Description |

200 -|-----|------|-------------|

201 -| `id` | String | Full HuggingFace model identifier |

202 -| `name` | String | Human-friendly display name |

203 -| `org` | String | Organisation or publisher |

204 -| `description` | String | Short description |

205 -| `expected_size_bytes` | Integer | Approximate total size in bytes |

206 -

207 ----

208 -

209 -#### `Onde.supported_model_ids` → Array

210 -

211 -Returns the list of all supported model IDs as strings.

212 -

213 -```ruby

214 -Onde.supported_model_ids

215 -# => [

216 -# "bartowski/Qwen2.5-1.5B-Instruct-GGUF",

217 -# "bartowski/Qwen2.5-3B-Instruct-GGUF"

218 -# ]

219 -```

220 -

221 ----

222 -

223 -### Sampling Configuration

224 -

225 -These methods return Hash representations of the inference sampling parameters. They are useful for inspecting defaults or passing to future inference APIs.

226 -

227 -#### `Onde.default_sampling_config` → Hash

228 -

229 -Creative chat defaults.

230 -

231 -```ruby

232 -Onde.default_sampling_config

233 -# => {

234 -# "temperature" => 0.7,

235 -# "top_p" => 0.95,

236 -# "top_k" => nil,

237 -# "min_p" => nil,

238 -# "max_tokens" => 512,

239 -# "frequency_penalty" => nil,

240 -# "presence_penalty" => nil

241 -# }

242 -```

243 -

244 -#### `Onde.deterministic_sampling_config` → Hash

245 -

246 -Greedy decoding (temperature = 0.0).

247 -

248 -```ruby

249 -Onde.deterministic_sampling_config

250 -# => { "temperature" => 0.0, "top_p" => nil, ..., "max_tokens" => 512 }

251 -```

252 -

253 -#### `Onde.mobile_sampling_config` → Hash

254 -

255 -Conservative defaults for constrained devices (lower `max_tokens`).

256 -

257 -```ruby

258 -Onde.mobile_sampling_config

259 -# => { "temperature" => 0.7, "top_p" => 0.95, ..., "max_tokens" => 128 }

260 -```

261 -

262 ----

263 -

264 -### Constants

265 -

266 -| Constant | Type | Description |

267 -|----------|------|-------------|

268 -| `Onde::VERSION` | String | Gem version (e.g. `"0.1.0"`) |

269 -| `Onde::NATIVE_VERSION` | String | Rust crate version (e.g. `"0.1.0"`) |

270 -| `Onde::SUPPORTED_MODELS` | Array | Frozen array of all supported model ID strings |

271 -| `Onde::SUPPORTED_MODEL_INFO` | Array | Frozen array of frozen Hashes with model metadata |

272 -

273 -```ruby

274 -Onde::SUPPORTED_MODELS

275 -# => ["bartowski/Qwen2.5-1.5B-Instruct-GGUF", "bartowski/Qwen2.5-3B-Instruct-GGUF"]

276 -

277 -Onde::SUPPORTED_MODEL_INFO.first

278 -# => {

279 -# "id" => "bartowski/Qwen2.5-1.5B-Instruct-GGUF",

280 -# "name" => "Qwen 2.5 1.5B (GGUF)",

281 -# "org" => "Qwen / Alibaba",

282 -# "description" => "Lightest pre-quantized chat model — ...",

283 -# "expected_size_bytes" => 986048768

284 -# }

285 -```

286 -

287 -Both constants are frozen at load time — attempting to modify them raises `FrozenError`.

288 -

289 -## Usage Examples

290 -

291 -### Rails Admin Dashboard

292 -

293 -```ruby

294 -# app/controllers/admin/models_controller.rb

295 -class Admin::ModelsController < ApplicationController

296 - def index

297 - @supported = Onde.list_supported_models["models"]

298 - @local = Onde.list_local_models

299 - end

300 -

301 - def destroy

302 - Onde.delete_model(params[:model_id])

303 - redirect_to admin_models_path, notice: "Model cache deleted."

304 - rescue RuntimeError => e

305 - redirect_to admin_models_path, alert: e.message

306 - end

307 -end

308 -```

309 -

310 -### Background Job — Cache Cleanup

311 -

312 -```ruby

313 -# app/jobs/cleanup_hf_cache_job.rb

314 -class CleanupHfCacheJob < ApplicationJob

315 - queue_as :maintenance

316 -

317 - def perform

318 - local = Onde.list_local_models

319 - logger.info "[Onde] HF cache: #{local["total_size_display"]} across #{local["models"].length} models"

320 -

321 - # Delete models that are no longer in the supported list

322 - local["models"].each do |model|

323 - unless Onde::SUPPORTED_MODELS.include?(model["model_id"])

324 - logger.info "[Onde] Removing unsupported model: #{model["model_id"]} (#{model["size_display"]})"

325 - Onde.delete_model(model["model_id"])

326 - end

327 - end

328 - end

329 -end

330 -```

331 -

332 -### Rake Task — Model Status Report

333 -

334 -```ruby

335 -# lib/tasks/onde.rake

336 -namespace :onde do

337 - desc "Show HuggingFace model cache status"

338 - task status: :environment do

339 - puts "Cache: #{Onde.cache_path || "(not found)"}"

340 - puts

341 -

342 - Onde.list_supported_models["models"].each do |m|

343 - icon = m["is_downloaded"] ? "✓" : (m["is_incomplete"] ? "⏳" : "✗")

344 - local = m["is_downloaded"] ? m["local_size_display"] : (m["is_incomplete"] ? m["local_size_display"] : "—")

345 - puts "[#{icon}] #{m["name"].ljust(25)} #{local.rjust(10)} / #{m["expected_size_display"].rjust(10)} (#{m["org"]})"

346 - end

347 -

348 - local = Onde.list_local_models

349 - puts

350 - puts "Total on disk: #{local["total_size_display"]}"

351 - end

352 -end

353 -```

354 -

355 -Run with:

356 -

357 -```bash

358 -bundle exec rake onde:status

359 -```

360 -

361 -Example output:

362 -

363 -```

364 -Cache: /Users/you/.cache/huggingface/hub

365 -

366 -[✓] Qwen 2.5 1.5B (GGUF) 1.84 GB / 940.4 MB (Qwen / Alibaba)

367 -[✓] Qwen 2.5 3B (GGUF) 3.59 GB / 1.80 GB (Qwen / Alibaba)

368 -

369 -Total on disk: 5.43 GB

370 -```

371 -

372 -### API Endpoint

373 -

374 -```ruby

375 -# app/controllers/api/v1/models_controller.rb

376 -# frozen_string_literal: true

377 -

378 -module Api

379 - module V1

380 - class ModelsController < ApplicationController

381 - def index

382 - render json: Onde.list_supported_models

383 - end

384 -

385 - def show

386 - info = Onde.model_info(params[:id])

387 - if info

388 - render json: info

389 - else

390 - render json: { error: "Model not found" }, status: :not_found

391 - end

392 - end

393 - end

394 - end

395 -end

396 -```

397 -

398 -## Architecture

399 -

400 -```

401 -gem/

402 -├── Cargo.toml # Workspace root (isolates from parent onde crate)

403 -├── Gemfile # Dev deps: rake-compiler, rb_sys ~> 0.9.63

404 -├── Rakefile # RbSys::ExtensionTask build config

405 -├── onde.gemspec # Gem specification

406 -├── rust-toolchain.toml # Pinned to Rust 1.92

407 -├── .gitignore

408 -├── bin/

409 -│ ├── console # IRB with gem pre-loaded

410 -│ └── setup # One-step dev setup script

411 -├── lib/

412 -│ ├── onde.rb # Entry point — loads native ext + defines module

413 -│ └── onde/

414 -│ ├── version.rb # Onde::VERSION = '0.1.0'

415 -│ └── onde_ruby.bundle # Compiled native extension (git-ignored)

416 -└── ext/

417 - └── onde-ruby/

418 - ├── Cargo.toml # Rust crate: depends on `onde` + `magnus`

419 - ├── extconf.rb # rb_sys build hook

420 - └── src/

421 - └── lib.rs # Magnus bindings — the Rust ↔ Ruby bridge

422 -```

423 -

424 -### How the binding works

425 -

426 -```

427 -┌───────────────────────────┐

428 -│ Ruby │

429 -│ require "onde" │

430 -│ Onde.list_local_models │

431 -└───────────┬───────────────┘

432 - │ FFI call via magnus

433 - ▼

434 -┌───────────────────────────┐

435 -│ onde-ruby (cdylib) │

436 -│ ext/onde-ruby/src/lib.rs│

437 -│ │

438 -│ #[magnus::init] │

439 -│ fn init(ruby) { │

440 -│ module.define_*() │

441 -│ } │

442 -└───────────┬───────────────┘

443 - │ Rust dependency

444 - ▼

445 -┌───────────────────────────┐

446 -│ onde (Rust crate) │

447 -│ lib/crates/onde/src/ │

448 -│ │

449 -│ hf_cache:: │

450 -│ inference::models:: │

451 -│ inference::types:: │

452 -└───────────┬───────────────┘

453 - │

454 - ▼

455 -┌───────────────────────────┐

456 -│ mistral.rs │

457 -│ HuggingFace Hub │

458 -│ Metal / CUDA / CPU │

459 -└───────────────────────────┘

460 -```

461 -

462 -### Data flow

463 -

464 -All Rust structs are converted to Ruby Hashes via `serde_json` round-tripping:

465 -

466 -```

467 -Rust struct ─→ serde_json::Value ─→ Ruby Hash/Array/String/Integer/nil

468 -```

469 -

470 -This keeps the binding layer thin — no custom Ruby classes are needed. Every Onde struct that implements `serde::Serialize` can be exposed to Ruby with a single `to_ruby_value()` call.

471 -

472 -### Workspace isolation

473 -

474 -The `gem/Cargo.toml` defines a Cargo workspace with `ext/onde-ruby` as its only member. This prevents `cargo metadata` from walking up to the parent `lib/crates/onde/Cargo.toml` and confusing the rb_sys build system.

475 -

476 -Without this workspace file, running `cargo metadata` from `gem/` resolves to the parent `onde` crate, and `RbSys::Cargo::Metadata` cannot find the `onde-ruby` package.

477 -

478 -### Naming convention

479 -

480 -| Layer | Name | Why |

481 -|-------|------|-----|

482 -| Cargo package | `onde-ruby` | Avoids collision with the parent `onde` crate in the dependency graph |

483 -| Cargo lib output | `onde_ruby` | Rust convention: hyphens become underscores in lib names |

484 -| Ruby require | `onde/onde_ruby` | Matches the compiled `.bundle`/`.so` filename |

485 -| Ruby module | `Onde` | Clean public API name |

486 -| Gem name | `onde` | Simple, matches the project name |

487 -

488 -## Internals

489 -

490 -### Key files

491 -

492 -| File | Purpose |

493 -|------|---------|

494 -| `ext/onde-ruby/src/lib.rs` | All Magnus bindings — defines the `Onde` Ruby module, singleton methods, and constants |

495 -| `ext/onde-ruby/Cargo.toml` | Rust crate config — depends on `onde` (path) + `magnus` 0.7 + `serde_json` |

496 -| `ext/onde-ruby/extconf.rb` | Build hook — calls `create_rust_makefile("onde/onde_ruby")` |

497 -| `lib/onde.rb` | Ruby entry point — `require_relative 'onde/onde_ruby'` loads the native ext |

498 -| `Rakefile` | `RbSys::ExtensionTask.new('onde-ruby', GEMSPEC)` wires up compilation |

499 -| `Cargo.toml` (root) | Workspace file isolating the gem from the parent crate |

500 -

501 -### Dependencies

502 -

503 -**Ruby side:**

504 -

505 -- `rake` ~> 13.0 — build orchestration

506 -- `rake-compiler` — native extension compilation framework

507 -- `rb_sys` ~> 0.9.63 — Rust ↔ Ruby build glue

508 -

509 -**Rust side:**

510 -

511 -- `magnus` 0.7 — Ruby FFI bindings for Rust

512 -- `serde` + `serde_json` — struct serialization to Ruby Hashes

513 -- `onde` (path dependency) — the core Onde crate

514 -

515 -### Adding a new method

516 -

517 -1. Add the Rust function in `ext/onde-ruby/src/lib.rs`:

518 -

519 - ```rust

520 - fn my_new_method(arg: String) -> Result<magnus::Value, Error> {

521 - let ruby = Ruby::get().expect("called outside Ruby");

522 - // ... call into onde crate ...

523 - Ok(ruby.str_new("result").as_value())

524 - }

525 - ```

526 -

527 -2. Register it in the `init` function:

528 -

529 - ```rust

530 - module.define_singleton_method("my_new_method", function!(my_new_method, 1))?;

531 - ```

532 -

533 -3. Recompile:

534 -

535 - ```bash

536 - bundle exec rake compile

537 - ```

538 -

539 -4. Test:

540 -

541 - ```bash

542 - bundle exec ruby -e "require 'onde'; puts Onde.my_new_method('hello')"

543 - ```

544 -

545 -### Adding a new constant

546 -

547 -In the `init` function in `ext/onde-ruby/src/lib.rs`:

548 -

549 -```rust

550 -module.const_set("MY_CONSTANT", ruby.str_new("value"))?;

551 -```

552 -

553 -Constants are available as `Onde::MY_CONSTANT` in Ruby. Call `.freeze()` on Arrays and Hashes before setting them as constants to prevent mutation.

554 -

555 -## Troubleshooting

556 -

557 -### `cargo` not found during `bundle install`

558 -

559 -Rust is not on your `$PATH`. Install via [rustup](https://rustup.rs):

560 -

561 -```bash

562 -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh

563 -source ~/.cargo/env

564 -```

565 -

566 -### Compilation fails with `rustc X.Y.Z is not supported`

567 -

568 -The gem requires Rust >= 1.92. Update your toolchain:

569 -

570 -```bash

571 -rustup update

572 -# or install the specific version:

573 -rustup toolchain install 1.92

574 -```

575 -

576 -The `rust-toolchain.toml` in the gem directory pins the version — rustup will select it automatically when you `cd` into the directory.

577 -

578 -### `Could not find Cargo package metadata for "onde-ruby"`

579 -

580 -The workspace `Cargo.toml` at `gem/` is missing or corrupted. It must exist and contain:

581 -

582 -```toml

583 -[workspace]

584 -members = ["ext/onde-ruby"]

585 -resolver = "2"

586 -```

587 -

588 -Without this file, `cargo metadata` resolves to the parent `onde` crate and rb_sys cannot find `onde-ruby`.

589 -

590 -### Output filename collision warning

591 -

592 -You may see:

593 -

594 -```

595 -warning: output filename collision at .../libonde.dylib

596 - note: the lib target `onde` in package `onde-ruby` has the same output

597 - filename as the lib target `onde` in package `onde`

598 -```

599 -

600 -This is a Cargo warning (not an error) caused by the extension crate's lib output name (`onde_ruby`) sharing a target directory with the parent `onde` crate. It does not affect compilation. The warning will become a hard error in a future Cargo version — at that point, set a separate `target-dir` in the workspace.

601 -

602 -### `LoadError: cannot load such file -- onde/onde_ruby`

603 -

604 -The native extension hasn't been compiled. Run:

605 -

606 -```bash

607 -bundle exec rake compile

608 -```

609 -

610 -### Stale `.bundle` / `.so` after code changes

611 -

612 -If you change `ext/onde-ruby/src/lib.rs` and the changes don't take effect:

613 -

614 -```bash

615 -bundle exec rake clobber compile

616 -```

617 -

618 -`clobber` cleans all build artifacts before recompiling.

619 -

620 -## Version Compatibility

621 -

623 -|----------|-----------|----------------|------|--------|

624 -| 0.1.0 | 0.1.0 | >= 1.92 | >= 3.0 | 0.7 |

625 -

626 -The `Onde::NATIVE_VERSION` constant always reflects the Rust crate version compiled into the gem. Compare it against `Onde::VERSION` to detect mismatches:

627 -

628 -```ruby

629 -if Onde::VERSION != Onde::NATIVE_VERSION

630 - warn "Onde version mismatch: gem=#{Onde::VERSION} native=#{Onde::NATIVE_VERSION}"

631 -end

632 -```

docs/swift-package.md deleted

-328

@@ -1,328 +0,0 @@

1 ----

2 -title: "Swift Package"

3 -description: "Build and distribute the Onde Swift package (XCFramework) for iOS, tvOS, and macOS using UniFFI bindings."

4 ----

5 -

6 -# Swift Package

7 -

8 -The Onde Swift package distributes a pre-built `OndeFramework.xcframework` via Swift Package Manager. The framework is built from Rust source using UniFFI bindings — no `cargo-swift` required. The project owns its own `uniffi-bindgen` binary that generates the Swift glue code directly.

9 -

10 -## Installation

11 -

12 -### Swift Package Manager

13 -

14 -Add the package in Xcode: **File → Add Package Dependencies** and enter:

15 -

16 -```

17 -https://github.com/ondeinference/onde-swift

18 -```

19 -

20 -Or add it to your `Package.swift`:

21 -

22 -```swift

23 -dependencies: [

24 - .package(url: "https://github.com/ondeinference/onde-swift", from: "0.1.0")

25 -],

26 -targets: [

27 - .target(name: "MyApp", dependencies: ["Onde"])

28 -]

29 -```

30 -

31 -## Supported Platforms

32 -

33 -| Platform | Minimum Version | GPU Backend |

34 -| -------- | --------------- | ----------- |

35 -| iOS | 14.0 | Metal |

36 -| tvOS | 13.0 | Metal |

37 -| macOS | 12.0 | Metal |

38 -

39 -## Swift API

40 -

41 -### Basic Chat

42 -

43 -```swift

44 -import Onde

45 -

46 -// Create the engine

47 -let engine = OndeChatEngine()

48 -

49 -// Load the platform-appropriate default model:

50 -// iOS / tvOS → Qwen 2.5 1.5B (~941 MB)

51 -// macOS → Qwen 2.5 3B (~1.93 GB)

52 -let loadTimeSecs = try await engine.loadDefaultModel(

53 - systemPrompt: "You are a helpful assistant.",

54 - sampling: nil // uses platform-aware defaults

55 -)

56 -print("Model loaded in \(loadTimeSecs)s")

57 -

58 -// Multi-turn chat (history managed automatically)

59 -let result = try await engine.sendMessage(message: "What can you help me with?")

60 -print(result.text)

61 -print(result.durationDisplay) // e.g. "4.5s"

62 -

63 -let followUp = try await engine.sendMessage(message: "Tell me more about that.")

64 -print(followUp.text)

65 -

66 -// Unload model to free memory

67 -await engine.unloadModel()

68 -```

69 -

70 -### Streaming

71 -

72 -```swift

73 -import Onde

74 -

75 -class StreamHandler: StreamChunkListener {

76 - func onChunk(chunk: StreamChunk) -> Bool {

77 - print(chunk.delta, terminator: "")

78 - if chunk.done { print() }

79 - return !chunk.done // return false to cancel early

80 - }

81 -}

82 -

83 -let engine = OndeChatEngine()

84 -try await engine.loadDefaultModel(systemPrompt: nil, sampling: nil)

85 -

86 -try await streamChatMessage(

87 - engine: engine,

88 - message: "Tell me a story.",

89 - listener: StreamHandler()

90 -)

91 -```

92 -

93 -Streaming is exposed as a free function rather than an `OndeChatEngine` method because UniFFI 0.31 does not support `callback_interface` parameters on Object methods.

94 -

95 -### One-Shot Generation

96 -

97 -`generate` runs inference on an explicit message list without modifying the engine's conversation history. Useful for prompt enhancement, summarisation, or any case where you don't want side effects on the chat history.

98 -

99 -```swift

100 -import Onde

101 -

102 -let engine = OndeChatEngine()

103 -try await engine.loadDefaultModel(systemPrompt: nil, sampling: nil)

104 -

105 -let result = try await engine.generate(

106 - messages: [userMessage(content: "Expand into a detailed prompt: a cat in space")],

107 - sampling: deterministicSamplingConfig()

108 -)

109 -print(result.text)

110 -```

111 -

112 -### Custom Model and Sampling

113 -

114 -```swift

115 -import Onde

116 -

117 -let engine = OndeChatEngine()

118 -

119 -let sampling = SamplingConfig(

120 - temperature: 0.9,

121 - topP: 0.95,

122 - topK: nil,

123 - minP: nil,

124 - maxTokens: 256,

125 - frequencyPenalty: nil,

126 - presencePenalty: nil

127 -)

128 -

129 -try await engine.loadGgufModel(

130 - config: qwen253bConfig(),

131 - systemPrompt: "You are a music expert.",

132 - sampling: sampling

133 -)

134 -```

135 -

136 -### History and Status

137 -

138 -```swift

139 -// Check engine status

140 -let info = await engine.info()

141 -print(info.status) // .ready

142 -print(info.historyLength) // number of turns

143 -

144 -// Retrieve full conversation history

145 -let history = await engine.history()

146 -for msg in history {

147 - print("\(msg.role): \(msg.content)")

148 -}

149 -

150 -// Clear history without unloading the model

151 -let removed = await engine.clearHistory()

152 -print("Removed \(removed) turns")

153 -

154 -// Append a message without running inference

155 -await engine.pushHistory(message: userMessage(content: "..."))

156 -```

157 -

158 -## Free Functions Reference

159 -

160 -### Model Configurations

161 -

162 -```swift

163 -defaultModelConfig() // platform-appropriate default (1.5B on iOS/tvOS, 3B on macOS)

164 -qwen251_5bConfig() // Qwen 2.5 1.5B Q4_K_M (~941 MB)

165 -qwen253bConfig() // Qwen 2.5 3B Q4_K_M (~1.93 GB)

166 -```

167 -

168 -### Sampling Presets

169 -

170 -```swift

171 -defaultSamplingConfig() // temp=0.7, top_p=0.95, max_tokens=512

172 -deterministicSamplingConfig() // temp=0.0, greedy decoding

173 -mobileSamplingConfig() // temp=0.7, max_tokens=128

174 -```

175 -

176 -### Message Constructors

177 -

178 -```swift

179 -systemMessage(content:) // ChatRole.system

180 -userMessage(content:) // ChatRole.user

181 -assistantMessage(content:) // ChatRole.assistant

182 -```

183 -

184 -## Type Reference

185 -

186 -| Rust Type | Swift Type | Description |

187 -| --------- | ---------- | ----------- |

188 -| `ChatRole` | `enum ChatRole` | `.system`, `.user`, `.assistant` |

189 -| `ChatMessage` | `struct ChatMessage` | `role` + `content` |

190 -| `SamplingConfig` | `struct SamplingConfig` | Temperature, top-p, max tokens, etc. |

191 -| `GgufModelConfig` | `struct GgufModelConfig` | Model repo, filename, display name |

192 -| `InferenceResult` | `struct InferenceResult` | `text`, `durationSecs`, `durationDisplay`, `finishReason` |

193 -| `StreamChunk` | `struct StreamChunk` | `delta`, `done`, `finishReason` |

194 -| `EngineStatus` | `enum EngineStatus` | `.unloaded`, `.loading`, `.ready`, `.generating`, `.error` |

195 -| `EngineInfo` | `struct EngineInfo` | `status`, `modelName`, `approxMemory`, `historyLength` |

196 -| `InferenceError` | `enum InferenceError: Error` | Thrown by `loadGgufModel`, `sendMessage`, `generate`, `streamChatMessage` |

197 -| `OndeChatEngine` | `class OndeChatEngine` | Main inference engine, thread-safe |

198 -| `StreamChunkListener` | `protocol StreamChunkListener` | Implement `onChunk(chunk:) -> Bool` |

199 -

200 -## Sandboxed App Setup (iOS / macOS App Store)

201 -

202 -On sandboxed platforms, `~/.cache` is inaccessible. Before any inference code runs, configure `HF_HOME` and `TMPDIR` to point inside the app's container:

203 -

204 -```swift

205 -import Foundation

206 -

207 -func setupInferenceEnvironment() {

208 - let appSupport = FileManager.default

209 - .urls(for: .applicationSupportDirectory, in: .userDomainMask)[0]

210 -

211 - let hfHome = appSupport.appendingPathComponent("huggingface")

212 - let hfHub = hfHome.appendingPathComponent("hub")

213 - try? FileManager.default.createDirectory(at: hfHub, withIntermediateDirectories: true)

214 -

215 - setenv("HF_HOME", hfHome.path, 1)

216 - setenv("HF_HUB_CACHE", hfHub.path, 1)

217 -

218 - // mistral.rs uses TMPDIR for temporary files during model loading

219 - let tmp = appSupport.appendingPathComponent("tmp")

220 - try? FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true)

221 - setenv("TMPDIR", tmp.path, 1)

222 -}

223 -```

224 -

225 -Call this once at app launch, before calling any `OndeChatEngine` method.

226 -

227 -## Building the XCFramework Locally

228 -

229 -If you are contributing to Onde or need a custom build, you can assemble the `OndeFramework.xcframework` from source.

230 -

231 -### Prerequisites

232 -

233 -```bash

234 -# Stable toolchain for iOS and macOS targets

235 -rustup toolchain install stable

236 -

237 -# Nightly toolchain for tvOS (tier-3 targets require -Z build-std)

238 -rustup toolchain install nightly

239 -rustup component add rust-src --toolchain nightly

240 -

241 -# Apple targets

242 -rustup target add \

243 - aarch64-apple-ios \

244 - x86_64-apple-ios \

245 - aarch64-apple-ios-sim \

246 - aarch64-apple-darwin \

247 - x86_64-apple-darwin

248 -```

249 -

250 -### Build

251 -

252 -```bash

253 -# 1. Build the uniffi-bindgen CLI (host)

254 -cargo build --manifest-path uniffi-bindgen/Cargo.toml --release

255 -BINDGEN=uniffi-bindgen/target/release/uniffi-bindgen

256 -

257 -# 2. Compile staticlibs per target

258 -cargo build --target aarch64-apple-ios --release

259 -cargo build --target x86_64-apple-ios --release

260 -cargo build --target aarch64-apple-ios-sim --release

261 -cargo build --target aarch64-apple-darwin --release

262 -cargo build --target x86_64-apple-darwin --release

263 -cargo +nightly build -Z build-std --target aarch64-apple-tvos --release

264 -cargo +nightly build -Z build-std --target x86_64-apple-tvos --release

265 -cargo +nightly build -Z build-std --target aarch64-apple-tvos-sim --release

266 -

267 -# 3. lipo simulator and macOS slices

268 -lipo -create \

269 - target/x86_64-apple-ios/release/libonde.a \

270 - target/aarch64-apple-ios-sim/release/libonde.a \

271 - -output /tmp/libonde-ios-sim.a

272 -

273 -lipo -create \

274 - target/x86_64-apple-tvos/release/libonde.a \

275 - target/aarch64-apple-tvos-sim/release/libonde.a \

276 - -output /tmp/libonde-tvos-sim.a

277 -

278 -lipo -create \

279 - target/aarch64-apple-darwin/release/libonde.a \

280 - target/x86_64-apple-darwin/release/libonde.a \

281 - -output /tmp/libonde-macos.a

282 -

283 -# 4. Generate Swift bindings

284 -mkdir -p sdk/Onde/Sources/Onde

285 -$BINDGEN generate \

286 - --library target/aarch64-apple-ios/release/libonde.a \

287 - --language swift \

288 - --out-dir sdk/Onde/Sources/Onde \

289 - --config uniffi.toml

290 -

291 -# 5. Assemble the XCFramework

292 -HEADERS=sdk/Onde/Sources/Onde

293 -xcodebuild -create-xcframework \

294 - -library target/aarch64-apple-ios/release/libonde.a -headers $HEADERS \

295 - -library /tmp/libonde-ios-sim.a -headers $HEADERS \

296 - -library target/aarch64-apple-tvos/release/libonde.a -headers $HEADERS \

297 - -library /tmp/libonde-tvos-sim.a -headers $HEADERS \

298 - -library /tmp/libonde-macos.a -headers $HEADERS \

299 - -output sdk/Onde/OndeFramework.xcframework

300 -```

301 -

302 -### Local Xcode Integration

303 -

304 -During development, use the local path form in `Package.swift`:

305 -

306 -```swift

307 -.binaryTarget(name: "OndeFramework", path: "./OndeFramework.xcframework")

308 -```

309 -

310 -In Xcode: **File → Add Package Dependencies → Add Local** → select `sdk/Onde/`.

311 -

312 -## Known Issues

313 -

314 -### `___chkstk_darwin` linker error on tvOS

315 -

316 -`aws-lc-sys` (a transitive dependency via `hf-hub`) references `___chkstk_darwin`, a stack probing symbol that tvOS does not export. The `build.rs` in the onde crate provides a no-op assembly stub (`tvos_chkstk.s`) that resolves this automatically. No action is needed — ensure `tvos_chkstk.s` is present at the crate root.

317 -

318 -This does not affect iOS, macOS, Android, Windows, or Linux builds.

319 -

320 -### Metal Toolchain missing (Xcode 26+)

321 -

322 -On Xcode 26+, the Metal compiler is a separate downloadable component. If you see empty `.metallib` files (92 bytes) or `"Error while loading function: fused_glu_float"` at runtime:

323 -

324 -```bash

325 -xcodebuild -downloadComponent MetalToolchain

326 -cargo clean -p mistralrs-quant

327 -cargo check

328 -```

scripts/tvos_chkstk.s renamed

src/hf_cache.rs

@@ -243,6 +243,7 @@ fn list_revisions(model_dir: &PathBuf) -> Vec<String> {

243 }

244

245 /// Build a [`ModelDownloadProgress`] payload from raw byte counts.

246 +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]

247 fn make_progress(model_id: &str, downloaded: u64, total: u64, done: bool) -> ModelDownloadProgress {

248 let progress = if total > 0 {

249 (downloaded as f64 / total as f64).min(1.0)

src/inference/engine.rs

+8 -3

@@ -88,9 +88,11 @@ use super::types::*;

88 target_os = "linux",

89 target_os = "android"

90 ))]

91 -use mistralrs::{

92 - GgufModelBuilder, IsqBits, Model, RequestBuilder, TextMessageRole, TextModelBuilder,

93 -};

91 +use mistralrs::{GgufModelBuilder, Model, RequestBuilder, TextMessageRole};

92 +

93 +// ISQ types are only used by load_isq_model, which is macOS-only.

94 +#[cfg(target_os = "macos")]

95 +use mistralrs::{IsqBits, TextModelBuilder};

97 // ── Internals ────────────────────────────────────────────────────────────────

@@ -105,6 +107,7 @@ use mistralrs::{

107 ))]

108 enum LoadedModelConfig {

109 Gguf(GgufModelConfig),

110 + #[cfg(target_os = "macos")]

111 Isq(IsqModelConfig),

112 }

113

@@ -120,6 +123,7 @@ impl LoadedModelConfig {

123 fn display_name(&self) -> &str {

124 match self {

125 LoadedModelConfig::Gguf(c) => &c.display_name,

126 + #[cfg(target_os = "macos")]

127 LoadedModelConfig::Isq(c) => &c.display_name,

128 }

129 }

@@ -127,6 +131,7 @@ impl LoadedModelConfig {

131 fn approx_memory(&self) -> &str {

132 match self {

133 LoadedModelConfig::Gguf(c) => &c.approx_memory,

134 + #[cfg(target_os = "macos")]

135 LoadedModelConfig::Isq(c) => &c.approx_memory,

136 }

137 }