@ondeinference / onde / commits / 1c72793

Fix running on tvOS

Seto Elkahfi committed May 22, 2026 at 21:53 UTC 1c72793776d84d49a1e29d761bd755ebfcf2bdae

4 files changed +231 -2

src/hf_cache.rs

+112

@@ -677,6 +677,106 @@ pub fn repair_hf_cache_symlinks(model_id: &str) {

677 }

678 }

679

680 +/// Ensure snapshot entries exist for known GGUF files.

681 +///

682 +/// When a download completes but the process is killed before hf-hub creates

683 +/// the snapshot symlink, the blob exists but hf-hub considers it a cache miss

684 +/// and re-downloads. This function creates hard links from the snapshot

685 +/// directory to the blob directory for any missing entries.

686 +pub fn ensure_snapshot_entries(model_id: &str, expected_files: &[&str]) {

687 + let cache_path = match model_cache_path(model_id) {

688 + Some(p) if p.is_dir() => p,

689 + _ => return,

690 + };

691 +

692 + let blobs_dir = cache_path.join("blobs");

693 + let snapshots_dir = cache_path.join("snapshots");

694 + let refs_dir = cache_path.join("refs");

695 +

696 + if !blobs_dir.is_dir() {

697 + return;

698 + }

699 +

700 + // Collect non-lock, non-part blob files.

701 + let blob_entries: Vec<(String, u64)> = match fs::read_dir(&blobs_dir) {

702 + Ok(entries) => entries

703 + .flatten()

704 + .filter_map(|e| {

705 + let name = e.file_name().to_string_lossy().to_string();

706 + if name.ends_with(".lock") || name.ends_with(".part") {

707 + return None;

708 + }

709 + let meta = e.metadata().ok()?;

710 + if meta.is_file() && meta.len() > 0 {

711 + Some((name, meta.len()))

712 + } else {

713 + None

714 + }

715 + })

716 + .collect(),

717 + Err(_) => return,

718 + };

719 +

720 + if blob_entries.is_empty() {

721 + return;

722 + }

723 +

724 + // Find the revision from refs/main, or from an existing snapshot dir.

725 + let revision = fs::read_to_string(refs_dir.join("main"))

726 + .ok()

727 + .map(|s| s.trim().to_string())

728 + .or_else(|| {

729 + fs::read_dir(&snapshots_dir)

730 + .ok()?

731 + .flatten()

732 + .find(|e| e.path().is_dir())

733 + .map(|e| e.file_name().to_string_lossy().to_string())

734 + });

735 +

736 + let revision = match revision {

737 + Some(r) if !r.is_empty() => r,

738 + _ => return,

739 + };

740 +

741 + let snap_dir = snapshots_dir.join(&revision);

742 + let _ = fs::create_dir_all(&snap_dir);

743 +

744 + for expected_file in expected_files {

745 + let snap_file = snap_dir.join(expected_file);

746 + if snap_file.exists() {

747 + continue;

748 + }

749 +

750 + // For single-blob repos, link the only blob. For multi-blob repos,

751 + // we can't determine the mapping without metadata.

752 + let blob_to_link = if blob_entries.len() == 1 {

753 + Some(&blob_entries[0].0)

754 + } else {

755 + None

756 + };

757 +

758 + if let Some(blob_name) = blob_to_link {

759 + let blob_path = blobs_dir.join(blob_name);

760 + if fs::hard_link(&blob_path, &snap_file).is_ok() {

761 + info!(

762 + "ensure_snapshot_entries: hard-linked {} → {}",

763 + expected_file, blob_name

764 + );

765 + } else if fs::copy(&blob_path, &snap_file).is_ok() {

766 + info!(

767 + "ensure_snapshot_entries: copied {} → {}",

768 + expected_file, blob_name

769 + );

770 + } else {

771 + warn!(

772 + "ensure_snapshot_entries: failed to link/copy blob for {}",

773 + expected_file

774 + );

775 + }

776 + }

777 + }

778 +}

779 +

780 // ---------------------------------------------------------------------------

781 // Query / mutation functions

782 // ---------------------------------------------------------------------------

@@ -952,6 +1052,18 @@ where

1052 repair_hf_cache_symlinks(&model_id);

1053 ensure_model_cache_dir(&model_id);

1054

1055 + // Recover from interrupted downloads where the blob exists but the

1056 + // snapshot entry was never created (process killed after download,

1057 + // before hf-hub finalised the cache structure).

1058 + let expected_files: Vec<&str> = match model_id.as_str() {

1059 + BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF => vec![QWEN25_1_5B_GGUF_FILE],

1060 + BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF => vec![QWEN25_3B_GGUF_FILE],

1061 + _ => vec![],

1062 + };

1063 + if !expected_files.is_empty() {

1064 + ensure_snapshot_entries(&model_id, &expected_files);

1065 + }

1066 +

1067 // Emit an initial progress event based on the actual cache size so that

1068 // resumed downloads don't visually restart from 0%.

1069 let initial_bytes = model_cache_path(&model_id)

src/inference/engine.rs

+46 -2

@@ -391,12 +391,37 @@ impl ChatEngine {

391 crate::hf_cache::clean_stale_lock_files(&config.model_id);

392 crate::hf_cache::repair_hf_cache_symlinks(&config.model_id);

393

394 + // If the blob was downloaded but the process was killed before hf-hub

395 + // created the snapshot entry, hf-hub will re-download. Create the

396 + // missing snapshot hard links so the cache hit succeeds.

397 + let file_refs: Vec<&str> = config.files.iter().map(|s| s.as_str()).collect();

398 + crate::hf_cache::ensure_snapshot_entries(&config.model_id, &file_refs);

399 +

400 let start = Instant::now();

401

402 let mut builder = GgufModelBuilder::new(&config.model_id, config.files.clone())

403 .with_token_source(super::token::hf_token_source())

404 .with_logging();

405

406 + // tvOS: Apple TV 4K has only 4 GB RAM. Reduce memory footprint by

407 + // limiting concurrent sequences, disabling the prefix cache,

408 + // disabling the KV cache, and capping context length at 512 tokens

409 + // so the auto device mapper allocates far less memory.

410 + #[cfg(target_os = "tvos")]

411 + {

412 + use mistralrs_core::AutoDeviceMapParams;

413 + use mistralrs::DeviceMapSetting;

414 +

415 + builder = builder

416 + .with_max_num_seqs(1)

417 + .with_prefix_cache_n(None)

418 + .with_no_kv_cache()

419 + .with_device_mapping(DeviceMapSetting::Auto(AutoDeviceMapParams::Text {

420 + max_seq_len: 512,

421 + max_batch_size: 1,

422 + }));

423 + }

424 +

425 // On Android the GGUF embedded tokenizer is not supported by the

426 // candle backend — an explicit tok_model_id is required.

427 if let Some(ref tok_id) = config.tok_model_id {

@@ -1896,6 +1921,24 @@ const DEEPSEEK_CODER_CHAT_TEMPLATE: &str = include_str!("deepseek_coder_chat_tem

1921 /// These mirror the constants in [`super::models`] but return a ready-to-use

1922 /// [`GgufModelConfig`].

1923 impl GgufModelConfig {

1924 + /// Qwen 2.5 0.5B Instruct (GGUF Q4_K_M), ~380 MB.

1925 + ///

1926 + /// Smallest option. Used on tvOS (Apple TV) where the memory limit is 2 GB.

1927 + pub fn qwen25_0_5b() -> Self {

1928 + Self {

1929 + model_id: super::models::BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF.into(),

1930 + files: vec![super::models::QWEN25_0_5B_GGUF_FILE.into()],

1931 + tok_model_id: if cfg!(target_os = "android") {

1932 + Some(super::models::QWEN25_0_5B_TOK_MODEL_ID.into())

1933 + } else {

1934 + None

1935 + },

1936 + display_name: "Qwen 2.5 0.5B".into(),

1937 + approx_memory: "~380 MB (GGUF Q4_K_M)".into(),

1938 + chat_template: None,

1939 + }

1940 + }

1941 +

1942 /// Qwen 2.5 1.5B Instruct (GGUF Q4_K_M) — ~941 MB.

1943 ///

1944 /// Lightest option, ideal for iOS and memory-constrained Android devices.

@@ -2079,9 +2122,10 @@ impl GgufModelConfig {

2122 /// Both are dedicated coding models (trained on 5.5T code + math tokens)

2123 /// and use the `qwen2` GGUF architecture supported by mistral.rs.

2124 pub fn platform_default() -> Self {

2082 - if cfg!(any(

2125 + if cfg!(target_os = "tvos") {

2126 + Self::qwen25_0_5b()

2127 + } else if cfg!(any(

2128 target_os = "ios",

2084 - target_os = "tvos",

2129 target_os = "visionos",

2130 target_os = "watchos",

2131 target_os = "android"

src/inference/ffi.rs

+56

@@ -308,6 +308,12 @@ pub fn default_model_config() -> GgufModelConfig {

308 GgufModelConfig::platform_default()

309 }

310

311 +/// Return the Qwen 2.5 0.5B GGUF model configuration (~380 MB).

312 +#[uniffi::export]

313 +pub fn qwen25_0_5b_config() -> GgufModelConfig {

314 + GgufModelConfig::qwen25_0_5b()

315 +}

316 +

317 /// Return the Qwen 2.5 1.5B GGUF model configuration (~941 MB).

318 #[uniffi::export]

319 pub fn qwen25_1_5b_config() -> GgufModelConfig {

@@ -356,6 +362,41 @@ pub fn assistant_message(content: String) -> ChatMessage {

362 ChatMessage::assistant(content)

363 }

364

365 +/// Configure the HF cache directory for sandboxed platforms.

366 +///

367 +/// Call this before any model-loading method (`load_gguf_model`,

368 +/// `load_default_model`, etc.) on iOS, tvOS, and Android where the

369 +/// default `~/.cache/huggingface` path is inaccessible.

370 +///

371 +/// `path` should be a writable app data directory (e.g. the App Group

372 +/// container or Application Support directory on Apple platforms).

373 +///

374 +/// This sets the `HF_HOME` and `HF_HUB_CACHE` environment variables

375 +/// and creates the directory structure.

376 +#[uniffi::export]

377 +pub fn configure_cache_dir(path: String) {

378 + let base = std::path::PathBuf::from(&path);

379 + let hf_home = base.join("models");

380 + let hf_hub_cache = hf_home.join("hub");

381 +

382 + if let Err(e) = std::fs::create_dir_all(&hf_hub_cache) {

383 + log::warn!(

384 + "configure_cache_dir: could not create {}: {}",

385 + hf_hub_cache.display(),

386 + e

387 + );

388 + }

389 +

390 + std::env::set_var("HF_HOME", &hf_home);

391 + std::env::set_var("HF_HUB_CACHE", &hf_hub_cache);

392 +

393 + log::info!(

394 + "configure_cache_dir: HF_HOME={}, HF_HUB_CACHE={}",

395 + hf_home.display(),

396 + hf_hub_cache.display()

397 + );

398 +}

399 +

400 // ═══════════════════════════════════════════════════════════════════════════

401 // Tests

402 // ═══════════════════════════════════════════════════════════════════════════

@@ -401,6 +442,21 @@ mod tests {

442 assert_eq!(m.role, ChatRole::Assistant);

443 }

444

445 + #[test]

446 + fn configure_cache_dir_sets_env() {

447 + let tmp = std::env::temp_dir().join("onde-test-cache-dir");

448 + configure_cache_dir(tmp.to_string_lossy().to_string());

449 +

450 + let hf_home = std::env::var("HF_HOME").unwrap();

451 + assert!(hf_home.contains("models"));

452 +

453 + let hf_hub = std::env::var("HF_HUB_CACHE").unwrap();

454 + assert!(hf_hub.contains("hub"));

455 +

456 + // Cleanup

457 + let _ = std::fs::remove_dir_all(&tmp);

458 + }

459 +

460 #[tokio::test]

461 async fn onde_chat_engine_new_is_unloaded() {

462 let engine = OndeChatEngine::new();

src/inference/models.rs

+17

@@ -12,6 +12,14 @@ pub const QWEN25_1_5B_GGUF_FILE: &str = "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf";

12 /// embedded in the GGUF file is used instead to avoid an extra network download.

13 pub const QWEN25_1_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-1.5B-Instruct";

15 +/// Pre-quantized Qwen 2.5 0.5B Instruct (GGUF Q4_K_M), smallest model (~380 MB).

16 +/// Used on tvOS (Apple TV 4K) where the hard memory limit is 2 GB.

17 +pub const BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF: &str = "bartowski/Qwen2.5-0.5B-Instruct-GGUF";

18 +/// The specific GGUF filename to download from the bartowski 0.5B repo.

19 +pub const QWEN25_0_5B_GGUF_FILE: &str = "Qwen2.5-0.5B-Instruct-Q4_K_M.gguf";

20 +/// Base model repo used for the HF tokenizer on Android.

21 +pub const QWEN25_0_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-0.5B-Instruct";

22 +

23 /// Pre-quantized Qwen 2.5 Coder 1.5B Instruct (GGUF Q4_K_M) — dedicated coding model (~941 MB).

24 ///

25 /// Uses the `qwen2` GGUF architecture, identical to Qwen2.5-1.5B-Instruct, so it loads through

@@ -125,6 +133,7 @@ pub const DEEPSEEK_CODER_6_7B_TOK_MODEL_ID: &str = "deepseek-ai/deepseek-coder-6

133 /// Used by `list_local_hf_models` to filter the HuggingFace cache

134 /// to only show models that can actually be used for generation.

135 pub const SUPPORTED_MODELS: &[&str] = &[

136 + BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF,

137 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,

138 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,

139 BARTOWSKI_QWEN3_4B_GGUF,

@@ -160,6 +169,13 @@ pub struct SupportedModelInfo {

169 /// When adding a new model, add its constant ID to [`SUPPORTED_MODELS`] **and**

170 /// a corresponding entry here so the frontend can show it in the model list UI.

171 pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[

172 + SupportedModelInfo {

173 + id: BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF,

174 + name: "Qwen 2.5 0.5B (GGUF)",

175 + org: "Qwen / Alibaba",

176 + description: "Smallest pre-quantized chat model, used on tvOS Apple TV (~380 MB).",

177 + expected_size_bytes: 397_808_192,

178 + },

179 SupportedModelInfo {

180 id: BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,

181 name: "Qwen 2.5 1.5B (GGUF)",

@@ -242,6 +258,7 @@ pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[

258 /// embedded tokenizer is used automatically.

259 pub fn tok_model_id_for_repo(hf_repo_id: &str) -> Option<&'static str> {

260 match hf_repo_id {

261 + BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF => Some(QWEN25_0_5B_TOK_MODEL_ID),

262 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF => Some(QWEN25_1_5B_TOK_MODEL_ID),

263 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF => Some(QWEN25_3B_TOK_MODEL_ID),

264 BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF => Some(QWEN25_CODER_1_5B_TOK_MODEL_ID),