@ondeinference / onde / commits / 25b4077

Add load_assigned_model to ChatEngine for SDK model assignment

- Adds ChatEngine::load_assigned_model to fetch and load the model assigned to an Onde app via the SDK model-config endpoint, authenticating with onde_app_id and onde_app_secret. - Falls back to platform default model on 404 or incomplete response. - Returns InferenceError::ModelBuild on network/HTTP errors. - Adds tok_model_id_for_repo to models.rs for Android tokenizer resolution. - Updates dependencies in Cargo.toml and Cargo.lock for smbcloud-gresiq-sdk.

keypair34 committed Apr 18, 2026 at 18:30 UTC 25b40773867cfcf56b6a6c8f2e467c572dd2768c
4 files changed +252 -3
Cargo.lock
+111 -3
@@ -1280,7 +1280,7 @@ dependencies = [
1280 "bytemuck_derive",
1281 "hashbrown 0.15.5",
1282 "regex-syntax",
1283 - "strum",
1283 + "strum 0.27.2",
1284 ]
1285
1286 [[package]]
@@ -2201,6 +2201,12 @@ dependencies = [
2201 "unicode-segmentation",
2202 ]
2203
2204 +[[package]]
2205 +name = "heck"
2206 +version = "0.4.1"
2207 +source = "registry+https://github.com/rust-lang/crates.io-index"
2208 +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
2209 +
2210 [[package]]
2211 name = "heck"
2212 version = "0.5.0"
@@ -2368,6 +2374,7 @@ dependencies = [
2374 "hyper",
2375 "hyper-util",
2376 "rustls",
2377 + "rustls-native-certs",
2378 "rustls-pki-types",
2379 "tokio",
2380 "tokio-rustls",
@@ -2937,6 +2944,12 @@ dependencies = [
2944 "libc",
2945 ]
2946
2947 +[[package]]
2948 +name = "maplit"
2949 +version = "1.0.2"
2950 +source = "registry+https://github.com/rust-lang/crates.io-index"
2951 +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
2952 +
2953 [[package]]
2954 name = "markup5ever"
2955 version = "0.36.1"
@@ -3205,7 +3218,7 @@ dependencies = [
3218 "serde_json",
3219 "serde_plain",
3220 "statrs",
3208 - "strum",
3221 + "strum 0.27.2",
3222 "symphonia",
3223 "sysinfo",
3224 "thiserror 2.0.18",
@@ -3627,7 +3640,9 @@ dependencies = [
3640 "log",
3641 "mistralrs",
3642 "mistralrs-core",
3643 + "reqwest 0.12.28",
3644 "serde",
3645 + "smbcloud-gresiq-sdk",
3646 "thiserror 2.0.18",
3647 "tokio",
3648 "tsync",
@@ -4388,6 +4403,7 @@ dependencies = [
4403 "pin-project-lite",
4404 "quinn",
4405 "rustls",
4406 + "rustls-native-certs",
4407 "rustls-pki-types",
4408 "serde",
4409 "serde_json",
@@ -4934,6 +4950,17 @@ dependencies = [
4950 "serde",
4951 ]
4952
4953 +[[package]]
4954 +name = "serde_repr"
4955 +version = "0.1.20"
4956 +source = "registry+https://github.com/rust-lang/crates.io-index"
4957 +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
4958 +dependencies = [
4959 + "proc-macro2",
4960 + "quote",
4961 + "syn 2.0.117",
4962 +]
4963 +
4964 [[package]]
4965 name = "serde_spanned"
4966 version = "1.1.1"
@@ -5121,6 +5148,54 @@ version = "0.3.2"
5148 source = "registry+https://github.com/rust-lang/crates.io-index"
5149 checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
5150
5151 +[[package]]
5152 +name = "smbcloud-gresiq-sdk"
5153 +version = "0.3.34"
5154 +source = "registry+https://github.com/rust-lang/crates.io-index"
5155 +checksum = "ad2c1b5ed77504ba145e7281a6622f961cb5fda0797d9c36ef4141cc447fce44"
5156 +dependencies = [
5157 + "log",
5158 + "reqwest 0.12.28",
5159 + "serde",
5160 + "serde_json",
5161 + "smbcloud-network",
5162 + "thiserror 2.0.18",
5163 + "tokio",
5164 +]
5165 +
5166 +[[package]]
5167 +name = "smbcloud-model"
5168 +version = "0.3.35"
5169 +source = "registry+https://github.com/rust-lang/crates.io-index"
5170 +checksum = "eeba5e1371b12e9573e6eba49ea99ea990ec1b6befe7bfcafaf41b43e1cf331d"
5171 +dependencies = [
5172 + "chrono",
5173 + "log",
5174 + "serde",
5175 + "serde_json",
5176 + "serde_repr",
5177 + "spinners",
5178 + "strum 0.27.2",
5179 + "strum_macros 0.27.2",
5180 + "thiserror 2.0.18",
5181 + "tsync",
5182 +]
5183 +
5184 +[[package]]
5185 +name = "smbcloud-network"
5186 +version = "0.3.35"
5187 +source = "registry+https://github.com/rust-lang/crates.io-index"
5188 +checksum = "1c236db19d4cea01cf48b584838fcc898745514c8e01c1c0cad0cba976cabefb"
5189 +dependencies = [
5190 + "clap 4.6.0",
5191 + "log",
5192 + "reqwest 0.12.28",
5193 + "serde",
5194 + "serde_json",
5195 + "smbcloud-model",
5196 + "wasm-bindgen",
5197 +]
5198 +
5199 [[package]]
5200 name = "socket2"
5201 version = "0.6.3"
@@ -5153,6 +5228,17 @@ dependencies = [
5228 "vob",
5229 ]
5230
5231 +[[package]]
5232 +name = "spinners"
5233 +version = "4.2.0"
5234 +source = "registry+https://github.com/rust-lang/crates.io-index"
5235 +checksum = "071af1a9d34b78b8db3ca4424b0ea4d87052d607dbe96287aebaccd596cabc86"
5236 +dependencies = [
5237 + "lazy_static",
5238 + "maplit",
5239 + "strum 0.24.1",
5240 +]
5241 +
5242 [[package]]
5243 name = "spm_precompiled"
5244 version = "0.1.4"
@@ -5279,13 +5365,35 @@ dependencies = [
5365 "syn 1.0.109",
5366 ]
5367
5368 +[[package]]
5369 +name = "strum"
5370 +version = "0.24.1"
5371 +source = "registry+https://github.com/rust-lang/crates.io-index"
5372 +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
5373 +dependencies = [
5374 + "strum_macros 0.24.3",
5375 +]
5376 +
5377 [[package]]
5378 name = "strum"
5379 version = "0.27.2"
5380 source = "registry+https://github.com/rust-lang/crates.io-index"
5381 checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
5382 dependencies = [
5288 - "strum_macros",
5383 + "strum_macros 0.27.2",
5384 +]
5385 +
5386 +[[package]]
5387 +name = "strum_macros"
5388 +version = "0.24.3"
5389 +source = "registry+https://github.com/rust-lang/crates.io-index"
5390 +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
5391 +dependencies = [
5392 + "heck 0.4.1",
5393 + "proc-macro2",
5394 + "quote",
5395 + "rustversion",
5396 + "syn 1.0.109",
5397 ]
5398
5399 [[package]]
Cargo.toml
+3
@@ -25,10 +25,13 @@ anyhow = "1"
25 home = "0.5.4"
26 log = "^0.4"
27 serde = { version = "1.0", features = ["derive"] }
28 +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
29 tokio = { version = "1", features = ["macros", "rt", "sync"] }
30 tsync = "2"
31
32 thiserror = "2"
33 +smbcloud-gresiq-sdk = { version = "0.3" }
34 +# smbcloud-gresiq-sdk = { path = "../smbcloud-cli/crates/smbcloud-gresiq-sdk" }
35
36 # Pinned to =0.31.0 — the versions MUST be identical across the crate and
37 # the uniffi-bindgen binary crate.
src/inference/engine.rs
+122
@@ -210,6 +210,8 @@ pub struct ChatEngine {
210 target_os = "linux",
211 target_os = "android"
212 ))]
213 +
214 +
215 impl ChatEngine {
216 // ── Construction ─────────────────────────────────────────────────────
217
@@ -358,6 +360,126 @@ impl ChatEngine {
360 Ok(elapsed)
361 }
362
363 + /// Fetch the model assigned to this Onde app from the SDK model-config
364 + /// endpoint and load it.
365 + ///
366 + /// Authenticates using the app's own `onde_app_id` + `onde_app_secret`
367 + /// (the SDK credentials shown in the ondeinference.com dashboard).
368 + /// No end-user JWT is required — model assignment is an operator-level
369 + /// configuration that is independent of which user is currently signed in.
370 + ///
371 + /// **Fallback behaviour:**
372 + /// - HTTP 404 (no model assigned) → loads [`GgufModelConfig::platform_default()`].
373 + /// - Missing `hf_repo_id` or `gguf_file` in response → loads platform default.
374 + /// - Any HTTP or network error → returns [`InferenceError::ModelBuild`] so
375 + /// the caller can apply its own fallback.
376 + pub async fn load_assigned_model(
377 + &self,
378 + environment: smbcloud_gresiq_sdk::Environment,
379 + onde_app_id: &str,
380 + onde_app_secret: &str,
381 + system_prompt: Option<String>,
382 + sampling: Option<SamplingConfig>,
383 + ) -> Result<std::time::Duration, InferenceError> {
384 + use log::{info, warn};
385 +
386 + #[derive(serde::Deserialize)]
387 + struct ModelConfigResponse {
388 + hf_repo_id: Option<String>,
389 + gguf_file: Option<String>,
390 + name: Option<String>,
391 + approx_size_bytes: Option<i64>,
392 + }
393 +
394 + let url = format!(
395 + "{}://{}/v1/client/onde_sdk/model_config?app_id={}&app_secret={}",
396 + environment.api_protocol(),
397 + environment.api_host(),
398 + onde_app_id,
399 + onde_app_secret,
400 + );
401 +
402 + let response = reqwest::Client::new()
403 + .get(&url)
404 + .header("Content-Type", "application/json")
405 + .send()
406 + .await
407 + .map_err(|e| InferenceError::ModelBuild {
408 + reason: format!("SDK model_config request failed: {e}"),
409 + })?;
410 +
411 + if response.status().as_u16() == 404 {
412 + warn!(
413 + "ChatEngine: no model assigned to Onde app {onde_app_id}; loading platform default."
414 + );
415 + return self
416 + .load_gguf_model(GgufModelConfig::platform_default(), system_prompt, sampling)
417 + .await;
418 + }
419 +
420 + if !response.status().is_success() {
421 + return Err(InferenceError::ModelBuild {
422 + reason: format!(
423 + "SDK model_config returned HTTP {}",
424 + response.status().as_u16()
425 + ),
426 + });
427 + }
428 +
429 + let resp: ModelConfigResponse = response
430 + .json()
431 + .await
432 + .map_err(|e| InferenceError::ModelBuild {
433 + reason: format!("Failed to parse model_config response: {e}"),
434 + })?;
435 +
436 + let hf_repo_id = resp.hf_repo_id.as_deref().unwrap_or_default();
437 + let gguf_file = resp.gguf_file.as_deref().unwrap_or_default();
438 +
439 + if hf_repo_id.is_empty() || gguf_file.is_empty() {
440 + warn!(
441 + "ChatEngine: model_config response missing hf_repo_id or gguf_file; loading platform default."
442 + );
443 + return self
444 + .load_gguf_model(GgufModelConfig::platform_default(), system_prompt, sampling)
445 + .await;
446 + }
447 +
448 + #[cfg(target_os = "android")]
449 + let tok_model_id = super::models::tok_model_id_for_repo(hf_repo_id)
450 + .map(|s| s.to_string());
451 + #[cfg(not(target_os = "android"))]
452 + let tok_model_id: Option<String> = None;
453 +
454 + let approx_memory = resp
455 + .approx_size_bytes
456 + .map(|b| {
457 + let gb = b as f64 / 1_073_741_824.0;
458 + if gb >= 1.0 {
459 + format!("~{:.2} GB", gb)
460 + } else {
461 + format!("~{} MB", b / 1_048_576)
462 + }
463 + })
464 + .unwrap_or_else(|| "—".to_string());
465 +
466 + info!(
467 + "ChatEngine: resolved SDK model assignment → {} / {} ({})",
468 + hf_repo_id, gguf_file, approx_memory
469 + );
470 +
471 + let config = GgufModelConfig {
472 + model_id: hf_repo_id.to_string(),
473 + files: vec![gguf_file.to_string()],
474 + tok_model_id,
475 + display_name: resp.name.unwrap_or_else(|| hf_repo_id.to_string()),
476 + approx_memory,
477 + };
478 +
479 + self.load_gguf_model(config, system_prompt, sampling).await
480 + }
481 +
482 +
483 /// Load an ISQ (in-situ quantised) model into the engine.
484 ///
485 /// Unlike [`load_gguf_model`], this downloads the full-precision safetensors
src/inference/models.rs
+16
@@ -139,3 +139,19 @@ pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[
139 expected_size_bytes: 2_596_306_912,
140 },
141 ];
142 +
143 +/// Return the explicit tokenizer model ID required on Android for `hf_repo_id`.
144 +///
145 +/// The candle GGUF backend cannot parse the tokenizer embedded inside GGUF
146 +/// files; an explicit `tok_model_id` triggers a separate tokenizer download
147 +/// from the base model repo. Returns `None` on iOS and macOS where the
148 +/// embedded tokenizer is used automatically.
149 +pub fn tok_model_id_for_repo(hf_repo_id: &str) -> Option<&'static str> {
150 + match hf_repo_id {
151 + BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF => Some(QWEN25_1_5B_TOK_MODEL_ID),
152 + BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF => Some(QWEN25_3B_TOK_MODEL_ID),
153 + BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF => Some(QWEN25_CODER_1_5B_TOK_MODEL_ID),
154 + BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF => Some(QWEN25_CODER_3B_TOK_MODEL_ID),
155 + _ => None,
156 + }
157 +}