@setoelkahfi / sigit / commits / 99d4bfe

Add Local Inference on/off toggle with state-aware model picker

Introduce a persisted "Local Inference" setting that becomes the explicit local-vs-cloud mode switch and drives how the model picker is presented. - New src/settings.rs: TOML-backed preferences at $SIGIT_CONFIG_DIR/settings.toml (local_inference, default on / local-first), with a SIGIT_LOCAL_INFERENCE env override. Mirrors the credentials.rs storage pattern. - models.rs: group picker items by inference nature (Local vs Cloud) and sort the active mode's group first so the highlighted group leads the list. - TUI (chat.rs): /local [on|off] command, a "Local inference: ON/OFF" banner, and Local/Cloud group headers with the inactive group dimmed (still shown, so the cloud offering stays surfaced). Selecting a model from the inactive group flips the persisted mode to match. - ACP (main.rs): /local slash command, a second "Local Inference" select config option (On/Off) so panel-only clients without slash-command support can flip the mode, ConfigOptionUpdate refresh, and startup resolution that routes to a cloud tier when local inference is off and an account is signed in (falling back to on-device with a notice when not signed in). - Serialize the env-mutating credentials/settings unit tests behind a shared lock to avoid a parallel-test race on SIGIT_CONFIG_DIR. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016K7GDe9d6cLd9RkWYCFGmm

Claude committed Jun 29, 2026 at 16:42 UTC 99d4bfe91b0643d042fed0af0a84ffcec91740a1
5 files changed +526 -23
src/chat.rs
+119 -4
@@ -78,7 +78,9 @@ mod tui {
78 use onde::inference::{ChatEngine, SamplingConfig};
79
80 use crate::backend::{InferenceBackend, LocalBackend, OpenAiBackend, ToolResult, ToolSpec};
81 - use crate::models::{ModelCacheHealth, ModelPickerItem, ModelSource, build_model_picker_items};
81 + use crate::models::{
82 + InferenceKind, ModelCacheHealth, ModelPickerItem, ModelSource, build_model_picker_items,
83 + };
84 use ratatui::{
85 Frame,
86 layout::{Constraint, Layout, Position},
@@ -470,10 +472,68 @@ mod tui {
472 let inner = block.inner(popup);
473 frame.render_widget(block, popup);
474
475 + let active_kind = crate::models::active_inference_kind();
476 let mut lines = Vec::new();
477 +
478 + // State banner: which mode is active, and how to flip it.
479 + let (state_word, state_style) = match active_kind {
480 + InferenceKind::Local => (
481 + "ON (on-device)",
482 + Style::default().fg(Color::Green).bg(Color::Black),
483 + ),
484 + InferenceKind::Cloud => (
485 + "OFF (siGit Code Cloud)",
486 + Style::default().fg(Color::Magenta).bg(Color::Black),
487 + ),
488 + };
489 + lines.push(Line::from(vec![
490 + Span::styled(
491 + "Local inference: ",
492 + Style::default()
493 + .fg(Color::White)
494 + .bg(Color::Black)
495 + .add_modifier(Modifier::BOLD),
496 + ),
497 + Span::styled(state_word, state_style.add_modifier(Modifier::BOLD)),
498 + Span::styled(
499 + " toggle with /local on|off",
500 + Style::default().fg(Color::DarkGray).bg(Color::Black),
501 + ),
502 + ]));
503 + lines.push(Line::from("").style(Style::default().bg(Color::Black)));
504 +
505 let mut last_section: Option<ModelSource> = None;
506 + let mut last_kind: Option<InferenceKind> = None;
507
508 for (index, item) in app.model_picker_items.iter().enumerate() {
509 + let item_kind = item.source.kind();
510 + let item_active = item_kind == active_kind;
511 +
512 + // Top-level group header (Local / Cloud) whenever the nature changes.
513 + if last_kind != Some(item_kind) {
514 + if last_kind.is_some() {
515 + lines.push(Line::from("").style(Style::default().bg(Color::Black)));
516 + }
517 + let group_label = match item_kind {
518 + InferenceKind::Local => "LOCAL — on-device inference",
519 + InferenceKind::Cloud => "CLOUD — siGit Code Cloud",
520 + };
521 + let group_style = if item_active {
522 + Style::default()
523 + .fg(Color::White)
524 + .bg(Color::Black)
525 + .add_modifier(Modifier::BOLD | Modifier::UNDERLINED)
526 + } else {
527 + Style::default().fg(Color::DarkGray).bg(Color::Black)
528 + };
529 + lines.push(
530 + Line::from(vec![Span::styled(group_label, group_style)])
531 + .style(Style::default().bg(Color::Black)),
532 + );
533 + last_kind = Some(item_kind);
534 + last_section = None;
535 + }
536 +
537 if last_section != Some(item.source) {
538 if last_section.is_some() {
539 lines.push(Line::from("").style(Style::default().bg(Color::Black)));
@@ -522,9 +582,16 @@ mod tui {
582 ),
583 };
584
585 + // Dim the section header when it belongs to the inactive group.
586 + let section_style = if item_active {
587 + section_style
588 + } else {
589 + Style::default().fg(Color::DarkGray).bg(Color::Black)
590 + };
591 +
592 lines.push(
593 Line::from(vec![
527 - Span::styled(format!("{section_mark} "), section_style),
594 + Span::styled(format!(" {section_mark} "), section_style),
595 Span::styled(section_name, section_style),
596 ])
597 .style(Style::default().bg(Color::Black)),
@@ -561,12 +628,17 @@ mod tui {
628
629 let base_style = if selected {
630 Style::default().fg(Color::Black).bg(Color::Green)
564 - } else {
631 + } else if item_active {
632 Style::default().fg(Color::White).bg(Color::Black)
633 + } else {
634 + // Inactive group: still visible (we surface the offering) but dimmed.
635 + Style::default().fg(Color::DarkGray).bg(Color::Black)
636 };
637
638 let source_style = if selected {
639 Style::default().fg(Color::Black).bg(Color::Green)
640 + } else if !item_active {
641 + Style::default().fg(Color::DarkGray).bg(Color::Black)
642 } else {
643 match item.source {
644 ModelSource::Onde => Style::default().fg(Color::Green).bg(Color::Black),
@@ -653,6 +725,8 @@ mod tui {
725 Status,
726 /// picker UI, or jump straight to model N
727 Models(Option<usize>),
728 + /// toggle on-device inference mode. `Some(true/false)` sets it, `None` flips it.
729 + Local(Option<bool>),
730 /// `/login <email> <password>` — the raw argument, parsed when executed.
731 Login(Option<String>),
732 Logout,
@@ -674,6 +748,7 @@ mod tui {
748 "/clear" => SlashCommand::Clear,
749 "/status" => SlashCommand::Status,
750 "/models" => SlashCommand::Models(arg.and_then(|s| s.parse::<usize>().ok())),
751 + "/local" => SlashCommand::Local(parse_on_off(arg)),
752 "/login" => SlashCommand::Login(arg.map(str::to_string)),
753 "/logout" => SlashCommand::Logout,
754 "/whoami" => SlashCommand::Whoami,
@@ -682,6 +757,16 @@ mod tui {
757 })
758 }
759
760 + /// `on`/`off` (and synonyms) → `Some(bool)`; missing or unrecognized → `None`
761 + /// (meaning "toggle the current value").
762 + fn parse_on_off(arg: Option<&str>) -> Option<bool> {
763 + match arg.map(|s| s.trim().to_ascii_lowercase())?.as_str() {
764 + "on" | "true" | "1" | "yes" => Some(true),
765 + "off" | "false" | "0" | "no" => Some(false),
766 + _ => None,
767 + }
768 + }
769 +
770 // ── Rendering ─────────────────────────────────────────────────────────────
771
772 fn render(frame: &mut Frame, app: &mut App) {
@@ -1143,6 +1228,7 @@ mod tui {
1228 "/help — show this message\n\
1229 /models — open the model picker\n\
1230 /models N — switch to model N\n\
1231 + /local [on|off]— toggle on-device inference mode\n\
1232 /login E P — sign in to siGit Code Cloud\n\
1233 /logout — sign out\n\
1234 /whoami — show the signed-in account\n\
@@ -1195,6 +1281,8 @@ mod tui {
1281 ));
1282 app.current_model_name = provider.display_name.clone();
1283 app.tool_calling = true;
1284 + // Selecting a cloud tier puts us in cloud mode.
1285 + let _ = crate::settings::set_local_inference(false);
1286 app.messages.push(ChatMessage::system(format!(
1287 "Switched to {}.",
1288 provider.display_name
@@ -1220,7 +1308,9 @@ mod tui {
1308 }
1309
1310 // Route inference on-device; the loader thread below
1223 - // fills the engine the LocalBackend reads from.
1311 + // fills the engine the LocalBackend reads from. Selecting
1312 + // an on-device model puts us in local mode.
1313 + let _ = crate::settings::set_local_inference(true);
1314 app.backend = Arc::new(LocalBackend::new(Arc::clone(&engine)));
1315
1316 let loading_msg = if model.cache_health
@@ -1286,6 +1376,31 @@ mod tui {
1376 }
1377 }
1378 },
1379 + SlashCommand::Local(value) => {
1380 + let enabled = value.unwrap_or(!crate::settings::local_inference_enabled());
1381 + match crate::settings::set_local_inference(enabled) {
1382 + Ok(()) => {
1383 + let state = if enabled { "on" } else { "off" };
1384 + let hint = if enabled {
1385 + "On-device models are highlighted. Type /models to pick one."
1386 + } else {
1387 + "siGit Code Cloud tiers are highlighted. Type /models to pick one."
1388 + };
1389 + app.messages.push(ChatMessage::system(format!(
1390 + "Local inference is {state}. {hint}"
1391 + )));
1392 + // Refresh the picker so emphasis/order reflects the new mode.
1393 + if app.show_model_picker {
1394 + app.open_model_picker(&engine);
1395 + }
1396 + }
1397 + Err(error) => {
1398 + app.messages.push(ChatMessage::system(format!(
1399 + "error: could not save local inference setting: {error}"
1400 + )));
1401 + }
1402 + }
1403 + }
1404 SlashCommand::Login(arg) => {
1405 let message = match arg.as_deref().and_then(crate::account::parse_login_args) {
1406 Some((email, password)) => {
src/credentials.rs
+3
@@ -81,6 +81,9 @@ mod tests {
81
82 #[test]
83 fn round_trips_credentials_via_temp_dir() {
84 + let _guard = crate::ENV_TEST_LOCK
85 + .lock()
86 + .unwrap_or_else(|poisoned| poisoned.into_inner());
87 let dir = std::env::temp_dir().join(format!("sigit_creds_test_{}", std::process::id()));
88 let _ = std::fs::remove_dir_all(&dir);
89 // SAFETY: single-threaded test; restores below.
src/main.rs
+216 -17
@@ -34,9 +34,16 @@ mod chat;
34 mod credentials;
35 mod models;
36 mod provider;
37 +mod settings;
38 mod setup;
39 mod tools;
40
41 +/// Serializes tests that mutate process-global env vars (`SIGIT_CONFIG_DIR`
42 +/// etc.). `cargo test` runs tests in parallel within a binary, so without this
43 +/// the credentials and settings round-trip tests clobber each other's env.
44 +#[cfg(test)]
45 +pub(crate) static ENV_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
46 +
47 use std::io::IsTerminal;
48 #[cfg(unix)]
49 use std::io::{BufWriter, Write};
@@ -660,6 +667,11 @@ impl SiGitAgent {
667 "model number to switch to (optional)",
668 )),
669 ),
670 + with_hint(
671 + "local",
672 + "Toggle on-device inference mode",
673 + "on|off (optional)",
674 + ),
675 with_hint("login", "Sign in to siGit Code Cloud", "<email> <password>"),
676 AvailableCommand::new("logout", "Sign out of siGit Code Cloud"),
677 AvailableCommand::new("whoami", "Show the signed-in account"),
@@ -868,6 +880,9 @@ impl SiGitAgent {
880 )))
881 .await;
882
883 + // Honor the persisted Local Inference toggle (off + signed in → cloud).
884 + self.apply_startup_inference_mode().await;
885 +
886 let config_options = {
887 let guard = self.current_model.lock().unwrap();
888 build_model_config_options(&guard)
@@ -916,6 +931,9 @@ impl SiGitAgent {
931 )))
932 .await;
933
934 + // Honor the persisted Local Inference toggle (off + signed in → cloud).
935 + self.apply_startup_inference_mode().await;
936 +
937 let config_options = {
938 let guard = self.current_model.lock().unwrap();
939 build_model_config_options(&guard)
@@ -962,6 +980,9 @@ impl SiGitAgent {
980 )))
981 .await;
982
983 + // Honor the persisted Local Inference toggle (off + signed in → cloud).
984 + self.apply_startup_inference_mode().await;
985 +
986 let config_options = {
987 let guard = self.current_model.lock().unwrap();
988 build_model_config_options(&guard)
@@ -1299,10 +1320,32 @@ impl SiGitAgent {
1320 *guard = cloud_config;
1321 }
1322
1323 + // Explicitly choosing a cloud tier puts us in cloud mode.
1324 + let _ = settings::set_local_inference(false);
1325 +
1326 log::info!("switched to cloud tier {tier}");
1327 Some(cfg.display_name)
1328 }
1329
1330 + /// Apply the persisted Local Inference mode at session start. When local
1331 + /// inference is off and an account is signed in, route to a cloud tier so the
1332 + /// on-device model is never loaded; otherwise leave the on-device backend in
1333 + /// place. Call after the session cwd is set so the cloud system prompt picks
1334 + /// it up. Does not flip the stored setting on the not-signed-in fallback.
1335 + async fn apply_startup_inference_mode(&self) {
1336 + if settings::local_inference_enabled() {
1337 + return;
1338 + }
1339 + if self.switch_to_cloud_tier("balanced").await.is_some() {
1340 + log::info!("startup: local inference off; routing inference to siGit Code Cloud");
1341 + } else {
1342 + log::warn!(
1343 + "local inference is off but no account is signed in; staying on-device. \
1344 + Run /login or set Local Inference on."
1345 + );
1346 + }
1347 + }
1348 +
1349 /// Route inference back on-device. Used after leaving a cloud tier for a
1350 /// local model. The `LocalBackend` reads the live `engine`, so this just
1351 /// repoints the active backend.
@@ -1391,6 +1434,37 @@ impl SiGitAgent {
1434 args.value
1435 );
1436
1437 + // ── Local Inference toggle ──────────────────────────────────────────
1438 + if args.config_id.0.as_ref() == LOCAL_INFERENCE_CONFIG_ID {
1439 + let enabled = match args.value.0.as_ref() {
1440 + LOCAL_INFERENCE_ON => true,
1441 + LOCAL_INFERENCE_OFF => false,
1442 + other => {
1443 + return Err(agent_client_protocol::Error::new(
1444 + -32602,
1445 + format!("unknown Local Inference value: {other}"),
1446 + ));
1447 + }
1448 + };
1449 + if let Err(error) = settings::set_local_inference(enabled) {
1450 + return Err(agent_client_protocol::Error::new(
1451 + -32603,
1452 + format!("could not save Local Inference setting: {error}"),
1453 + ));
1454 + }
1455 + let message = if enabled {
1456 + "Local inference is on. On-device models are highlighted; pick one from Model."
1457 + } else {
1458 + "Local inference is off. siGit Code Cloud tiers are highlighted; pick one from Model."
1459 + };
1460 + self.send_assistant_message(cx, args.session_id.clone(), format!("\n\n{message}"))
1461 + .ok();
1462 + // Rebuild so the Model picker reflects the new emphasis/order.
1463 + let current = self.current_model.lock().unwrap().clone();
1464 + let config_options = build_model_config_options(&current);
1465 + return Ok(SetSessionConfigOptionResponse::new(config_options));
1466 + }
1467 +
1468 if args.config_id.0.as_ref() != MODEL_CONFIG_ID {
1469 return Err(agent_client_protocol::Error::new(
1470 -32602,
@@ -1645,6 +1719,8 @@ impl SiGitAgent {
1719 Ok(new_config) => {
1720 // Route inference back on-device (in case we were on a cloud tier).
1721 self.reset_to_local_backend().await;
1722 + // Selecting an on-device model puts us in local mode.
1723 + let _ = settings::set_local_inference(true);
1724
1725 let completion_title = if needs_download {
1726 format!("✓ {} downloaded and loaded", new_config.display_name)
@@ -1703,6 +1779,15 @@ impl SiGitAgent {
1779 /// config option ID for the model picker in Zed's agent panel
1780 const MODEL_CONFIG_ID: &str = "sigit-model";
1781
1782 +/// config option ID for the Local Inference on/off toggle. Surfaced as a
1783 +/// two-option `select` so ACP clients without slash-command support (e.g. Xcode)
1784 +/// can still flip the mode from the agent panel.
1785 +const LOCAL_INFERENCE_CONFIG_ID: &str = "sigit-local-inference";
1786 +
1787 +/// `select` value ids for the Local Inference toggle.
1788 +const LOCAL_INFERENCE_ON: &str = "local-inference-on";
1789 +const LOCAL_INFERENCE_OFF: &str = "local-inference-off";
1790 +
1791 /// Replace non-ASCII chars so a downstream byte-index truncation can't split a
1792 /// multi-byte char. Zed slices the model-picker label at a fixed byte offset
1793 /// (`agent_ui/src/config_options.rs`) and panics — crashing the whole editor —
@@ -1718,12 +1803,18 @@ fn build_model_config_options(current_model: &GgufModelConfig) -> Vec<SessionCon
1803 // The full list, including the siGit Code Cloud tiers, so the panel picker
1804 // mirrors the TUI `/models`. Cloud entries are sign-in gated at selection.
1805 let items = models::build_model_picker_items();
1806 + let active_kind = models::active_inference_kind();
1807
1808 let options: Vec<SessionConfigSelectOption> = items
1809 .iter()
1810 .filter(|item| item.cache_health != setup::ModelCacheHealth::Incomplete)
1811 .map(|item| {
1812 let mut desc_parts = Vec::new();
1813 + // Mark options in the inactive mode so the active group reads as the
1814 + // recommended set (the list is already ordered active-group-first).
1815 + if item.source.kind() != active_kind {
1816 + desc_parts.push("inactive mode".to_string());
1817 + }
1818 if item.tool_calling {
1819 desc_parts.push("tool calling".to_string());
1820 }
@@ -1764,8 +1855,36 @@ fn build_model_config_options(current_model: &GgufModelConfig) -> Vec<SessionCon
1855 })
1856 .collect();
1857
1858 + // Local Inference on/off toggle, modeled as a two-option select so panel-only
1859 + // ACP clients (no slash commands) can flip the mode.
1860 + let local_on = settings::local_inference_enabled();
1861 + let local_current = SessionConfigValueId::new(if local_on {
1862 + LOCAL_INFERENCE_ON
1863 + } else {
1864 + LOCAL_INFERENCE_OFF
1865 + });
1866 + let local_options = vec![
1867 + SessionConfigSelectOption::new(
1868 + SessionConfigValueId::new(LOCAL_INFERENCE_ON),
1869 + "On (on-device)".to_string(),
1870 + )
1871 + .description("Run inference on-device; on-device models are highlighted".to_string()),
1872 + SessionConfigSelectOption::new(
1873 + SessionConfigValueId::new(LOCAL_INFERENCE_OFF),
1874 + "Off (siGit Code Cloud)".to_string(),
1875 + )
1876 + .description("Use siGit Code Cloud; cloud tiers are highlighted".to_string()),
1877 + ];
1878 + let local_option = SessionConfigOption::select(
1879 + LOCAL_INFERENCE_CONFIG_ID,
1880 + "Local Inference",
1881 + local_current,
1882 + local_options,
1883 + )
1884 + .description("Toggle on-device inference; changes which models are highlighted");
1885 +
1886 if options.is_empty() {
1768 - return vec![];
1887 + return vec![local_option];
1888 }
1889
1890 let current_value = SessionConfigValueId::new(current_model.model_id.as_str());
@@ -1774,6 +1893,7 @@ fn build_model_config_options(current_model: &GgufModelConfig) -> Vec<SessionCon
1893 SessionConfigOption::select(MODEL_CONFIG_ID, "Model", current_value, options)
1894 .category(SessionConfigOptionCategory::Model)
1895 .description("Select an on-device model or a siGit Code Cloud tier"),
1896 + local_option,
1897 ]
1898 }
1899
@@ -1797,6 +1917,8 @@ enum SlashCommand {
1917 Clear,
1918 Status,
1919 Models(Option<usize>),
1920 + /// toggle on-device inference mode. `Some(true/false)` sets it, `None` flips it.
1921 + Local(Option<bool>),
1922 /// `/login <email> <password>` — the raw argument, parsed when executed.
1923 Login(Option<String>),
1924 Logout,
@@ -1820,6 +1942,7 @@ fn parse_slash(input: &str) -> Option<SlashCommand> {
1942 "/clear" => SlashCommand::Clear,
1943 "/status" => SlashCommand::Status,
1944 "/models" => SlashCommand::Models(argument.and_then(|v| v.parse::<usize>().ok())),
1945 + "/local" => SlashCommand::Local(parse_on_off(argument)),
1946 "/login" => SlashCommand::Login(argument.map(str::to_string)),
1947 "/logout" => SlashCommand::Logout,
1948 "/whoami" => SlashCommand::Whoami,
@@ -1829,6 +1952,16 @@ fn parse_slash(input: &str) -> Option<SlashCommand> {
1952 })
1953 }
1954
1955 +/// `on`/`off` (and synonyms) → `Some(bool)`; missing or unrecognized → `None`
1956 +/// (meaning "toggle the current value").
1957 +fn parse_on_off(arg: Option<&str>) -> Option<bool> {
1958 + match arg.map(|s| s.trim().to_ascii_lowercase())?.as_str() {
1959 + "on" | "true" | "1" | "yes" => Some(true),
1960 + "off" | "false" | "0" | "no" => Some(false),
1961 + _ => None,
1962 + }
1963 +}
1964 +
1965 fn format_models_list(current_model: &GgufModelConfig) -> String {
1966 let items = models::build_model_picker_items();
1967 if items.is_empty() {
@@ -1914,6 +2047,7 @@ async fn exec_slash_acp(
2047 "/help - show this message\n\
2048 /models - list available models\n\
2049 /models N - switch to model N\n\
2050 + /local [on|off]- toggle on-device inference mode\n\
2051 /login E P - sign in to siGit Code Cloud\n\
2052 /logout - sign out\n\
2053 /whoami - show the signed-in account\n\
@@ -2004,6 +2138,7 @@ async fn exec_slash_acp(
2138 match agent.switch_model_by_id(&model.config.model_id).await {
2139 Ok(new_config) => {
2140 agent.reset_to_local_backend().await;
2141 + let _ = settings::set_local_inference(true);
2142 agent.engine.clear_history().await;
2143 agent
2144 .send_assistant_message(
@@ -2037,6 +2172,7 @@ async fn exec_slash_acp(
2172
2173 let switched = agent.switch_model_by_id(&model.config.model_id).await?;
2174 agent.reset_to_local_backend().await;
2175 + let _ = settings::set_local_inference(true);
2176 agent.engine.clear_history().await;
2177
2178 agent
@@ -2050,6 +2186,33 @@ async fn exec_slash_acp(
2186 }
2187 }
2188 }
2189 + SlashCommand::Local(value) => {
2190 + let enabled = value.unwrap_or(!settings::local_inference_enabled());
2191 + let message = match settings::set_local_inference(enabled) {
2192 + Ok(()) if enabled => "Local inference is on. On-device models are highlighted; \
2193 + pick one with /models."
2194 + .to_string(),
2195 + Ok(()) => "Local inference is off. siGit Code Cloud tiers are highlighted; \
2196 + pick one with /models."
2197 + .to_string(),
2198 + Err(error) => format!("error: could not save local inference setting: {error}"),
2199 + };
2200 + agent
2201 + .send_assistant_message(cx, session_id.clone(), message)
2202 + .ok();
2203 + // Refresh the panel so the Model picker reflects the new emphasis.
2204 + let config_options = {
2205 + let current = agent.current_model.lock().unwrap();
2206 + build_model_config_options(&current)
2207 + };
2208 + agent
2209 + .send_tool_call_update(
2210 + cx,
2211 + session_id,
2212 + SessionUpdate::ConfigOptionUpdate(ConfigOptionUpdate::new(config_options)),
2213 + )
2214 + .ok();
2215 + }
2216 SlashCommand::Login(argument) => {
2217 let message = match argument.as_deref().and_then(account::parse_login_args) {
2218 Some((email, password)) => match account::authenticate(&email, &password).await {
@@ -2277,22 +2440,58 @@ async fn run_interactive(tty: std::fs::File, mut cleanup_tty: std::fs::File) ->
2440 (backend, label)
2441 }
2442 None => {
2280 - // On-device: load the local GGUF model on a real thread.
2281 - let loader_engine = Arc::clone(&engine);
2282 - let system_prompt = system_prompt_for_model(tool_calling).to_string();
2283 - std::thread::spawn(move || {
2284 - let rt =
2285 - tokio::runtime::Runtime::new().expect("failed to create loader runtime");
2286 - let result = rt.block_on(loader_engine.load_gguf_model(
2287 - config,
2288 - Some(system_prompt),
2289 - Some(sampling),
2290 - ));
2291 - let _ = load_tx.send(result.map(|_| ()).map_err(|e| e.to_string()));
2292 - });
2293 - let backend =
2294 - Arc::new(LocalBackend::new(Arc::clone(&engine))) as Arc<dyn InferenceBackend>;
2295 - (backend, startup_model_name)
2443 + // Honor the Local Inference toggle: when off and signed in, start
2444 + // on a cloud tier instead of loading an on-device model. When off
2445 + // but not signed in, fall back to on-device (a usable backend) —
2446 + // the user can /login or /local on.
2447 + let cloud_when_off = if settings::local_inference_enabled() {
2448 + None
2449 + } else {
2450 + provider::cloud_tier_provider("balanced")
2451 + };
2452 +
2453 + match cloud_when_off {
2454 + Some(provider) => {
2455 + log::info!(
2456 + "inference: local inference off; using {} (model {})",
2457 + provider.display_name,
2458 + provider.model
2459 + );
2460 + let _ = load_tx.send(Ok(()));
2461 + let label = provider.display_name.clone();
2462 + let backend = Arc::new(OpenAiBackend::new(
2463 + provider.base_url,
2464 + provider.api_key,
2465 + provider.model,
2466 + Some(SYSTEM_PROMPT.to_string()),
2467 + )) as Arc<dyn InferenceBackend>;
2468 + (backend, label)
2469 + }
2470 + None => {
2471 + if !settings::local_inference_enabled() {
2472 + log::warn!(
2473 + "local inference is off but no account is signed in; \
2474 + falling back to on-device. Run /login or /local on."
2475 + );
2476 + }
2477 + // On-device: load the local GGUF model on a real thread.
2478 + let loader_engine = Arc::clone(&engine);
2479 + let system_prompt = system_prompt_for_model(tool_calling).to_string();
2480 + std::thread::spawn(move || {
2481 + let rt = tokio::runtime::Runtime::new()
2482 + .expect("failed to create loader runtime");
2483 + let result = rt.block_on(loader_engine.load_gguf_model(
2484 + config,
2485 + Some(system_prompt),
2486 + Some(sampling),
2487 + ));
2488 + let _ = load_tx.send(result.map(|_| ()).map_err(|e| e.to_string()));
2489 + });
2490 + let backend = Arc::new(LocalBackend::new(Arc::clone(&engine)))
2491 + as Arc<dyn InferenceBackend>;
2492 + (backend, startup_model_name)
2493 + }
2494 + }
2495 }
2496 };
2497
src/models.rs
+35 -2
@@ -20,6 +20,33 @@ pub(crate) enum ModelSource {
20 Cloud,
21 }
22
23 +/// The broad nature of where inference runs. The picker groups by this and the
24 +/// `local_inference` setting decides which group is the active (highlighted) one.
25 +#[derive(Clone, Copy, PartialEq, Eq)]
26 +pub(crate) enum InferenceKind {
27 + Local,
28 + Cloud,
29 +}
30 +
31 +impl ModelSource {
32 + /// On-device sources are `Local`; the cloud tiers are `Cloud`.
33 + pub(crate) fn kind(self) -> InferenceKind {
34 + match self {
35 + ModelSource::Cloud => InferenceKind::Cloud,
36 + _ => InferenceKind::Local,
37 + }
38 + }
39 +}
40 +
41 +/// The active inference mode, from the persisted `local_inference` setting.
42 +pub(crate) fn active_inference_kind() -> InferenceKind {
43 + if crate::settings::local_inference_enabled() {
44 + InferenceKind::Local
45 + } else {
46 + InferenceKind::Cloud
47 + }
48 +}
49 +
50 #[derive(Clone)]
51 pub(crate) struct ModelPickerItem {
52 pub(crate) display_name: String,
@@ -159,9 +186,15 @@ pub(crate) fn build_model_picker_items() -> Vec<ModelPickerItem> {
186 });
187 }
188
189 + // Group by inference nature, active mode first (so the highlighted group
190 + // leads the picker), then by source, then alphabetically.
191 + let active = active_inference_kind();
192 + let group_rank =
193 + |item: &ModelPickerItem| -> u8 { if item.source.kind() == active { 0 } else { 1 } };
194 items.sort_by(|left, right| {
163 - left.source
164 - .cmp(&right.source)
195 + group_rank(left)
196 + .cmp(&group_rank(right))
197 + .then_with(|| left.source.cmp(&right.source))
198 .then_with(|| left.display_name.cmp(&right.display_name))
199 });
200
src/settings.rs new
+153
@@ -0,0 +1,153 @@
1 +//! Local user preferences.
2 +//!
3 +//! Persisted as TOML at `$SIGIT_CONFIG_DIR/settings.toml` or
4 +//! `~/.config/sigit/settings.toml`. Mirrors the storage pattern of
5 +//! [`crate::credentials`] but holds preferences rather than secrets, so it is
6 +//! not permission-restricted.
7 +//!
8 +//! The only setting today is `local_inference`: whether on-device inference is
9 +//! the active mode. It is the source of truth for the local/cloud toggle and
10 +//! drives how `/models` presents the picker. It is stored locally so the toggle
11 +//! works even on ACP clients that do not support slash commands (e.g. Xcode),
12 +//! where it is also surfaced as a session config option.
13 +
14 +use std::path::PathBuf;
15 +
16 +use serde::{Deserialize, Serialize};
17 +
18 +/// Env override for `local_inference`. When set to a truthy/falsy value it wins
19 +/// over the stored file for reads (matching the existing `SIGIT_*` override
20 +/// style); it never writes the file.
21 +const LOCAL_INFERENCE_ENV: &str = "SIGIT_LOCAL_INFERENCE";
22 +
23 +fn default_local_inference() -> bool {
24 + true
25 +}
26 +
27 +/// Persisted preferences. New fields must carry `#[serde(default)]` so older
28 +/// files keep deserializing.
29 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
30 +pub struct Settings {
31 + /// Whether on-device inference is the active mode. `true` (local-first) on a
32 + /// fresh install.
33 + #[serde(default = "default_local_inference")]
34 + pub local_inference: bool,
35 +}
36 +
37 +impl Default for Settings {
38 + fn default() -> Self {
39 + Self {
40 + local_inference: default_local_inference(),
41 + }
42 + }
43 +}
44 +
45 +/// Config directory: `$SIGIT_CONFIG_DIR` or `~/.config/sigit`.
46 +fn config_dir() -> Option<PathBuf> {
47 + if let Ok(dir) = std::env::var("SIGIT_CONFIG_DIR") {
48 + return Some(PathBuf::from(dir));
49 + }
50 + let home = std::env::var("HOME").ok()?;
51 + Some(PathBuf::from(home).join(".config/sigit"))
52 +}
53 +
54 +fn settings_path() -> Option<PathBuf> {
55 + config_dir().map(|dir| dir.join("settings.toml"))
56 +}
57 +
58 +/// Parse an env value as a boolean. Accepts `1/0`, `true/false`, `on/off`,
59 +/// `yes/no` (case-insensitive). Returns `None` for anything unrecognized so a
60 +/// stray value falls back to the stored setting instead of silently flipping.
61 +fn parse_bool_env(value: &str) -> Option<bool> {
62 + match value.trim().to_ascii_lowercase().as_str() {
63 + "1" | "true" | "on" | "yes" => Some(true),
64 + "0" | "false" | "off" | "no" => Some(false),
65 + _ => None,
66 + }
67 +}
68 +
69 +/// Load stored settings, or defaults if the file is absent or unreadable.
70 +pub fn load() -> Settings {
71 + let Some(path) = settings_path() else {
72 + return Settings::default();
73 + };
74 + match std::fs::read_to_string(&path) {
75 + Ok(contents) => toml::from_str::<Settings>(&contents).unwrap_or_else(|error| {
76 + log::warn!("settings: ignoring settings.toml: {error}");
77 + Settings::default()
78 + }),
79 + Err(_) => Settings::default(),
80 + }
81 +}
82 +
83 +/// Persist settings, creating the config dir if needed.
84 +pub fn store(settings: &Settings) -> Result<(), String> {
85 + let dir = config_dir().ok_or_else(|| "cannot resolve config directory".to_string())?;
86 + std::fs::create_dir_all(&dir).map_err(|error| format!("create {dir:?}: {error}"))?;
87 + let path = dir.join("settings.toml");
88 + let body = toml::to_string(settings).map_err(|error| format!("serialize settings: {error}"))?;
89 + std::fs::write(&path, body).map_err(|error| format!("write {path:?}: {error}"))?;
90 + Ok(())
91 +}
92 +
93 +/// Whether on-device inference is the active mode. The `SIGIT_LOCAL_INFERENCE`
94 +/// env var, when set to a recognized boolean, overrides the stored value.
95 +pub fn local_inference_enabled() -> bool {
96 + if let Ok(raw) = std::env::var(LOCAL_INFERENCE_ENV)
97 + && let Some(value) = parse_bool_env(&raw)
98 + {
99 + return value;
100 + }
101 + load().local_inference
102 +}
103 +
104 +/// Persist a new `local_inference` value, preserving any other settings.
105 +pub fn set_local_inference(enabled: bool) -> Result<(), String> {
106 + let mut settings = load();
107 + settings.local_inference = enabled;
108 + store(&settings)
109 +}
110 +
111 +#[cfg(test)]
112 +mod tests {
113 + use super::*;
114 +
115 + // One test (not several) because each mutates the process-global
116 + // `SIGIT_CONFIG_DIR` / `SIGIT_LOCAL_INFERENCE` env vars; splitting would let
117 + // them race under `cargo test`'s parallel runner.
118 + #[test]
119 + fn defaults_round_trip_and_env_override() {
120 + let _guard = crate::ENV_TEST_LOCK
121 + .lock()
122 + .unwrap_or_else(|poisoned| poisoned.into_inner());
123 + let dir = std::env::temp_dir().join(format!("sigit_settings_{}", std::process::id()));
124 + let _ = std::fs::remove_dir_all(&dir);
125 + // SAFETY: single-threaded test; restores below.
126 + unsafe { std::env::set_var("SIGIT_CONFIG_DIR", &dir) };
127 + unsafe { std::env::remove_var(LOCAL_INFERENCE_ENV) };
128 +
129 + // Fresh install: no file → local-first.
130 + assert!(
131 + load().local_inference,
132 + "fresh install should be local-first"
133 + );
134 + assert!(local_inference_enabled());
135 +
136 + set_local_inference(false).unwrap();
137 + assert!(!load().local_inference);
138 + assert!(!local_inference_enabled());
139 +
140 + // Env override wins over the stored `false`.
141 + unsafe { std::env::set_var(LOCAL_INFERENCE_ENV, "true") };
142 + assert!(local_inference_enabled());
143 + unsafe { std::env::set_var(LOCAL_INFERENCE_ENV, "garbage") };
144 + assert!(
145 + !local_inference_enabled(),
146 + "unrecognized env value falls back to stored setting"
147 + );
148 +
149 + unsafe { std::env::remove_var(LOCAL_INFERENCE_ENV) };
150 + unsafe { std::env::remove_var("SIGIT_CONFIG_DIR") };
151 + let _ = std::fs::remove_dir_all(&dir);
152 + }
153 +}