Load on-device model only on explicit request

index 0c94f62..fe29e96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### What changed + +- On-device models are no longer loaded implicitly. The chat UI and ACP sessions come up immediately, and the local model is brought into memory only when you run the new `/load` command (or pick one in `/models`). Prompts sent before a model is loaded now return a hint instead of blocking on a multi-minute download. + ## 1.2.2 Streams assistant tokens as they arrive, on-device and over the cloud.

index 953a708..830dffa 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -384,10 +384,19 @@ mod tui { self.messages.push(ChatMessage::system( "In this world, nothing can be said to be certain, except death and taxes. ~ Pak Sigit", )); - self.messages.push(ChatMessage::system(format!( - "Current model: {}", - self.current_model_name - ))); + if self.backend.is_remote() { + self.messages.push(ChatMessage::system(format!( + "Current model: {}", + self.current_model_name + ))); + } else { + // On-device models are never loaded implicitly; prompt the user to + // load one explicitly before their first message. + self.messages.push(ChatMessage::system(format!( + "No on-device model loaded. Run /load to load {}, or /models to choose one.", + self.current_model_name + ))); + } self.messages .push(ChatMessage::system("Type /help for commands.")); } @@ -653,6 +662,8 @@ mod tui { Status, /// picker UI, or jump straight to model N Models(Option<usize>), + /// explicitly load the selected (or default) on-device model + Load, /// `/login <email> <password>` — the raw argument, parsed when executed. Login(Option<String>), Logout, @@ -674,6 +685,7 @@ mod tui { "/clear" => SlashCommand::Clear, "/status" => SlashCommand::Status, "/models" => SlashCommand::Models(arg.and_then(|s| s.parse::<usize>().ok())), + "/load" => SlashCommand::Load, "/login" => SlashCommand::Login(arg.map(str::to_string)), "/logout" => SlashCommand::Logout, "/whoami" => SlashCommand::Whoami, @@ -1129,6 +1141,105 @@ mod tui { } } + // ── Explicit on-device model loading ────────────────────────────────────── + + /// The local model `/load` should bring up: the persisted selection if it + /// still resolves to a known model, otherwise the first on-device (non-cloud) + /// entry in the picker. + fn default_local_model_item(app: &App) -> Option<ModelPickerItem> { + if let Some(selected) = crate::setup::load_selected_model() + && let Some(item) = app.model_picker_items.iter().find(|item| { + item.config.model_id == selected.model_id + && item + .config + .files + .iter() + .any(|file| file == &selected.gguf_file) + }) + { + return Some(item.clone()); + } + app.model_picker_items + .iter() + .find(|item| item.cloud_tier.is_none()) + .cloned() + } + + /// Load `model` on-device on a dedicated loader thread, routing inference to a + /// fresh `LocalBackend` and driving the switch-progress UI. The caller is + /// responsible for any cloud-tier handling; this path is on-device only. + fn start_local_model_load<B: ratatui::backend::Backend>( + app: &mut App, + model: ModelPickerItem, + engine: Arc<ChatEngine>, + terminal: &mut ratatui::Terminal<B>, + ) { + if model.cache_health == ModelCacheHealth::Incomplete { + app.messages.push(ChatMessage::system(format!( + "error: {} has an incomplete local cache and cannot be selected yet.", + model.display_name + ))); + return; + } + + // Route inference on-device; the loader thread below fills the engine the + // LocalBackend reads from. + app.backend = Arc::new(LocalBackend::new(Arc::clone(&engine))); + + let loading_msg = if model.cache_health == ModelCacheHealth::NotDownloaded { + format!( + "Downloading and loading {} ({})… this may take a few minutes.", + model.display_name, model.description + ) + } else { + format!("Loading {}…", model.display_name) + }; + + app.messages.push(ChatMessage::system(loading_msg)); + terminal.draw(|frame| render(frame, app)).ok(); + + let (tx, rx) = mpsc::channel(1); + app.model_load_rx = Some(rx); + app.switching_model = true; + app.switching_model_id = Some(model.config.model_id.clone()); + // Only show download progress for models not yet cached. + app.download_progress = if model.cache_health == ModelCacheHealth::NotDownloaded { + Some((0, 0)) + } else { + None + }; + + let sampling = SamplingConfig { + max_tokens: Some(model.max_tokens), + ..SamplingConfig::default() + }; + + // own thread + runtime so block_in_place doesn't starve the TUI loop + let system_prompt = crate::system_prompt_for_model(model.tool_calling); + let engine_handle = Arc::clone(&engine); + let tool_calling = model.tool_calling; + std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("failed to create model-loader runtime"); + let update = rt.block_on(async move { + match engine_handle + .load_gguf_model( + model.config.clone(), + Some(system_prompt.to_string()), + Some(sampling), + ) + .await + { + Ok(_) => ModelLoadUpdate::Loaded(model.display_name.clone()), + Err(err) => ModelLoadUpdate::Error(err.to_string()), + } + }); + // capacity-1 channel, receiver alive while switching + let _ = tx.blocking_send(update); + }); + // applied on ModelLoadUpdate::Loaded + app.pending_tool_calling = Some(tool_calling); + } + // ── Slash command execution ─────────────────────────────────────────────── async fn exec_slash<B: ratatui::backend::Backend>( @@ -1143,6 +1254,7 @@ mod tui { "/help — show this message\n\ /models — open the model picker\n\ /models N — switch to model N\n\ + /load — load the selected on-device model\n\ /login E P — sign in to siGit Code Cloud\n\ /logout — sign out\n\ /whoami — show the signed-in account\n\ @@ -1210,82 +1322,22 @@ mod tui { return; } - if model.cache_health == ModelCacheHealth::Incomplete { - app.close_model_picker(); - app.messages.push(ChatMessage::system(format!( - "error: {} has an incomplete local cache and cannot be selected yet.", - model.display_name - ))); - return; - } - - // Route inference on-device; the loader thread below - // fills the engine the LocalBackend reads from. - app.backend = Arc::new(LocalBackend::new(Arc::clone(&engine))); - - let loading_msg = if model.cache_health - == ModelCacheHealth::NotDownloaded - { - format!( - "Downloading and loading {} ({})… this may take a few minutes.", - model.display_name, model.description - ) - } else { - format!("Loading {}…", model.display_name) - }; - app.close_model_picker(); - app.messages.push(ChatMessage::system(loading_msg)); - terminal.draw(|frame| render(frame, app)).ok(); - - let (tx, rx) = mpsc::channel(1); - app.model_load_rx = Some(rx); - app.switching_model = true; - app.switching_model_id = Some(model.config.model_id.clone()); - // Only show download progress for models not yet cached. - app.download_progress = - if model.cache_health == ModelCacheHealth::NotDownloaded { - Some((0, 0)) - } else { - None - }; - - let sampling = SamplingConfig { - max_tokens: Some(model.max_tokens), - ..SamplingConfig::default() - }; - - // own thread + runtime so block_in_place doesn't starve the TUI loop - let system_prompt = crate::system_prompt_for_model(model.tool_calling); - let engine_handle = Arc::clone(&engine); - let tool_calling = model.tool_calling; - std::thread::spawn(move || { - let rt = tokio::runtime::Runtime::new() - .expect("failed to create model-loader runtime"); - let update = rt.block_on(async move { - match engine_handle - .load_gguf_model( - model.config.clone(), - Some(system_prompt.to_string()), - Some(sampling), - ) - .await - { - Ok(_) => { - ModelLoadUpdate::Loaded(model.display_name.clone()) - } - Err(err) => ModelLoadUpdate::Error(err.to_string()), - } - }); - // capacity-1 channel, receiver alive while switching - let _ = tx.blocking_send(update); - }); - // applied on ModelLoadUpdate::Loaded - app.pending_tool_calling = Some(tool_calling); + start_local_model_load(app, model, Arc::clone(&engine), terminal); } } } }, + SlashCommand::Load => match default_local_model_item(app) { + None => { + app.messages.push(ChatMessage::system( + "No local model available to load. Use /models to see the list.", + )); + } + Some(model) => { + start_local_model_load(app, model, Arc::clone(&engine), terminal); + } + }, SlashCommand::Login(arg) => { let message = match arg.as_deref().and_then(crate::account::parse_login_args) { Some((email, password)) => { @@ -1703,6 +1755,21 @@ mod tui { continue; } + // On-device inference needs a model in memory, and we + // never load one implicitly: the user loads it with + // /load (or /models). Refuse rather than erroring out + // deep in the backend. + if !app.backend.is_remote() + && engine.info().await.status == onde::inference::EngineStatus::Unloaded + { + app.messages.push(ChatMessage::user(&text)); + app.messages.push(ChatMessage::system( + "No on-device model is loaded. Run /load to load the selected \ + model, or /models to choose one.", + )); + continue; + } + // ── spawn inference ────────────────────────────── app.messages.push(ChatMessage::user(&text)); app.start_thinking();

index 8d6de8e..397c680 100644 --- a/src/main.rs +++ b/src/main.rs @@ -660,6 +660,7 @@ impl SiGitAgent { "model number to switch to (optional)", )), ), + AvailableCommand::new("load", "Load the selected on-device model"), with_hint("login", "Sign in to siGit Code Cloud", "<email> <password>"), AvailableCommand::new("logout", "Sign out of siGit Code Cloud"), AvailableCommand::new("whoami", "Show the signed-in account"), @@ -1131,10 +1132,21 @@ impl SiGitAgent { let backend = self.backend.lock().await.clone(); // Only on-device inference needs a local model in memory. Cloud tiers run - // over the network, so skip the lazy load and the readiness wait for them. - if !backend.is_remote() { - self.start_startup_model_load_if_needed(); - self.await_model_ready(cx, &session_id).await?; + // over the network, so they never need a local model. We never load the + // on-device model implicitly: the user loads it explicitly with `/load` + // (or by picking one in `/models`). If a prompt arrives before that, guide + // them rather than blocking on a multi-minute download/load. + if !backend.is_remote() + && self.engine.info().await.status == onde::inference::EngineStatus::Unloaded + { + self.send_assistant_message( + cx, + session_id, + "No on-device model is loaded. Run `/load` to load the selected model, \ + or `/models` to choose one.", + ) + .ok(); + return Ok(PromptResponse::new(StopReason::EndTurn)); } // ── tool-calling loop ──────────────────────────────────────────── @@ -1411,12 +1423,16 @@ impl SiGitAgent { } } - // Zed re-fires the last selection on connect; no-op if it's already loaded + // Zed re-fires the last selection when a thread opens. That re-fire must + // not load anything: on-device models are loaded only on an explicit + // request (`/load`, or actively picking a *different* model below), so a + // re-fire of the already-current selection is a no-op. Otherwise opening a + // new thread would silently load the local model — exactly what we avoid. { let current = self.current_model.lock().unwrap(); if current.model_id == model_id { log::info!( - "set_session_config_option: {} is already the active model, skipping", + "set_session_config_option: {} is already the active selection, skipping", current.display_name ); let config_options = build_model_config_options(&current); @@ -1797,6 +1813,8 @@ enum SlashCommand { Clear, Status, Models(Option<usize>), + /// Explicitly load the selected (or default) on-device model. + Load, /// `/login <email> <password>` — the raw argument, parsed when executed. Login(Option<String>), Logout, @@ -1820,6 +1838,7 @@ fn parse_slash(input: &str) -> Option<SlashCommand> { "/clear" => SlashCommand::Clear, "/status" => SlashCommand::Status, "/models" => SlashCommand::Models(argument.and_then(|v| v.parse::<usize>().ok())), + "/load" => SlashCommand::Load, "/login" => SlashCommand::Login(argument.map(str::to_string)), "/logout" => SlashCommand::Logout, "/whoami" => SlashCommand::Whoami, @@ -1914,6 +1933,7 @@ async fn exec_slash_acp( "/help - show this message\n\ /models - list available models\n\ /models N - switch to model N\n\ + /load - load the selected on-device model\n\ /login E P - sign in to siGit Code Cloud\n\ /logout - sign out\n\ /whoami - show the signed-in account\n\ @@ -2050,6 +2070,25 @@ async fn exec_slash_acp( } } } + SlashCommand::Load => { + // Explicitly load the on-device model. This is the only path that + // brings a local model into memory; prompts never do it implicitly. + // If a cloud tier is active, fall back to a local default so we don't + // try to load the (file-less) cloud config as GGUF. + let on_cloud = { + let guard = agent.current_model.lock().unwrap(); + guard.model_id.starts_with("sigit-cloud:") + }; + if on_cloud { + let default_config = default_local_model_config(); + *agent.current_model.lock().unwrap() = default_config; + agent.reset_to_local_backend().await; + } + // `await_model_ready` drives the download/load progress UI and reports + // success or failure to the editor. + agent.start_startup_model_load_if_needed(); + agent.await_model_ready(cx, &session_id).await?; + } SlashCommand::Login(argument) => { let message = match argument.as_deref().and_then(account::parse_login_args) { Some((email, password)) => match account::authenticate(&email, &password).await { @@ -2219,42 +2258,11 @@ async fn run_interactive(tty: std::fs::File, mut cleanup_tty: std::fs::File) -> .map(|selection| selection.display_name.clone()) .unwrap_or_else(|| GgufModelConfig::qwen25_3b().display_name); - let config = startup_selection - .as_ref() - .and_then(|selection| { - models::local_picker_items() - .into_iter() - .find(|item| { - selection - .selected_model - .as_ref() - .map(|selected| { - item.config.model_id == selected.model_id - && item - .config - .files - .iter() - .any(|file| file == &selected.gguf_file) - }) - .unwrap_or(false) - }) - .map(|item| item.config) - }) - .unwrap_or_else(GgufModelConfig::qwen25_3b); - let sampling = SamplingConfig { - max_tokens: Some(8192), - ..SamplingConfig::default() - }; - - // std::sync::mpsc on a real thread so model loading can't starve the TUI + // Signals the loading phase to finish. On-device models are no longer loaded + // at startup, so this resolves immediately for both backends; it stays a + // channel so the loading-phase plumbing in `chat::run_with` is unchanged. let (load_tx, load_rx) = std::sync::mpsc::channel::<Result<(), String>>(); - let tool_calling = models::local_picker_items() - .iter() - .find(|item| item.config.model_id == config.model_id) - .map(|item| item.tool_calling) - .unwrap_or(false); - // Pick the inference backend: a configured provider if present, else on-device. let (inference_backend, startup_model_name): (Arc<dyn InferenceBackend>, String) = match provider::active_provider() { @@ -2277,19 +2285,10 @@ async fn run_interactive(tty: std::fs::File, mut cleanup_tty: std::fs::File) -> (backend, label) } None => { - // On-device: load the local GGUF model on a real thread. - let loader_engine = Arc::clone(&engine); - let system_prompt = system_prompt_for_model(tool_calling).to_string(); - std::thread::spawn(move || { - let rt = - tokio::runtime::Runtime::new().expect("failed to create loader runtime"); - let result = rt.block_on(loader_engine.load_gguf_model( - config, - Some(system_prompt), - Some(sampling), - )); - let _ = load_tx.send(result.map(|_| ()).map_err(|e| e.to_string())); - }); + // On-device: do NOT load the local GGUF model implicitly. The user + // loads it explicitly with /load (or /models) from the chat, so the + // UI comes up immediately without a multi-minute download/load. + let _ = load_tx.send(Ok(())); let backend = Arc::new(LocalBackend::new(Arc::clone(&engine))) as Arc<dyn InferenceBackend>; (backend, startup_model_name) @@ -2332,11 +2331,11 @@ async fn run_interactive(tty: std::fs::File, mut cleanup_tty: std::fs::File) -> // ── ACP server mode ─────────────────────────────────────────────────────────── -async fn run_acp_server() -> anyhow::Result<()> { - log::info!("ACP mode — starting agent server"); - - let startup_selection = setup::startup_model_selection(); - let config = startup_selection +/// The on-device model `/load` should bring up by default: the persisted +/// selection if it still resolves to a known local model, otherwise the built-in +/// default (`qwen25_3b`). +fn default_local_model_config() -> GgufModelConfig { + setup::startup_model_selection() .as_ref() .and_then(|selection| { selection.selected_model.as_ref().and_then(|selected| { @@ -2353,7 +2352,13 @@ async fn run_acp_server() -> anyhow::Result<()> { .map(|item| item.config) }) }) - .unwrap_or_else(GgufModelConfig::qwen25_3b); + .unwrap_or_else(GgufModelConfig::qwen25_3b) +} + +async fn run_acp_server() -> anyhow::Result<()> { + log::info!("ACP mode — starting agent server"); + + let config = default_local_model_config(); let needs_download = models::local_picker_items() .iter() @@ -2373,8 +2378,9 @@ async fn run_acp_server() -> anyhow::Result<()> { let engine = Arc::new(ChatEngine::new()); - // Delay model loading until the first real prompt so initialize/session/new - // stay lightweight and registry auth checks don't trip over model startup. + // The on-device model is never loaded implicitly; the user loads it with + // `/load` (or by picking one in `/models`). So initialize/session/new stay + // lightweight and `model_ready` starts true (nothing is loading). let model_ready = Arc::new(AtomicBool::new(true)); let startup_model_load_started = Arc::new(AtomicBool::new(false)); let model_load_error: Arc<std::sync::Mutex<Option<String>>> =