Drop old model weights outside lock when loading new model
keypair34 committed
Apr 25, 2026 at 21:43 UTC
78c5c43bac88b04be31f1f9321c0dc7eb9daf2c1
1 file changed
+15
-8
src/inference/engine.rs
+15
-8
@@ -418,14 +418,21 @@ impl ChatEngine {
418
let pulse_model_id = config.model_id.clone();
419
let pulse_model_name = config.display_name.clone();
420
421
- let mut guard = self.inner.lock().await;
422
- *guard = Some(LoadedModel {
423
- model: Arc::new(model),
424
- config: LoadedModelConfig::Gguf(config),
425
- history: Vec::new(),
426
- system_prompt,
427
- sampling,
428
- });
421
+ // Swap the new model in and take the old one out — all under the lock
422
+ // so the engine is never seen as None by a concurrent prompt. Then
423
+ // drop the old weights *outside* the lock so we don't block inference
424
+ // while the allocator frees potentially several GB of tensors.
425
+ let old_model = {
426
+ let mut guard = self.inner.lock().await;
427
+ guard.replace(LoadedModel {
428
+ model: Arc::new(model),
429
+ config: LoadedModelConfig::Gguf(config),
430
+ history: Vec::new(),
431
+ system_prompt,
432
+ sampling,
433
+ })
434
+ };
435
+ drop(old_model); // free old weights outside the lock
436
437
if let Some(ref pulse) = self.pulse {
438
pulse.record_model_loaded(pulse_model_id, pulse_model_name, elapsed.as_millis() as u64);