Report mid-stream model errors to pulse as errors
The streaming loop breaks out on InternalError, ValidationError, and ModelError, but the inference event after the loop was still sent with status "success". Those arms now set the status to "error". This is also the usual way a stream ends with no ttft_ms. Co-Authored-By: siGit Code <noreply@sigit.si>
Seto Elkahfi committed
Sep 23, 2026 at 22:37 UTC
54c395d5b98042bf87a5efc9828d7de7709ab880
2 files changed
+9
-3
src/inference/engine.rs
+5
-1
@@ -1244,6 +1244,7 @@ impl ChatEngine {
1244
let mut assembled = String::new();
1245
let mut last_finish_reason = None;
1246
let mut ttft_ms = None;
1247
+ let mut status = "success";
1248
1249
// `Stream::next()` is an inherent async method on
1250
// mistralrs::Stream that returns `Option<Response>`.
@@ -1277,14 +1278,17 @@ impl ChatEngine {
1278
}
1279
mistralrs::Response::InternalError(e) => {
1280
log::error!("ChatEngine stream internal error: {}", e);
1281
+ status = "error";
1282
break;
1283
}
1284
mistralrs::Response::ValidationError(e) => {
1285
log::error!("ChatEngine stream validation error: {}", e);
1286
+ status = "error";
1287
break;
1288
}
1289
mistralrs::Response::ModelError(msg, _) => {
1290
log::error!("ChatEngine stream model error: {}", msg);
1291
+ status = "error";
1292
break;
1293
}
1294
_ => {
@@ -1317,7 +1321,7 @@ impl ChatEngine {
1321
crate::pulse::next_request_id(),
1322
started.elapsed().as_millis() as u64,
1323
ttft_ms,
1320
- "success".to_string(),
1324
+ status.to_string(),
1325
);
1326
}
1327
src/pulse/events.rs
+4
-2
@@ -37,8 +37,10 @@ pub struct InferenceEvent {
37
/// once and reports `None` rather than a stand-in value.
38
#[serde(skip_serializing_if = "Option::is_none")]
39
pub ttft_ms: Option<u64>,
40
- /// `"success"`, `"cancelled"`, or `"error"`. Currently always
41
- /// `"success"` here — inference errors throw before we reach this point.
40
+ /// `"success"`, `"cancelled"`, or `"error"`. A blocking request only
41
+ /// reports `"success"`, since its errors return before reaching pulse. A
42
+ /// stream can fail after it starts, so a mid-stream model error reports
43
+ /// `"error"`, usually with no `ttft_ms`.
44
pub status: String,
45
/// Same Onde app as the corresponding `ModelLoadedEvent`. `None` for SDK
46
/// builds without app credentials or direct Rust consumers.