A high-throughput and memory-efficient inference and serving engine for LLMs
[Rust Frontend] Add /abort_requests endpoint (#44382)
Signed-off-by: Sahil Singh <sahiilsiingh37@gmail.com>
Sahil Singh committed
Jun 17, 2026 at 12:10 UTC
295232a26a42837326531b48e891d54bab210dcb
3 files changed
+145
rust/src/server/src/routes.rs
+2
@@ -1,3 +1,4 @@
1
+mod abort_requests;
2
mod cache;
3
mod collective_rpc;
4
mod health;
@@ -91,6 +92,7 @@ fn build_router_with_options(
92
.route("/reset_mm_cache", post(cache::reset_mm_cache))
93
.route("/reset_encoder_cache", post(cache::reset_encoder_cache))
94
.route("/collective_rpc", post(collective_rpc::collective_rpc))
95
+ .route("/abort_requests", post(abort_requests::abort_requests))
96
.route("/sleep", post(sleep::sleep))
97
.route("/wake_up", post(sleep::wake_up))
98
.route("/is_sleeping", get(sleep::is_sleeping))
rust/src/server/src/routes/abort_requests.rs
new
+37
@@ -0,0 +1,37 @@
1
+use std::sync::Arc;
2
+
3
+use axum::Json;
4
+use axum::extract::State;
5
+use axum::extract::rejection::JsonRejection;
6
+use axum::http::StatusCode;
7
+use serde::Deserialize;
8
+
9
+use crate::error::ApiError;
10
+use crate::state::AppState;
11
+use crate::utils::utility_call_error;
12
+
13
+#[derive(Debug, Deserialize)]
14
+pub(crate) struct AbortRequestsRequest {
15
+ request_ids: Option<Vec<String>>,
16
+}
17
+
18
+pub async fn abort_requests(
19
+ State(state): State<Arc<AppState>>,
20
+ body: Result<Json<AbortRequestsRequest>, JsonRejection>,
21
+) -> Result<StatusCode, ApiError> {
22
+ let Json(body) = body.map_err(|error| ApiError::json_parse_error(error.body_text()))?;
23
+ let request_ids = body.request_ids.ok_or_else(|| {
24
+ ApiError::invalid_request(
25
+ "Missing 'request_ids' in request body".to_string(),
26
+ Some("request_ids"),
27
+ )
28
+ })?;
29
+
30
+ state
31
+ .chat
32
+ .abort(&request_ids)
33
+ .await
34
+ .map_err(|error| utility_call_error("abort_requests", error))?;
35
+
36
+ Ok(StatusCode::OK)
37
+}
rust/src/server/src/routes/tests.rs
+106
@@ -4921,6 +4921,111 @@ async fn is_paused_route_returns_json_payload() {
4921
);
4922
}
4923
4924
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4925
+#[serial]
4926
+async fn abort_requests_route_returns_ok_for_well_formed_body() {
4927
+ let (app, engine_task) =
4928
+ test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await;
4929
+
4930
+ let response = app
4931
+ .clone()
4932
+ .call(
4933
+ Request::builder()
4934
+ .method("POST")
4935
+ .uri("/abort_requests")
4936
+ .header("content-type", "application/json")
4937
+ .body(Body::from(r#"{"request_ids":["req-1","req-2"]}"#))
4938
+ .expect("build request"),
4939
+ )
4940
+ .await
4941
+ .expect("call app");
4942
+
4943
+ let status = response.status();
4944
+ let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
4945
+ assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
4946
+ assert!(body.is_empty());
4947
+ engine_task.abort_and_join().await;
4948
+}
4949
+
4950
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4951
+#[serial]
4952
+async fn abort_requests_route_rejects_missing_request_ids() {
4953
+ let (app, engine_task) =
4954
+ test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await;
4955
+
4956
+ let response = app
4957
+ .clone()
4958
+ .call(
4959
+ Request::builder()
4960
+ .method("POST")
4961
+ .uri("/abort_requests")
4962
+ .header("content-type", "application/json")
4963
+ .body(Body::from(r#"{}"#))
4964
+ .expect("build request"),
4965
+ )
4966
+ .await
4967
+ .expect("call app");
4968
+
4969
+ assert_eq!(response.status(), StatusCode::BAD_REQUEST);
4970
+ let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
4971
+ let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
4972
+ assert_eq!(json["error"]["type"], "invalid_request_error");
4973
+ assert_eq!(json["error"]["param"], "request_ids");
4974
+ engine_task.abort_and_join().await;
4975
+}
4976
+
4977
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4978
+#[serial]
4979
+async fn abort_requests_route_rejects_malformed_json() {
4980
+ let (app, engine_task) =
4981
+ test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await;
4982
+
4983
+ let response = app
4984
+ .clone()
4985
+ .call(
4986
+ Request::builder()
4987
+ .method("POST")
4988
+ .uri("/abort_requests")
4989
+ .header("content-type", "application/json")
4990
+ .body(Body::from(r#"{"request_ids": "#))
4991
+ .expect("build request"),
4992
+ )
4993
+ .await
4994
+ .expect("call app");
4995
+
4996
+ assert_eq!(response.status(), StatusCode::BAD_REQUEST);
4997
+ let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
4998
+ let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
4999
+ assert_eq!(json["error"]["type"], "invalid_request_error");
5000
+ engine_task.abort_and_join().await;
5001
+}
5002
+
5003
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5004
+#[serial]
5005
+async fn abort_requests_route_accepts_empty_id_list() {
5006
+ let (app, engine_task) =
5007
+ test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await;
5008
+
5009
+ let response = app
5010
+ .clone()
5011
+ .call(
5012
+ Request::builder()
5013
+ .method("POST")
5014
+ .uri("/abort_requests")
5015
+ .header("content-type", "application/json")
5016
+ .body(Body::from(r#"{"request_ids":[]}"#))
5017
+ .expect("build request"),
5018
+ )
5019
+ .await
5020
+ .expect("call app");
5021
+
5022
+ let status = response.status();
5023
+ let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
5024
+ assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
5025
+ assert!(body.is_empty());
5026
+ engine_task.abort_and_join().await;
5027
+}
5028
+
5029
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5030
#[serial]
5031
async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
@@ -4941,6 +5046,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
5046
("POST", "/pause"),
5047
("POST", "/resume"),
5048
("POST", "/collective_rpc"),
5049
+ ("POST", "/abort_requests"),
5050
("POST", "/reset_prefix_cache"),
5051
("POST", "/reset_mm_cache"),
5052
("POST", "/reset_encoder_cache"),