[Rust Frontend][gRPC] Add LoRA lifecycle control (#52840)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Connor Carpenter <connorc@nvidia.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Connor Carpenter committed
Aug 25, 2026 at 00:13 UTC
1f9444a34ff4ebfba4d65c68971bb5306a11aa92
12 files changed
+471
-82
rust/Cargo.lock
+1
@@ -6025,6 +6025,7 @@ dependencies = [
6025
"socket2",
6026
"subtle",
6027
"tempfile",
6028
+ "thiserror 2.0.18",
6029
"thiserror-ext",
6030
"tls-listener",
6031
"tokio",
rust/proto/control.proto
+30
@@ -5,9 +5,19 @@ syntax = "proto3";
5
package vllm;
6
7
service Control {
8
+ // Server and model discovery.
9
rpc GetServerInfo (GetServerInfoRequest) returns (ServerInfo) {}
10
rpc GetModelInfo (GetModelInfoRequest) returns (ModelInfo) {}
11
+
12
+ // Request lifecycle.
13
rpc Abort (AbortRequest) returns (AbortResponse) {}
14
+
15
+ // LoRA lifecycle.
16
+ rpc LoadLora (LoadLoraRequest) returns (LoadLoraResponse) {}
17
+ rpc UnloadLora (UnloadLoraRequest) returns (UnloadLoraResponse) {}
18
+ rpc ListLoras (ListLorasRequest) returns (ListLorasResponse) {}
19
+
20
+ // KV event discovery.
21
rpc GetKvEventSources (GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse) {}
22
23
// Reinforcement-learning lifecycle and weight updates.
@@ -79,6 +89,26 @@ message AbortRequest {
89
90
message AbortResponse {}
91
92
+// ======================================================================================
93
+// LoRA lifecycle
94
+// ======================================================================================
95
+
96
+message LoraAdapter {
97
+ int64 lora_id = 1;
98
+ string lora_name = 2;
99
+ string source_path = 3;
100
+}
101
+
102
+message LoadLoraRequest {
103
+ string lora_name = 1;
104
+ string source_path = 2;
105
+}
106
+message LoadLoraResponse { LoraAdapter adapter = 1; }
107
+message UnloadLoraRequest { string lora_name = 1; }
108
+message UnloadLoraResponse { LoraAdapter adapter = 1; }
109
+message ListLorasRequest {}
110
+message ListLorasResponse { repeated LoraAdapter adapters = 1; }
111
+
112
// ======================================================================================
113
// Reinforcement-learning control
114
// ======================================================================================
rust/proto/inference.proto
+3
@@ -50,6 +50,9 @@ message GenerateRequest {
50
51
// Multimodal inputs aligned with placeholder markers in token_ids.
52
repeated MediaItem media = 14;
53
+
54
+ // Loaded LoRA name; empty selects the base model.
55
+ string lora_name = 15;
56
}
57
58
message RandomSampling {
rust/src/engine-core-client/src/protocol/lora.rs
+24
-3
@@ -2,6 +2,7 @@
2
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
5
+use thiserror::Error;
6
7
use crate::protocol::OpaqueValue;
8
@@ -24,6 +25,16 @@ pub struct LoraRequest {
25
pub is_3d_lora_weight: bool,
26
}
27
28
+#[derive(Debug, Error, PartialEq, Eq)]
29
+pub enum LoraRequestError {
30
+ #[error("lora_name must not be empty")]
31
+ EmptyName,
32
+ #[error("lora_int_id must be nonzero")]
33
+ ZeroId,
34
+ #[error("lora_path must not be empty")]
35
+ EmptyPath,
36
+}
37
+
38
impl LoraRequest {
39
pub fn new(
40
lora_name: String,
@@ -31,8 +42,18 @@ impl LoraRequest {
42
lora_path: String,
43
load_inplace: bool,
44
is_3d_lora_weight: bool,
34
- ) -> Self {
35
- Self {
45
+ ) -> Result<Self, LoraRequestError> {
46
+ if lora_name.trim().is_empty() {
47
+ return Err(LoraRequestError::EmptyName);
48
+ }
49
+ if lora_int_id == 0 {
50
+ return Err(LoraRequestError::ZeroId);
51
+ }
52
+ if lora_path.trim().is_empty() {
53
+ return Err(LoraRequestError::EmptyPath);
54
+ }
55
+
56
+ Ok(Self {
57
lora_name,
58
lora_int_id,
59
lora_path,
@@ -40,6 +61,6 @@ impl LoraRequest {
61
tensorizer_config_dict: None,
62
load_inplace,
63
is_3d_lora_weight,
43
- }
64
+ })
65
}
66
}
rust/src/server/Cargo.toml
+1
@@ -28,6 +28,7 @@ serde_with.workspace = true
28
sha2.workspace = true
29
socket2.workspace = true
30
subtle.workspace = true
31
+thiserror.workspace = true
32
thiserror-ext.workspace = true
33
tls-listener.workspace = true
34
tokio.workspace = true
rust/src/server/src/grpc/control.rs
+91
-1
@@ -6,12 +6,14 @@ use std::sync::Arc;
6
use serde_json::Value as JsonValue;
7
use thiserror_ext::AsReport as _;
8
use tokio::sync::Mutex;
9
-use tonic::{Request, Response, Status};
9
+use tonic::{Code, Request, Response, Status};
10
use vllm_engine_core_client::EngineCoreClient;
11
use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse;
12
+use vllm_engine_core_client::protocol::lora::LoraRequest;
13
use vllm_engine_core_client::protocol::utility::PauseMode as EnginePauseMode;
14
15
use super::{ControlServer, pb};
16
+use crate::lora::{LoadLoraError, LoraDisabledError, UnloadLoraError};
17
use crate::state::AppState;
18
19
pub(crate) type ControlGrpcService = ControlServer<ControlServiceImpl>;
@@ -134,6 +136,41 @@ fn weight_version(value: String) -> Result<String, Status> {
136
Ok(value)
137
}
138
139
+fn lora_to_proto(adapter: &LoraRequest) -> pb::LoraAdapter {
140
+ pb::LoraAdapter {
141
+ lora_id: adapter.lora_int_id.min(i64::MAX as u64) as i64,
142
+ lora_name: adapter.lora_name.clone(),
143
+ source_path: adapter.lora_path.clone(),
144
+ }
145
+}
146
+
147
+fn load_lora_status(error: LoadLoraError) -> Status {
148
+ let code = match &error {
149
+ LoadLoraError::Disabled(_) => Code::FailedPrecondition,
150
+ LoadLoraError::InvalidRequest(_) => Code::InvalidArgument,
151
+ LoadLoraError::AlreadyLoaded { .. } | LoadLoraError::BaseModelName { .. } => {
152
+ Code::AlreadyExists
153
+ }
154
+ LoadLoraError::Engine { .. } | LoadLoraError::NotLoaded { .. } => Code::Internal,
155
+ };
156
+ Status::new(code, error.to_report_string())
157
+}
158
+
159
+fn unload_lora_status(error: UnloadLoraError) -> Status {
160
+ let code = match &error {
161
+ UnloadLoraError::Disabled(_) => Code::FailedPrecondition,
162
+ UnloadLoraError::NotFound { .. } => Code::NotFound,
163
+ UnloadLoraError::IntIdMismatch { .. }
164
+ | UnloadLoraError::Engine { .. }
165
+ | UnloadLoraError::NotRemoved { .. } => Code::Internal,
166
+ };
167
+ Status::new(code, error.to_report_string())
168
+}
169
+
170
+fn list_loras_status(error: LoraDisabledError) -> Status {
171
+ Status::failed_precondition(error.to_report_string())
172
+}
173
+
174
#[tonic::async_trait]
175
impl pb::control_server::Control for ControlServiceImpl {
176
async fn get_server_info(
@@ -201,6 +238,59 @@ impl pb::control_server::Control for ControlServiceImpl {
238
Ok(Response::new(pb::AbortResponse {}))
239
}
240
241
+ async fn load_lora(
242
+ &self,
243
+ request: Request<pb::LoadLoraRequest>,
244
+ ) -> Result<Response<pb::LoadLoraResponse>, Status> {
245
+ let request = request.into_inner();
246
+ let adapter = self
247
+ .state
248
+ .load_lora(request.lora_name, request.source_path, false, false)
249
+ .await
250
+ .map_err(load_lora_status)?;
251
+ Ok(Response::new(pb::LoadLoraResponse {
252
+ adapter: Some(lora_to_proto(&adapter)),
253
+ }))
254
+ }
255
+
256
+ async fn unload_lora(
257
+ &self,
258
+ request: Request<pb::UnloadLoraRequest>,
259
+ ) -> Result<Response<pb::UnloadLoraResponse>, Status> {
260
+ let name = request.into_inner().lora_name;
261
+ if name.trim().is_empty() {
262
+ return Err(Status::invalid_argument("lora_name is required"));
263
+ }
264
+
265
+ let adapter = self
266
+ .state
267
+ .list_loras()
268
+ .await
269
+ .map_err(list_loras_status)?
270
+ .into_iter()
271
+ .find(|adapter| adapter.lora_name == name)
272
+ .ok_or_else(|| Status::not_found(format!("LoRA adapter `{name}` is not loaded")))?;
273
+ let adapter = self
274
+ .state
275
+ .unload_lora(&name, Some(adapter.lora_int_id))
276
+ .await
277
+ .map_err(unload_lora_status)?;
278
+ Ok(Response::new(pb::UnloadLoraResponse {
279
+ adapter: Some(lora_to_proto(&adapter)),
280
+ }))
281
+ }
282
+
283
+ async fn list_loras(
284
+ &self,
285
+ _request: Request<pb::ListLorasRequest>,
286
+ ) -> Result<Response<pb::ListLorasResponse>, Status> {
287
+ let mut adapters = self.state.list_loras().await.map_err(list_loras_status)?;
288
+ adapters.sort_by(|left, right| left.lora_name.cmp(&right.lora_name));
289
+ Ok(Response::new(pb::ListLorasResponse {
290
+ adapters: adapters.iter().map(lora_to_proto).collect(),
291
+ }))
292
+ }
293
+
294
async fn get_kv_event_sources(
295
&self,
296
_request: Request<pb::GetKvEventSourcesRequest>,
rust/src/server/src/grpc/inference.rs
+14
@@ -58,12 +58,14 @@ impl InferenceServiceImpl {
58
} else {
59
proto_request.model.clone()
60
};
61
+ let lora_name = proto_request.lora_name.clone();
62
let media_count = proto_request.media.len();
63
let request_span = info_span!(
64
"grpc_inference",
65
%request_id,
66
rpc,
67
%model,
68
+ %lora_name,
69
media_count,
70
?data_parallel_rank,
71
);
@@ -76,6 +78,18 @@ impl InferenceServiceImpl {
78
text_request.arrival_time = Some(arrival_time);
79
text_request.data_parallel_rank = data_parallel_rank;
80
81
+ if !lora_name.is_empty() {
82
+ if !self.state.engine_core_client().ready_response().supports_lora {
83
+ return Err(Status::failed_precondition(
84
+ "engine was not started with LoRA enabled",
85
+ ));
86
+ }
87
+ let resolution = self.state.resolve_model_with_loras(Some(&lora_name)).await;
88
+ text_request.lora_request = Some(resolution.lora_request.ok_or_else(|| {
89
+ Status::not_found(format!("LoRA adapter `{lora_name}` is not loaded"))
90
+ })?);
91
+ }
92
+
93
let media = convert::media_parts_from_request(media)?;
94
if !media.is_empty() {
95
let Prompt::TokenIds(mut token_ids) = text_request.prompt else {
rust/src/server/src/grpc/tests.rs
+174
@@ -151,6 +151,35 @@ async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
151
.expect("send outputs");
152
}
153
154
+async fn reply_utility_bool(
155
+ dealer: &mut DealerSocket,
156
+ push: &mut PushSocket,
157
+ expected_method: &str,
158
+ result: bool,
159
+) {
160
+ let frames = recv_engine_message(dealer).await;
161
+ assert_eq!(frames[0].as_ref(), &[0x03]);
162
+ let payload = decode_value(&frames[1]).expect("decode utility payload");
163
+ let fields = payload.as_array().expect("utility payload array");
164
+ let call_id = fields[1].as_u64().expect("utility call id");
165
+ assert_eq!(fields[2].as_str(), Some(expected_method));
166
+ send_outputs(
167
+ push,
168
+ UtilityCallOutput {
169
+ output: UtilityOutput {
170
+ call_id: call_id.into(),
171
+ failure_message: None,
172
+ result: Some(UtilityResultEnvelope::without_type_info(Value::Boolean(
173
+ result,
174
+ ))),
175
+ },
176
+ ..Default::default()
177
+ }
178
+ .into(),
179
+ )
180
+ .await;
181
+}
182
+
183
async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
184
dealer.recv().await.expect("recv engine message").into_vec()
185
}
@@ -1558,6 +1587,151 @@ async fn control_reports_server_and_model_info() {
1587
server_task.abort();
1588
}
1589
1590
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1591
+#[serial]
1592
+async fn control_lora_lifecycle_selects_adapter_for_generation() {
1593
+ let mut ready = default_ready_response();
1594
+ ready.supports_lora = true;
1595
+ ready.max_loras = 4;
1596
+ let (inference_service, control_service, engine_health, engine_task) =
1597
+ setup_grpc_service_with_engine_script(
1598
+ b"engine-grpc-lora".to_vec(),
1599
+ ready,
1600
+ Arc::new(FakeTextBackend),
1601
+ |dealer, push| {
1602
+ boxed_test_future(async move {
1603
+ reply_utility_bool(dealer, push, "add_lora", true).await;
1604
+
1605
+ let frames = recv_engine_message(dealer).await;
1606
+ assert_eq!(frames[0].as_ref(), &[0x00]);
1607
+ let request: EngineCoreRequest =
1608
+ rmp_serde::from_slice(&frames[1]).expect("decode generation request");
1609
+ let lora = request.lora_request.expect("generation LoRA request");
1610
+ assert_eq!(lora.lora_name, "adapter");
1611
+ assert_eq!(lora.lora_int_id, 1);
1612
+ send_outputs(
1613
+ push,
1614
+ engine_outputs_for_request(
1615
+ &request.request_id,
1616
+ vec![(vec![b'!' as u32], Some(EngineCoreFinishReason::Stop))],
1617
+ ),
1618
+ )
1619
+ .await;
1620
+
1621
+ reply_utility_bool(dealer, push, "remove_lora", true).await;
1622
+ })
1623
+ },
1624
+ )
1625
+ .await;
1626
+ let (channel, server_task) = start_grpc_test_server(
1627
+ inference_service,
1628
+ control_service,
1629
+ engine_health,
1630
+ tokio_util::sync::CancellationToken::new(),
1631
+ )
1632
+ .await;
1633
+ let mut control_client = ControlClient::new(channel.clone());
1634
+ let mut inference_client = InferenceClient::new(channel);
1635
+ let loaded = control_client
1636
+ .load_lora(pb::LoadLoraRequest {
1637
+ lora_name: "adapter".to_string(),
1638
+ source_path: "adapter".to_string(),
1639
+ })
1640
+ .await
1641
+ .expect("load LoRA")
1642
+ .into_inner();
1643
+ assert_eq!(
1644
+ loaded.adapter.as_ref().map(|adapter| adapter.lora_id),
1645
+ Some(1)
1646
+ );
1647
+
1648
+ let duplicate = control_client
1649
+ .load_lora(pb::LoadLoraRequest {
1650
+ lora_name: "adapter".to_string(),
1651
+ source_path: "adapter".to_string(),
1652
+ })
1653
+ .await
1654
+ .expect_err("duplicate LoRA name should be rejected");
1655
+ assert_eq!(duplicate.code(), tonic::Code::AlreadyExists);
1656
+
1657
+ let listed = control_client
1658
+ .list_loras(pb::ListLorasRequest {})
1659
+ .await
1660
+ .expect("list LoRAs")
1661
+ .into_inner();
1662
+ assert_eq!(listed.adapters.len(), 1);
1663
+ assert_eq!(listed.adapters[0].lora_name, "adapter");
1664
+
1665
+ inference_client
1666
+ .generate(pb::GenerateRequest {
1667
+ request_id: "grpc-lora-request".to_string(),
1668
+ model: "test-model".to_string(),
1669
+ prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
1670
+ stopping: Some(pb::StoppingCriteria {
1671
+ max_new_tokens: 1,
1672
+ ..Default::default()
1673
+ }),
1674
+ lora_name: "adapter".to_string(),
1675
+ ..Default::default()
1676
+ })
1677
+ .await
1678
+ .expect("generate with LoRA");
1679
+
1680
+ let unloaded = control_client
1681
+ .unload_lora(pb::UnloadLoraRequest {
1682
+ lora_name: "adapter".to_string(),
1683
+ })
1684
+ .await
1685
+ .expect("unload LoRA")
1686
+ .into_inner();
1687
+ assert_eq!(
1688
+ unloaded.adapter.as_ref().map(|adapter| adapter.lora_id),
1689
+ Some(1)
1690
+ );
1691
+ assert!(
1692
+ control_client
1693
+ .list_loras(pb::ListLorasRequest {})
1694
+ .await
1695
+ .expect("list LoRAs after unload")
1696
+ .into_inner()
1697
+ .adapters
1698
+ .is_empty()
1699
+ );
1700
+
1701
+ engine_task.await.expect("mock engine task");
1702
+ server_task.abort();
1703
+}
1704
+
1705
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1706
+#[serial]
1707
+async fn control_list_loras_requires_lora_enabled_engine() {
1708
+ let (inference_service, control_service, engine_health, engine_task) =
1709
+ setup_grpc_service_with_engine_script(
1710
+ b"engine-grpc-lora-disabled".to_vec(),
1711
+ default_ready_response(),
1712
+ Arc::new(FakeTextBackend),
1713
+ |_, _| boxed_test_future(async move {}),
1714
+ )
1715
+ .await;
1716
+ let (channel, server_task) = start_grpc_test_server(
1717
+ inference_service,
1718
+ control_service,
1719
+ engine_health,
1720
+ tokio_util::sync::CancellationToken::new(),
1721
+ )
1722
+ .await;
1723
+ let mut control_client = ControlClient::new(channel);
1724
+ let status = control_client
1725
+ .list_loras(pb::ListLorasRequest {})
1726
+ .await
1727
+ .expect_err("list should require LoRA support");
1728
+ assert_eq!(status.code(), tonic::Code::FailedPrecondition);
1729
+ assert_eq!(status.message(), "engine was not started with LoRA enabled");
1730
+
1731
+ engine_task.await.expect("mock engine task");
1732
+ server_task.abort();
1733
+}
1734
+
1735
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1736
#[serial]
1737
async fn control_forwards_weight_update_without_pause_guard() {
rust/src/server/src/lora.rs
+56
-27
@@ -4,9 +4,10 @@
4
use std::sync::atomic::{AtomicU64, Ordering};
5
6
use indexmap::IndexMap;
7
+use thiserror::Error;
8
use tokio::sync::{Mutex, RwLock};
9
use vllm_engine_core_client::EngineCoreClient;
9
-use vllm_engine_core_client::protocol::lora::LoraRequest;
10
+use vllm_engine_core_client::protocol::lora::{LoraRequest, LoraRequestError};
11
12
/// Snapshot of the currently served model names plus the requested LoRA, if
13
/// the model name resolves to a dynamic adapter.
@@ -26,29 +27,52 @@ pub(crate) struct LoraManager {
27
update_lock: Mutex<()>,
28
}
29
29
-#[derive(Debug)]
30
+#[derive(Debug, Error)]
31
+#[error("engine was not started with LoRA enabled")]
32
+pub(crate) struct LoraDisabledError;
33
+
34
+#[derive(Debug, Error)]
35
pub(crate) enum LoadLoraError {
36
+ #[error(transparent)]
37
+ Disabled(#[from] LoraDisabledError),
38
+ #[error(transparent)]
39
+ InvalidRequest(#[from] LoraRequestError),
40
+ #[error("LoRA adapter `{lora_name}` is already loaded")]
41
AlreadyLoaded { lora_name: String },
42
+ #[error("LoRA adapter `{lora_name}` conflicts with a served base model")]
43
BaseModelName { lora_name: String },
33
- Engine(vllm_engine_core_client::Error),
44
+ #[error("failed to load LoRA adapter `{lora_name}`")]
45
+ Engine {
46
+ lora_name: String,
47
+ #[source]
48
+ source: vllm_engine_core_client::Error,
49
+ },
50
+ #[error("one or more engine ranks rejected LoRA adapter `{lora_name}`")]
51
NotLoaded { lora_name: String },
52
}
53
37
-#[derive(Debug)]
54
+#[derive(Debug, Error)]
55
pub(crate) enum UnloadLoraError {
39
- NotFound {
40
- lora_name: String,
41
- },
56
+ #[error(transparent)]
57
+ Disabled(#[from] LoraDisabledError),
58
+ #[error("LoRA adapter `{lora_name}` is not loaded")]
59
+ NotFound { lora_name: String },
60
+ #[error(
61
+ "requested lora_int_id {actual} does not match loaded adapter `{lora_name}` with id {expected}"
62
+ )]
63
IntIdMismatch {
64
lora_name: String,
65
expected: u64,
66
actual: u64,
67
},
47
- Engine(vllm_engine_core_client::Error),
48
- NotRemoved {
68
+ #[error("failed to unload LoRA adapter `{lora_name}`")]
69
+ Engine {
70
lora_name: String,
50
- lora_int_id: u64,
71
+ #[source]
72
+ source: vllm_engine_core_client::Error,
73
},
74
+ #[error("engine rejected removal of LoRA adapter `{lora_name}` with id {lora_int_id}")]
75
+ NotRemoved { lora_name: String, lora_int_id: u64 },
76
}
77
78
impl LoraManager {
@@ -97,16 +121,13 @@ impl LoraManager {
121
if base_model_names.iter().any(|name| name == &lora_name) {
122
return Err(LoadLoraError::BaseModelName { lora_name });
123
}
100
- if !load_inplace && self.requests.read().await.contains_key(&lora_name) {
124
+ let requests = self.requests.read().await;
125
+ let existing_lora_int_id = requests.get(&lora_name).map(|request| request.lora_int_id);
126
+ if !load_inplace && existing_lora_int_id.is_some() {
127
return Err(LoadLoraError::AlreadyLoaded { lora_name });
128
}
129
104
- let lora_int_id = self
105
- .requests
106
- .read()
107
- .await
108
- .get(&lora_name)
109
- .map(|request| request.lora_int_id)
130
+ let lora_int_id = existing_lora_int_id
131
.unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1);
132
let lora_request = LoraRequest::new(
133
lora_name.clone(),
@@ -114,12 +135,16 @@ impl LoraManager {
135
lora_path,
136
load_inplace,
137
is_3d_lora_weight,
117
- );
118
-
119
- let loaded = engine_core_client
120
- .add_lora(&lora_request)
121
- .await
122
- .map_err(LoadLoraError::Engine)?;
138
+ )
139
+ .map_err(LoadLoraError::InvalidRequest)?;
140
+ drop(requests);
141
+
142
+ let loaded = engine_core_client.add_lora(&lora_request).await.map_err(|source| {
143
+ LoadLoraError::Engine {
144
+ lora_name: lora_name.clone(),
145
+ source,
146
+ }
147
+ })?;
148
if !loaded {
149
return Err(LoadLoraError::NotLoaded { lora_name });
150
}
@@ -152,10 +177,14 @@ impl LoraManager {
177
});
178
}
179
155
- let removed = engine_core_client
156
- .remove_lora(lora_request.lora_int_id)
157
- .await
158
- .map_err(UnloadLoraError::Engine)?;
180
+ let removed =
181
+ engine_core_client
182
+ .remove_lora(lora_request.lora_int_id)
183
+ .await
184
+ .map_err(|source| UnloadLoraError::Engine {
185
+ lora_name: lora_request.lora_name.clone(),
186
+ source,
187
+ })?;
188
if !removed {
189
return Err(UnloadLoraError::NotRemoved {
190
lora_name: lora_request.lora_name,
rust/src/server/src/routes/lora.rs
+36
-49
@@ -108,17 +108,45 @@ fn validate_lora_path_access(
108
Ok(Some(canonical_path.to_string_lossy().into_owned()))
109
}
110
111
+fn load_lora_api_error(error: LoadLoraError) -> ApiError {
112
+ let message = error.to_report_string();
113
+ match error {
114
+ LoadLoraError::Disabled(_) => ApiError::invalid_request(message, None),
115
+ LoadLoraError::InvalidRequest(_) => ApiError::invalid_request(message, None),
116
+ LoadLoraError::AlreadyLoaded { .. } => ApiError::invalid_request(
117
+ format!(
118
+ "{message}. If you want to load the adapter in place, set 'load_inplace' to true."
119
+ ),
120
+ Some("lora_name"),
121
+ ),
122
+ LoadLoraError::BaseModelName { .. } => {
123
+ ApiError::invalid_request(message, Some("lora_name"))
124
+ }
125
+ LoadLoraError::Engine { .. } | LoadLoraError::NotLoaded { .. } => {
126
+ ApiError::server_error(message)
127
+ }
128
+ }
129
+}
130
+
131
+fn unload_lora_api_error(error: UnloadLoraError) -> ApiError {
132
+ let message = error.to_report_string();
133
+ match error {
134
+ UnloadLoraError::Disabled(_) => ApiError::invalid_request(message, None),
135
+ UnloadLoraError::NotFound { lora_name } => ApiError::model_not_found(lora_name),
136
+ UnloadLoraError::IntIdMismatch { .. } => {
137
+ ApiError::invalid_request(message, Some("lora_int_id"))
138
+ }
139
+ UnloadLoraError::Engine { .. } | UnloadLoraError::NotRemoved { .. } => {
140
+ ApiError::server_error(message)
141
+ }
142
+ }
143
+}
144
+
145
/// Dynamically load one LoRA adapter and expose it as an OpenAI model id.
146
pub async fn load_lora_adapter(
147
State(state): State<Arc<AppState>>,
148
ValidatedJson(request): ValidatedJson<LoadLoraAdapterRequest>,
149
) -> Result<String, ApiError> {
116
- if request.lora_name.is_empty() || request.lora_path.is_empty() {
117
- return Err(ApiError::invalid_request(
118
- "Both 'lora_name' and 'lora_path' must be provided.".to_string(),
119
- None,
120
- ));
121
- }
150
let allowed_prefixes = runtime_lora_allowed_path_prefixes();
151
let lora_path = validate_lora_path_access(&request.lora_path, allowed_prefixes.as_deref())?
152
.unwrap_or(request.lora_path);
@@ -132,25 +160,7 @@ pub async fn load_lora_adapter(
160
request.is_3d_lora_weight,
161
)
162
.await
135
- .map_err(|error| match error {
136
- LoadLoraError::AlreadyLoaded { lora_name } => ApiError::invalid_request(
137
- format!(
138
- "The lora adapter '{lora_name}' has already been loaded. If you want to load the adapter in place, set 'load_inplace' to true."
139
- ),
140
- Some("lora_name"),
141
- ),
142
- LoadLoraError::BaseModelName { lora_name } => ApiError::invalid_request(
143
- format!("The lora adapter name '{lora_name}' conflicts with a served base model."),
144
- Some("lora_name"),
145
- ),
146
- LoadLoraError::Engine(error) => ApiError::server_error(format!(
147
- "failed to load LoRA adapter '{lora_name}': {}",
148
- error.to_report_string()
149
- )),
150
- LoadLoraError::NotLoaded { lora_name } => ApiError::server_error(format!(
151
- "failed to load LoRA adapter '{lora_name}': engine rejected the adapter"
152
- )),
153
- })?;
163
+ .map_err(load_lora_api_error)?;
164
165
Ok(format!(
166
"Success: LoRA adapter '{lora_name}' added successfully."
@@ -172,30 +182,7 @@ pub async fn unload_lora_adapter(
182
let lora_request = state
183
.unload_lora(&request.lora_name, request.lora_int_id)
184
.await
175
- .map_err(|error| match error {
176
- UnloadLoraError::NotFound { lora_name } => ApiError::model_not_found(lora_name),
177
- UnloadLoraError::IntIdMismatch {
178
- lora_name,
179
- expected,
180
- actual,
181
- } => ApiError::invalid_request(
182
- format!(
183
- "The requested lora_int_id {actual} does not match loaded adapter '{lora_name}' with id {expected}."
184
- ),
185
- Some("lora_int_id"),
186
- ),
187
- UnloadLoraError::Engine(error) => ApiError::server_error(format!(
188
- "failed to unload LoRA adapter '{}': {}",
189
- request.lora_name,
190
- error.to_report_string()
191
- )),
192
- UnloadLoraError::NotRemoved {
193
- lora_name,
194
- lora_int_id,
195
- } => ApiError::server_error(format!(
196
- "failed to unload LoRA adapter '{lora_name}' with id {lora_int_id}"
197
- )),
198
- })?;
185
+ .map_err(unload_lora_api_error)?;
186
187
Ok(format!(
188
"Success: LoRA adapter '{}' removed successfully.",
rust/src/server/src/routes/tests.rs
+16
-1
@@ -28,6 +28,7 @@ use vllm_chat::{
28
};
29
use vllm_engine_core_client::mock_engine::default_ready_response;
30
use vllm_engine_core_client::protocol::decode_value;
31
+use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse;
32
use vllm_engine_core_client::protocol::logprobs::{
33
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
34
};
@@ -833,6 +834,19 @@ where
834
}
835
836
async fn test_admin_app_with_engine_script<F>(script: F) -> (axum::Router, MockEngineTask)
837
+where
838
+ F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
839
+{
840
+ let mut ready = default_ready_response();
841
+ ready.supports_lora = true;
842
+ ready.max_loras = 4;
843
+ test_admin_app_with_ready_and_engine_script(ready, script).await
844
+}
845
+
846
+async fn test_admin_app_with_ready_and_engine_script<F>(
847
+ ready: EngineCoreReadyResponse,
848
+ script: F,
849
+) -> (axum::Router, MockEngineTask)
850
where
851
F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
852
{
@@ -840,9 +854,10 @@ where
854
let handshake_address = ipc.handshake_endpoint();
855
let engine_id = b"engine-openai-admin".to_vec();
856
843
- let engine_task = MockEngineTask::new(spawn_mock_engine_task(
857
+ let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready(
858
handshake_address.clone(),
859
engine_id.clone(),
860
+ ready,
861
move |dealer, push| script(dealer, push),
862
));
863
rust/src/server/src/state.rs
+25
-1
@@ -15,7 +15,9 @@ use vllm_engine_core_client::protocol::lora::LoraRequest;
15
use vllm_engine_core_client::runtime::BackgroundShutdownRuntime;
16
17
use crate::config::{ApiServerOptions, CorsConfig};
18
-use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
18
+use crate::lora::{
19
+ LoadLoraError, LoraDisabledError, LoraManager, LoraModelResolution, UnloadLoraError,
20
+};
21
use crate::runtime::build_request_runtime;
22
use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
23
@@ -161,12 +163,22 @@ impl AppState {
163
self.lora_manager.served_lora_requests().await
164
}
165
166
+ /// Snapshot loaded LoRA adapters for the lifecycle API.
167
+ ///
168
+ /// Returns error if the engine was started without LoRA support.
169
+ pub async fn list_loras(&self) -> Result<Vec<LoraRequest>, LoraDisabledError> {
170
+ self.ensure_lora_enabled()?;
171
+ Ok(self.lora_manager.served_lora_requests().await)
172
+ }
173
+
174
/// Resolve the requested model against one dynamic LoRA registry snapshot.
175
pub async fn resolve_model_with_loras(&self, model_name: Option<&str>) -> LoraModelResolution {
176
self.lora_manager.resolve_model(&self.served_model_names, model_name).await
177
}
178
179
/// Load one dynamic LoRA adapter and register it as a public model name.
180
+ ///
181
+ /// Returns error if the engine was started without LoRA support.
182
pub async fn load_lora(
183
&self,
184
lora_name: String,
@@ -174,6 +186,7 @@ impl AppState {
186
load_inplace: bool,
187
is_3d_lora_weight: bool,
188
) -> Result<LoraRequest, LoadLoraError> {
189
+ self.ensure_lora_enabled()?;
190
self.lora_manager
191
.load_lora(
192
self.engine_core_client(),
@@ -188,16 +201,27 @@ impl AppState {
201
202
/// Remove one dynamic LoRA adapter from the engine and public model
203
/// registry.
204
+ ///
205
+ /// Returns error if the engine was started without LoRA support.
206
pub async fn unload_lora(
207
&self,
208
lora_name: &str,
209
lora_int_id: Option<u64>,
210
) -> Result<LoraRequest, UnloadLoraError> {
211
+ self.ensure_lora_enabled()?;
212
self.lora_manager
213
.unload_lora(self.engine_core_client(), lora_name, lora_int_id)
214
.await
215
}
216
217
+ fn ensure_lora_enabled(&self) -> Result<(), LoraDisabledError> {
218
+ self.engine_core_client()
219
+ .ready_response()
220
+ .supports_lora
221
+ .then_some(())
222
+ .ok_or(LoraDisabledError)
223
+ }
224
+
225
/// Return a reference to the underlying engine core client for utility
226
/// calls.
227
pub(crate) fn engine_core_client(&self) -> &EngineCoreClient {