@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6550ff12f2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 2, 02:15 UTC

[Rust Frontend] Add dynamic LoRA endpoints (#43778)

Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com>

Xunzhuo committed Jun 3, 2026 at 15:55 UTC 6550ff12f21721b1694e6299df0470b419cebb67
24 files changed +1079 -47
rust/src/chat/src/lib.rs
+1
@@ -189,6 +189,7 @@ impl ChatLlm {
189 cache_salt: request.cache_salt,
190 add_special_tokens: request.add_special_tokens,
191 data_parallel_rank: request.data_parallel_rank,
192 + lora_request: request.lora_request,
193 };
194 let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
195
rust/src/chat/src/request.rs
+5
@@ -3,6 +3,7 @@ use std::collections::HashMap;
3 use llm_multimodal::ImageDetail;
4 use serde::{Deserialize, Serialize};
5 use serde_json::Value;
6 +use vllm_engine_core_client::protocol::lora::LoraRequest;
7 pub use vllm_text::SamplingParams;
8 use vllm_text::TextDecodeOptions;
9 pub use vllm_tool_parser::Tool as ChatTool;
@@ -426,6 +427,9 @@ pub struct ChatRequest {
427 /// Override data parallel rank.
428 #[serde(default)]
429 pub data_parallel_rank: Option<u32>,
430 + /// LoRA adapter selected for this request.
431 + #[serde(default)]
432 + pub lora_request: Option<LoraRequest>,
433 }
434
435 impl ChatRequest {
@@ -445,6 +449,7 @@ impl ChatRequest {
449 cache_salt: None,
450 add_special_tokens: false,
451 data_parallel_rank: None,
452 + lora_request: None,
453 }
454 }
455
rust/src/cmd/src/cli/tests.rs
+11
@@ -87,6 +87,17 @@ fn serve_args_auto_forward_python_flags_without_separator() {
87 );
88 }
89
90 +#[test]
91 +fn serve_args_auto_forward_enable_lora_to_python() {
92 + let cli =
93 + Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--enable-lora"]).unwrap();
94 +
95 + let Command::Serve(args) = cli.command else {
96 + panic!("expected serve args");
97 + };
98 + assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]);
99 +}
100 +
101 #[test]
102 fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
103 let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap();
rust/src/cmd/src/cli/unsupported.rs
-9
@@ -326,15 +326,6 @@ pub struct EngineUnsupportedArgs {
326 #[arg(long)]
327 pub mm_processor_cache_type: Option<Unsupported>,
328
329 - /// If True, enable handling of LoRA adapters.
330 - #[arg(
331 - long,
332 - visible_alias = "no-enable-lora",
333 - default_missing_value = "true",
334 - num_args = 0..=1
335 - )]
336 - pub enable_lora: Option<Unsupported>,
337 -
329 /// Dictionary mapping specific modalities to LoRA model paths.
330 #[arg(long)]
331 pub default_mm_loras: Option<Unsupported>,
rust/src/engine-core-client/src/client.rs
+19
@@ -10,6 +10,7 @@ use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop
10 use crate::coordinator::CoordinatorHandle;
11 use crate::error::{Error, Result};
12 use crate::protocol::handshake::EngineCoreReadyResponse;
13 +use crate::protocol::lora::LoraRequest;
14 use crate::protocol::utility::EngineCoreUtilityRequest;
15 use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
16 use crate::transport::{self, ConnectedEngine};
@@ -659,6 +660,24 @@ impl EngineCoreClient {
660 Ok(results.into_iter().all(|ok| ok))
661 }
662
663 + /// Load or refresh one LoRA adapter on every connected engine.
664 + pub async fn add_lora(&self, lora_request: &LoraRequest) -> Result<bool> {
665 + Ok(self
666 + .call_utility::<bool, _>("add_lora", (lora_request,))
667 + .await?
668 + .into_iter()
669 + .all(|loaded| loaded))
670 + }
671 +
672 + /// Remove one LoRA adapter from every connected engine.
673 + pub async fn remove_lora(&self, lora_id: u64) -> Result<bool> {
674 + Ok(self
675 + .call_utility::<bool, _>("remove_lora", (lora_id,))
676 + .await?
677 + .into_iter()
678 + .all(|removed| removed))
679 + }
680 +
681 /// Put the engine to sleep.
682 pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> {
683 self.call_utility::<(), _>("sleep", (level, mode)).await?;
rust/src/engine-core-client/src/protocol/lora.rs new
+42
@@ -0,0 +1,42 @@
1 +use serde_tuple::{Deserialize_tuple, Serialize_tuple};
2 +
3 +use crate::protocol::OpaqueValue;
4 +
5 +/// Request for a LoRA adapter.
6 +///
7 +/// Mirrors Python `vllm.lora.request.LoRARequest`, which is a msgspec
8 +/// `array_like=True` struct. Keep the field order aligned with Python.
9 +#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
10 +pub struct LoraRequest {
11 + pub lora_name: String,
12 + pub lora_int_id: u64,
13 + pub lora_path: String,
14 + #[serde(default)]
15 + pub base_model_name: Option<String>,
16 + #[serde(default)]
17 + pub tensorizer_config_dict: Option<OpaqueValue>,
18 + #[serde(default)]
19 + pub load_inplace: bool,
20 + #[serde(default)]
21 + pub is_3d_lora_weight: bool,
22 +}
23 +
24 +impl LoraRequest {
25 + pub fn new(
26 + lora_name: String,
27 + lora_int_id: u64,
28 + lora_path: String,
29 + load_inplace: bool,
30 + is_3d_lora_weight: bool,
31 + ) -> Self {
32 + Self {
33 + lora_name,
34 + lora_int_id,
35 + lora_path,
36 + base_model_name: None,
37 + tensorizer_config_dict: None,
38 + load_inplace,
39 + is_3d_lora_weight,
40 + }
41 + }
42 +}
rust/src/engine-core-client/src/protocol/mod.rs
+2 -1
@@ -48,6 +48,7 @@ mod classified_outputs;
48 pub mod dtype;
49 pub mod handshake;
50 pub mod logprobs;
51 +pub mod lora;
52 pub mod multimodal;
53 pub mod stats;
54 pub mod tensor;
@@ -349,7 +350,7 @@ pub struct EngineCoreRequest {
350 pub pooling_params: Option<OpaqueValue>,
351 pub arrival_time: f64,
352 #[serde(default)]
352 - pub lora_request: Option<OpaqueValue>,
353 + pub lora_request: Option<lora::LoraRequest>,
354 #[serde(default)]
355 pub cache_salt: Option<String>,
356 #[serde(default)]
rust/src/llm/src/request.rs
+3 -2
@@ -2,8 +2,9 @@ use std::collections::BTreeMap;
2 use std::time::{SystemTime, UNIX_EPOCH};
3
4 use uuid::Uuid;
5 +use vllm_engine_core_client::protocol::lora::LoraRequest;
6 use vllm_engine_core_client::protocol::multimodal::MmFeatures;
6 -use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams, OpaqueValue};
7 +use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams};
8
9 use crate::error::{Error, Result};
10
@@ -34,7 +35,7 @@ pub struct GenerateRequest {
35 pub priority: i32,
36 pub data_parallel_rank: Option<u32>,
37 pub reasoning_ended: Option<bool>,
37 - pub lora_request: Option<OpaqueValue>,
38 + pub lora_request: Option<LoraRequest>,
39 }
40
41 #[derive(Debug)]
rust/src/server/src/grpc/convert.rs
+1
@@ -91,6 +91,7 @@ pub fn to_text_request(
91 cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
92 add_special_tokens: true,
93 data_parallel_rank: None,
94 + lora_request: None,
95 })
96 }
97
rust/src/server/src/lib.rs
+1
@@ -4,6 +4,7 @@ mod config;
4 mod error;
5 mod grpc;
6 mod listener;
7 +mod lora;
8 mod middleware;
9 mod routes;
10 mod state;
rust/src/server/src/lora.rs new
+168
@@ -0,0 +1,168 @@
1 +use std::collections::BTreeMap;
2 +use std::sync::atomic::{AtomicU64, Ordering};
3 +
4 +use tokio::sync::{Mutex, RwLock};
5 +use vllm_engine_core_client::EngineCoreClient;
6 +use vllm_engine_core_client::protocol::lora::LoraRequest;
7 +
8 +/// Snapshot of the currently served model names plus the requested LoRA, if
9 +/// the model name resolves to a dynamic adapter.
10 +#[derive(Debug, Clone)]
11 +pub(crate) struct LoraModelResolution {
12 + pub model_names: Vec<String>,
13 + pub lora_request: Option<LoraRequest>,
14 +}
15 +
16 +/// Runtime registry for dynamically loaded LoRA adapters.
17 +pub(crate) struct LoraManager {
18 + /// Dynamically loaded LoRA adapters keyed by public model name.
19 + requests: RwLock<BTreeMap<String, LoraRequest>>,
20 + /// Monotonic adapter id allocator. LoRA ids are one-indexed.
21 + id_counter: AtomicU64,
22 + /// Serialize dynamic LoRA registry updates around engine utility calls.
23 + update_lock: Mutex<()>,
24 +}
25 +
26 +#[derive(Debug)]
27 +pub(crate) enum LoadLoraError {
28 + AlreadyLoaded { lora_name: String },
29 + BaseModelName { lora_name: String },
30 + Engine(vllm_engine_core_client::Error),
31 + NotLoaded { lora_name: String },
32 +}
33 +
34 +#[derive(Debug)]
35 +pub(crate) enum UnloadLoraError {
36 + NotFound {
37 + lora_name: String,
38 + },
39 + IntIdMismatch {
40 + lora_name: String,
41 + expected: u64,
42 + actual: u64,
43 + },
44 + Engine(vllm_engine_core_client::Error),
45 + NotRemoved {
46 + lora_name: String,
47 + lora_int_id: u64,
48 + },
49 +}
50 +
51 +impl LoraManager {
52 + pub fn new() -> Self {
53 + Self {
54 + requests: RwLock::new(BTreeMap::new()),
55 + id_counter: AtomicU64::new(0),
56 + update_lock: Mutex::new(()),
57 + }
58 + }
59 +
60 + /// Return base served model names plus dynamically loaded LoRA adapter
61 + /// names.
62 + pub async fn served_model_names(&self, base_model_names: &[String]) -> Vec<String> {
63 + let mut names = base_model_names.to_vec();
64 + names.extend(self.requests.read().await.keys().cloned());
65 + names
66 + }
67 +
68 + /// Resolve the requested model against one consistent LoRA registry
69 + /// snapshot.
70 + pub async fn resolve_model(
71 + &self,
72 + base_model_names: &[String],
73 + model_name: Option<&str>,
74 + ) -> LoraModelResolution {
75 + let requests = self.requests.read().await;
76 + let mut model_names = base_model_names.to_vec();
77 + model_names.extend(requests.keys().cloned());
78 + let lora_request = model_name.and_then(|name| requests.get(name).cloned());
79 +
80 + LoraModelResolution {
81 + model_names,
82 + lora_request,
83 + }
84 + }
85 +
86 + /// Load one dynamic LoRA adapter and register it as a public model name.
87 + pub async fn load_lora(
88 + &self,
89 + engine_core_client: &EngineCoreClient,
90 + base_model_names: &[String],
91 + lora_name: String,
92 + lora_path: String,
93 + load_inplace: bool,
94 + is_3d_lora_weight: bool,
95 + ) -> Result<LoraRequest, LoadLoraError> {
96 + let _guard = self.update_lock.lock().await;
97 + if base_model_names.iter().any(|name| name == &lora_name) {
98 + return Err(LoadLoraError::BaseModelName { lora_name });
99 + }
100 + if !load_inplace && self.requests.read().await.contains_key(&lora_name) {
101 + return Err(LoadLoraError::AlreadyLoaded { lora_name });
102 + }
103 +
104 + let lora_int_id = self
105 + .requests
106 + .read()
107 + .await
108 + .get(&lora_name)
109 + .map(|request| request.lora_int_id)
110 + .unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1);
111 + let lora_request = LoraRequest::new(
112 + lora_name.clone(),
113 + lora_int_id,
114 + lora_path,
115 + load_inplace,
116 + is_3d_lora_weight,
117 + );
118 +
119 + let loaded = engine_core_client
120 + .add_lora(&lora_request)
121 + .await
122 + .map_err(LoadLoraError::Engine)?;
123 + if !loaded {
124 + return Err(LoadLoraError::NotLoaded { lora_name });
125 + }
126 + self.requests.write().await.insert(lora_name, lora_request.clone());
127 + Ok(lora_request)
128 + }
129 +
130 + /// Remove one dynamic LoRA adapter from the engine and public model
131 + /// registry.
132 + pub async fn unload_lora(
133 + &self,
134 + engine_core_client: &EngineCoreClient,
135 + lora_name: &str,
136 + requested_lora_int_id: Option<u64>,
137 + ) -> Result<LoraRequest, UnloadLoraError> {
138 + let _guard = self.update_lock.lock().await;
139 + let lora_request = self.requests.read().await.get(lora_name).cloned().ok_or_else(|| {
140 + UnloadLoraError::NotFound {
141 + lora_name: lora_name.to_string(),
142 + }
143 + })?;
144 +
145 + if let Some(actual) = requested_lora_int_id
146 + && actual != lora_request.lora_int_id
147 + {
148 + return Err(UnloadLoraError::IntIdMismatch {
149 + lora_name: lora_name.to_string(),
150 + expected: lora_request.lora_int_id,
151 + actual,
152 + });
153 + }
154 +
155 + let removed = engine_core_client
156 + .remove_lora(lora_request.lora_int_id)
157 + .await
158 + .map_err(UnloadLoraError::Engine)?;
159 + if !removed {
160 + return Err(UnloadLoraError::NotRemoved {
161 + lora_name: lora_request.lora_name,
162 + lora_int_id: lora_request.lora_int_id,
163 + });
164 + }
165 +
166 + Ok(self.requests.write().await.remove(lora_name).unwrap_or(lora_request))
167 + }
168 +}
rust/src/server/src/routes.rs
+36 -1
@@ -3,6 +3,7 @@ mod collective_rpc;
3 mod health;
4 mod inference;
5 mod load;
6 +mod lora;
7 mod metrics;
8 pub(crate) mod openai;
9 mod sleep;
@@ -25,12 +26,40 @@ fn server_dev_mode_enabled() -> bool {
26 .is_some_and(|value| value != 0)
27 }
28
29 +fn runtime_lora_updating_enabled() -> bool {
30 + std::env::var("VLLM_ALLOW_RUNTIME_LORA_UPDATING")
31 + .ok()
32 + .is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true"))
33 +}
34 +
35 /// Build the minimal OpenAI-compatible router for one configured model.
36 pub fn build_router(state: Arc<AppState>) -> Router {
30 - build_router_with_dev_mode(state, server_dev_mode_enabled())
37 + build_router_with_options(
38 + state,
39 + server_dev_mode_enabled(),
40 + runtime_lora_updating_enabled(),
41 + )
42 }
43
44 +#[cfg(test)]
45 fn build_router_with_dev_mode(state: Arc<AppState>, dev_mode_enabled: bool) -> Router {
46 + build_router_with_dev_mode_and_lora(state, dev_mode_enabled, false)
47 +}
48 +
49 +#[cfg(test)]
50 +fn build_router_with_dev_mode_and_lora(
51 + state: Arc<AppState>,
52 + dev_mode_enabled: bool,
53 + runtime_lora_updating_enabled: bool,
54 +) -> Router {
55 + build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled)
56 +}
57 +
58 +fn build_router_with_options(
59 + state: Arc<AppState>,
60 + dev_mode_enabled: bool,
61 + runtime_lora_updating_enabled: bool,
62 +) -> Router {
63 let mut router = Router::new()
64 // Health & monitoring
65 .route("/health", get(health::health))
@@ -44,6 +73,12 @@ fn build_router_with_dev_mode(state: Arc<AppState>, dev_mode_enabled: bool) -> R
73 // vLLM specific inference endpoints
74 .route("/inference/v1/generate", post(inference::generate));
75
76 + if runtime_lora_updating_enabled {
77 + router = router
78 + .route("/v1/load_lora_adapter", post(lora::load_lora_adapter))
79 + .route("/v1/unload_lora_adapter", post(lora::unload_lora_adapter));
80 + }
81 +
82 if dev_mode_enabled {
83 // Development-only
84 router = router
rust/src/server/src/routes/inference/generate.rs
+2 -2
@@ -42,8 +42,8 @@ pub async fn generate(
42 ValidatedJson(body): ValidatedJson<GenerateRequest>,
43 ) -> Response {
44 let request_context = resolve_request_context(&headers, body.request_id.as_deref());
45 - let prepared = match prepare_generate_request(body, state.served_model_names(), request_context)
46 - {
45 + let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
46 + let prepared = match prepare_generate_request(body, &lora_resolution, request_context) {
47 Ok(prepared) => prepared,
48 Err(error) => return error.into_response(),
49 };
rust/src/server/src/routes/inference/generate/convert.rs
+14 -4
@@ -3,6 +3,7 @@ use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
3 use super::types::GenerateRequest;
4 use super::validate;
5 use crate::error::ApiError;
6 +use crate::lora::LoraModelResolution;
7 use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params};
8
9 /// Lowered generate request plus the response request ID.
@@ -21,10 +22,10 @@ pub struct PreparedRequest {
22 /// text-generation format.
23 pub fn prepare_generate_request(
24 request: GenerateRequest,
24 - served_model_names: &[String],
25 + lora_resolution: &LoraModelResolution,
26 ctx: ResolvedRequestContext,
27 ) -> Result<PreparedRequest, ApiError> {
27 - validate::validate_request_compat(&request, served_model_names)?;
28 + validate::validate_request_compat(&request, &lora_resolution.model_names)?;
29
30 let stream = request.stream;
31 let include_usage = request
@@ -57,6 +58,7 @@ pub fn prepare_generate_request(
58 cache_salt: request.cache_salt,
59 add_special_tokens: false,
60 data_parallel_rank: ctx.data_parallel_rank,
61 + lora_request: lora_resolution.lora_request.clone(),
62 };
63
64 Ok(PreparedRequest {
@@ -76,9 +78,17 @@ mod tests {
78 use vllm_text::Prompt;
79
80 use super::prepare_generate_request;
81 + use crate::lora::LoraModelResolution;
82 use crate::routes::inference::generate::types::GenerateRequest;
83 use crate::utils::ResolvedRequestContext;
84
85 + fn served(names: &[&str]) -> LoraModelResolution {
86 + LoraModelResolution {
87 + model_names: names.iter().map(|s| s.to_string()).collect(),
88 + lora_request: None,
89 + }
90 + }
91 +
92 #[test]
93 fn prepare_generate_request_maps_token_prompt_and_sampling_params() {
94 let request: GenerateRequest = serde_json::from_value(json!({
@@ -100,7 +110,7 @@ mod tests {
110
111 let prepared = prepare_generate_request(
112 request,
103 - &["Qwen/Qwen1.5-0.5B-Chat".to_string()],
113 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
114 ResolvedRequestContext::default(),
115 )
116 .expect("prepare");
@@ -143,7 +153,7 @@ mod tests {
153
154 let prepared = prepare_generate_request(
155 request,
146 - &["Qwen/Qwen1.5-0.5B-Chat".to_string()],
156 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
157 ResolvedRequestContext::default(),
158 )
159 .expect("prepare");
rust/src/server/src/routes/lora.rs new
+296
@@ -0,0 +1,296 @@
1 +use std::path::{Component, Path, PathBuf};
2 +use std::sync::Arc;
3 +
4 +use axum::extract::State;
5 +use serde::Deserialize;
6 +use thiserror_ext::AsReport;
7 +use validator::Validate;
8 +
9 +use crate::error::ApiError;
10 +use crate::lora::{LoadLoraError, UnloadLoraError};
11 +use crate::routes::openai::utils::types::Normalizable;
12 +use crate::routes::openai::utils::validated_json::ValidatedJson;
13 +use crate::state::AppState;
14 +
15 +const RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV: &str = "VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES";
16 +
17 +#[derive(Debug, Deserialize, Validate)]
18 +pub(crate) struct LoadLoraAdapterRequest {
19 + lora_name: String,
20 + lora_path: String,
21 + #[serde(default)]
22 + load_inplace: bool,
23 + #[serde(default)]
24 + is_3d_lora_weight: bool,
25 +}
26 +
27 +impl Normalizable for LoadLoraAdapterRequest {}
28 +
29 +#[derive(Debug, Deserialize, Validate)]
30 +pub(crate) struct UnloadLoraAdapterRequest {
31 + lora_name: String,
32 + #[serde(default)]
33 + lora_int_id: Option<u64>,
34 +}
35 +
36 +impl Normalizable for UnloadLoraAdapterRequest {}
37 +
38 +fn runtime_lora_allowed_path_prefixes() -> Option<Vec<PathBuf>> {
39 + let prefixes = std::env::var_os(RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV)?;
40 + let prefixes: Vec<_> = std::env::split_paths(&prefixes)
41 + .filter(|path| !path.as_os_str().is_empty())
42 + .collect();
43 + (!prefixes.is_empty()).then_some(prefixes)
44 +}
45 +
46 +fn looks_like_local_lora_path(lora_path: &str) -> bool {
47 + let path = Path::new(lora_path);
48 + path.is_absolute()
49 + || lora_path.starts_with('~')
50 + || lora_path.starts_with('.')
51 + || path.components().any(|component| matches!(component, Component::ParentDir))
52 +}
53 +
54 +fn validate_lora_path_access(
55 + lora_path: &str,
56 + allowed_prefixes: Option<&[PathBuf]>,
57 +) -> Result<Option<String>, ApiError> {
58 + let path = Path::new(lora_path);
59 + if !looks_like_local_lora_path(lora_path) && !path.exists() {
60 + return Ok(None);
61 + }
62 +
63 + let Some(allowed_prefixes) = allowed_prefixes else {
64 + return Err(ApiError::invalid_request(
65 + format!(
66 + "Local LoRA adapter paths require {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV} to be configured."
67 + ),
68 + Some("lora_path"),
69 + ));
70 + };
71 +
72 + if !path.is_absolute() {
73 + return Err(ApiError::invalid_request(
74 + format!(
75 + "Local LoRA adapter paths must be absolute and under one of the prefixes configured by {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV}."
76 + ),
77 + Some("lora_path"),
78 + ));
79 + }
80 +
81 + let canonical_path = path.canonicalize().map_err(|_| {
82 + ApiError::invalid_request(
83 + "Local LoRA adapter path must exist and be accessible.".to_string(),
84 + Some("lora_path"),
85 + )
86 + })?;
87 + let canonical_prefixes = allowed_prefixes
88 + .iter()
89 + .map(|prefix| {
90 + prefix.canonicalize().map_err(|_| {
91 + ApiError::server_error(format!(
92 + "configured {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV} path prefix must exist and be accessible"
93 + ))
94 + })
95 + })
96 + .collect::<Result<Vec<_>, _>>()?;
97 +
98 + if !canonical_prefixes.iter().any(|prefix| canonical_path.starts_with(prefix)) {
99 + return Err(ApiError::invalid_request(
100 + "Local LoRA adapter path is outside the configured allowed prefixes.".to_string(),
101 + Some("lora_path"),
102 + ));
103 + }
104 +
105 + Ok(Some(canonical_path.to_string_lossy().into_owned()))
106 +}
107 +
108 +/// Dynamically load one LoRA adapter and expose it as an OpenAI model id.
109 +pub async fn load_lora_adapter(
110 + State(state): State<Arc<AppState>>,
111 + ValidatedJson(request): ValidatedJson<LoadLoraAdapterRequest>,
112 +) -> Result<String, ApiError> {
113 + if request.lora_name.is_empty() || request.lora_path.is_empty() {
114 + return Err(ApiError::invalid_request(
115 + "Both 'lora_name' and 'lora_path' must be provided.".to_string(),
116 + None,
117 + ));
118 + }
119 + let allowed_prefixes = runtime_lora_allowed_path_prefixes();
120 + let lora_path = validate_lora_path_access(&request.lora_path, allowed_prefixes.as_deref())?
121 + .unwrap_or(request.lora_path);
122 +
123 + let lora_name = request.lora_name;
124 + state
125 + .load_lora(
126 + lora_name.clone(),
127 + lora_path,
128 + request.load_inplace,
129 + request.is_3d_lora_weight,
130 + )
131 + .await
132 + .map_err(|error| match error {
133 + LoadLoraError::AlreadyLoaded { lora_name } => ApiError::invalid_request(
134 + format!(
135 + "The lora adapter '{lora_name}' has already been loaded. If you want to load the adapter in place, set 'load_inplace' to true."
136 + ),
137 + Some("lora_name"),
138 + ),
139 + LoadLoraError::BaseModelName { lora_name } => ApiError::invalid_request(
140 + format!("The lora adapter name '{lora_name}' conflicts with a served base model."),
141 + Some("lora_name"),
142 + ),
143 + LoadLoraError::Engine(error) => ApiError::server_error(format!(
144 + "failed to load LoRA adapter '{lora_name}': {}",
145 + error.to_report_string()
146 + )),
147 + LoadLoraError::NotLoaded { lora_name } => ApiError::server_error(format!(
148 + "failed to load LoRA adapter '{lora_name}': engine rejected the adapter"
149 + )),
150 + })?;
151 +
152 + Ok(format!(
153 + "Success: LoRA adapter '{lora_name}' added successfully."
154 + ))
155 +}
156 +
157 +/// Remove one LoRA adapter from the engine and frontend registry.
158 +pub async fn unload_lora_adapter(
159 + State(state): State<Arc<AppState>>,
160 + ValidatedJson(request): ValidatedJson<UnloadLoraAdapterRequest>,
161 +) -> Result<String, ApiError> {
162 + if request.lora_name.is_empty() {
163 + return Err(ApiError::invalid_request(
164 + "'lora_name' needs to be provided to unload a LoRA adapter.".to_string(),
165 + Some("lora_name"),
166 + ));
167 + }
168 +
169 + let lora_request = state
170 + .unload_lora(&request.lora_name, request.lora_int_id)
171 + .await
172 + .map_err(|error| match error {
173 + UnloadLoraError::NotFound { lora_name } => ApiError::model_not_found(lora_name),
174 + UnloadLoraError::IntIdMismatch {
175 + lora_name,
176 + expected,
177 + actual,
178 + } => ApiError::invalid_request(
179 + format!(
180 + "The requested lora_int_id {actual} does not match loaded adapter '{lora_name}' with id {expected}."
181 + ),
182 + Some("lora_int_id"),
183 + ),
184 + UnloadLoraError::Engine(error) => ApiError::server_error(format!(
185 + "failed to unload LoRA adapter '{}': {}",
186 + request.lora_name,
187 + error.to_report_string()
188 + )),
189 + UnloadLoraError::NotRemoved {
190 + lora_name,
191 + lora_int_id,
192 + } => ApiError::server_error(format!(
193 + "failed to unload LoRA adapter '{lora_name}' with id {lora_int_id}"
194 + )),
195 + })?;
196 +
197 + Ok(format!(
198 + "Success: LoRA adapter '{}' removed successfully.",
199 + lora_request.lora_name
200 + ))
201 +}
202 +
203 +#[cfg(test)]
204 +mod tests {
205 + use std::fs;
206 + use std::path::PathBuf;
207 + use std::time::{SystemTime, UNIX_EPOCH};
208 +
209 + use super::validate_lora_path_access;
210 +
211 + fn temp_lora_dir(test_name: &str) -> PathBuf {
212 + let suffix = SystemTime::now()
213 + .duration_since(UNIX_EPOCH)
214 + .expect("clock should be after unix epoch")
215 + .as_nanos();
216 + let path = std::env::temp_dir().join(format!(
217 + "vllm-lora-{test_name}-{}-{suffix}",
218 + std::process::id()
219 + ));
220 + fs::create_dir_all(&path).expect("create temp lora dir");
221 + path
222 + }
223 +
224 + #[test]
225 + fn lora_path_allows_hf_repo_ids_without_prefixes() {
226 + assert_eq!(
227 + validate_lora_path_access("org/adapter-a", None).expect("hf repo id should be allowed"),
228 + None
229 + );
230 + }
231 +
232 + #[test]
233 + fn lora_path_rejects_local_paths_without_prefixes() {
234 + assert!(validate_lora_path_access("/tmp/adapter-a", None).is_err());
235 + assert!(validate_lora_path_access("./adapter-a", None).is_err());
236 + assert!(validate_lora_path_access("~/adapter-a", None).is_err());
237 + assert!(validate_lora_path_access("subdir/../../../etc/sensitive", None).is_err());
238 + }
239 +
240 + #[test]
241 + fn lora_path_rejects_existing_bare_relative_paths_without_prefixes() {
242 + let root =
243 + PathBuf::from("target").join(format!("vllm-lora-relative-{}", std::process::id()));
244 + let adapter = root.join("adapter-a");
245 + fs::create_dir_all(&adapter).expect("create relative adapter dir");
246 +
247 + assert!(
248 + validate_lora_path_access(adapter.to_str().expect("utf-8 temp path"), None).is_err()
249 + );
250 +
251 + fs::remove_dir_all(root).ok();
252 + }
253 +
254 + #[test]
255 + fn lora_path_allows_absolute_paths_under_configured_prefixes() {
256 + let root = temp_lora_dir("allowed-prefix");
257 + let allowed = root.join("allowed");
258 + let adapter = allowed.join("adapter-a");
259 + fs::create_dir_all(&adapter).expect("create adapter dir");
260 +
261 + let prefixes = [allowed];
262 + let resolved =
263 + validate_lora_path_access(adapter.to_str().expect("utf-8 temp path"), Some(&prefixes))
264 + .expect("path under configured prefix should be allowed");
265 + assert_eq!(
266 + resolved.as_deref(),
267 + Some(
268 + adapter
269 + .canonicalize()
270 + .expect("canonical adapter")
271 + .to_str()
272 + .expect("utf-8 temp path")
273 + )
274 + );
275 +
276 + fs::remove_dir_all(root).ok();
277 + }
278 +
279 + #[test]
280 + fn lora_path_rejects_parent_escape_from_configured_prefixes() {
281 + let root = temp_lora_dir("parent-escape");
282 + let allowed = root.join("allowed");
283 + let private_adapter = root.join("private").join("adapter-a");
284 + fs::create_dir_all(&allowed).expect("create allowed dir");
285 + fs::create_dir_all(&private_adapter).expect("create private adapter dir");
286 +
287 + let escaped = allowed.join("../private/adapter-a");
288 + let prefixes = [allowed];
289 + assert!(
290 + validate_lora_path_access(escaped.to_str().expect("utf-8 temp path"), Some(&prefixes))
291 + .is_err()
292 + );
293 +
294 + fs::remove_dir_all(root).ok();
295 + }
296 +}
rust/src/server/src/routes/openai/chat_completions.rs
+2 -1
@@ -49,8 +49,9 @@ pub async fn chat_completions(
49 ) -> Response {
50 let stream = body.stream;
51 let request_context = resolve_request_context(&headers, body.request_id.as_deref());
52 + let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
53
53 - let prepared = match prepare_chat_request(body, state.served_model_names(), request_context) {
54 + let prepared = match prepare_chat_request(body, &lora_resolution, request_context) {
55 Ok(prepared) => prepared,
56 Err(error) => return error.into_response(),
57 };
rust/src/server/src/routes/openai/chat_completions/convert.rs
+18 -7
@@ -8,6 +8,7 @@ use vllm_chat::{
8 use super::types::ChatCompletionRequest;
9 use super::validate;
10 use crate::error::{ApiError, bail_invalid_request};
11 +use crate::lora::LoraModelResolution;
12 use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
13 use crate::routes::openai::utils::types::{
14 ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,
@@ -41,16 +42,21 @@ pub struct PreparedRequest {
42 /// Validate and lower one OpenAI chat completion request into the internal chat
43 /// format.
44 ///
44 -/// `served_model_names` must be non-empty; the first entry is used as the
45 -/// `model` field in responses.
45 +/// `lora_resolution.model_names` must be non-empty; the first entry is used as
46 +/// the base `model` field in responses when no LoRA adapter is selected.
47 pub(crate) fn prepare_chat_request(
48 request: ChatCompletionRequest,
48 - served_model_names: &[String],
49 + lora_resolution: &LoraModelResolution,
50 ctx: ResolvedRequestContext,
51 ) -> Result<PreparedRequest, ApiError> {
51 - validate::validate_request_compat(&request, served_model_names)?;
52 + validate::validate_request_compat(&request, &lora_resolution.model_names)?;
53
54 let request_id = format!("chatcmpl-{}", ctx.request_id);
55 + let response_model = lora_resolution
56 + .lora_request
57 + .as_ref()
58 + .map(|request| request.lora_name.clone())
59 + .unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default());
60 let echo = request
61 .echo
62 .then(|| extract_last_assistant_content(&request.messages))
@@ -131,11 +137,12 @@ pub(crate) fn prepare_chat_request(
137 cache_salt: request.cache_salt,
138 add_special_tokens: request.add_special_tokens,
139 data_parallel_rank: ctx.data_parallel_rank,
140 + lora_request: lora_resolution.lora_request.clone(),
141 };
142
143 Ok(PreparedRequest {
144 request_id,
138 - response_model: served_model_names.first().cloned().unwrap_or_default(),
145 + response_model,
146 include_usage,
147 requested_logprobs,
148 include_prompt_logprobs,
@@ -352,6 +359,7 @@ mod tests {
359 use vllm_text::output::TextDecodeOptions;
360
361 use super::prepare_chat_request;
362 + use crate::lora::LoraModelResolution;
363 use crate::routes::openai::chat_completions::types::{
364 AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
365 };
@@ -365,8 +373,11 @@ mod tests {
373 resolve_request_context(headers, request_id)
374 }
375
368 - fn served(names: &[&str]) -> Vec<String> {
369 - names.iter().map(|s| s.to_string()).collect()
376 + fn served(names: &[&str]) -> LoraModelResolution {
377 + LoraModelResolution {
378 + model_names: names.iter().map(|s| s.to_string()).collect(),
379 + lora_request: None,
380 + }
381 }
382
383 fn base_request() -> ChatCompletionRequest {
rust/src/server/src/routes/openai/completions.rs
+5 -5
@@ -44,12 +44,12 @@ pub async fn completions(
44 let stream = body.stream;
45 let logprobs = body.logprobs;
46 let request_context = resolve_request_context(&headers, body.request_id.as_deref());
47 + let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
48
48 - let prepared =
49 - match prepare_completion_request(body, state.served_model_names(), request_context) {
50 - Ok(prepared) => prepared,
51 - Err(error) => return error.into_response(),
52 - };
49 + let prepared = match prepare_completion_request(body, &lora_resolution, request_context) {
50 + Ok(prepared) => prepared,
51 + Err(error) => return error.into_response(),
52 + };
53 let request_span = tracing::info_span!(
54 "completions",
55 request_id = %prepared.request_id,
rust/src/server/src/routes/openai/completions/convert.rs
+18 -7
@@ -2,6 +2,7 @@ use vllm_text::{SamplingParams, TextDecodeOptions, TextRequest};
2
3 use super::types::CompletionRequest;
4 use crate::error::ApiError;
5 +use crate::lora::LoraModelResolution;
6 use crate::routes::openai::completions::validate;
7 use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
8 use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params};
@@ -30,16 +31,21 @@ pub struct PreparedRequest {
31 /// Validate and lower one OpenAI completions request into the internal
32 /// text-generation format.
33 ///
33 -/// `served_model_names` must be non-empty; the first entry is used as the
34 -/// `model` field in responses.
34 +/// `lora_resolution.model_names` must be non-empty; the first entry is used as
35 +/// the base `model` field in responses when no LoRA adapter is selected.
36 pub(crate) fn prepare_completion_request(
37 request: CompletionRequest,
37 - served_model_names: &[String],
38 + lora_resolution: &LoraModelResolution,
39 ctx: ResolvedRequestContext,
40 ) -> Result<PreparedRequest, ApiError> {
40 - validate::validate_request_compat(&request, served_model_names)?;
41 + validate::validate_request_compat(&request, &lora_resolution.model_names)?;
42
43 let request_id = format!("cmpl-{}", ctx.request_id);
44 + let response_model = lora_resolution
45 + .lora_request
46 + .as_ref()
47 + .map(|request| request.lora_name.clone())
48 + .unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default());
49
50 let logprobs = match request.logprobs {
51 Some(logprobs) => Some(i32::try_from(logprobs).map_err(|_| {
@@ -104,11 +110,12 @@ pub(crate) fn prepare_completion_request(
110 cache_salt: request.cache_salt,
111 add_special_tokens: request.add_special_tokens,
112 data_parallel_rank: ctx.data_parallel_rank,
113 + lora_request: lora_resolution.lora_request.clone(),
114 };
115
116 Ok(PreparedRequest {
117 request_id,
111 - response_model: served_model_names.first().cloned().unwrap_or_default(),
118 + response_model,
119 include_usage,
120 text_request,
121 echo,
@@ -124,6 +131,7 @@ mod tests {
131 use vllm_text::Prompt;
132
133 use super::prepare_completion_request;
134 + use crate::lora::LoraModelResolution;
135 use crate::routes::openai::completions::types::CompletionRequest;
136 use crate::utils::{ResolvedRequestContext, resolve_request_context};
137
@@ -131,8 +139,11 @@ mod tests {
139 resolve_request_context(headers, request_id)
140 }
141
134 - fn served(names: &[&str]) -> Vec<String> {
135 - names.iter().map(|s| s.to_string()).collect()
142 + fn served(names: &[&str]) -> LoraModelResolution {
143 + LoraModelResolution {
144 + model_names: names.iter().map(|s| s.to_string()).collect(),
145 + lora_request: None,
146 + }
147 }
148
149 fn base_request_json() -> serde_json::Value {
rust/src/server/src/routes/openai/models.rs
+4 -4
@@ -8,13 +8,13 @@ use crate::state::AppState;
8
9 /// Return all configured served model names in OpenAI `list models` format.
10 pub async fn list_models(State(state): State<Arc<AppState>>) -> Json<ListModelsResponse> {
11 + let model_names = state.served_model_names_with_loras().await;
12 Json(ListModelsResponse {
13 object: "list".to_string(),
13 - data: state
14 - .served_model_names()
15 - .iter()
14 + data: model_names
15 + .into_iter()
16 .map(|name| ModelObject {
17 - id: name.clone(),
17 + id: name,
18 object: "model".to_string(),
19 created: 0,
20 owned_by: "vllm-frontend-rs".to_string(),
rust/src/server/src/routes/tests.rs
+376 -3
@@ -43,7 +43,8 @@ use vllm_text::{Prompt, TextBackend};
43 use zeromq::prelude::{SocketRecv, SocketSend};
44 use zeromq::{DealerSocket, PushSocket, ZmqMessage};
45
46 -use super::{build_router, build_router_with_dev_mode};
46 +use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
47 +use crate::lora::LoraModelResolution;
48 use crate::routes::openai::chat_completions::convert::prepare_chat_request;
49 use crate::state::AppState;
50
@@ -141,6 +142,13 @@ fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason
142 ]
143 }
144
145 +fn assert_adapter_a_lora_request(request: &EngineCoreRequest) {
146 + let lora = request.lora_request.as_ref().expect("lora request");
147 + assert_eq!(lora.lora_name, "adapter-a");
148 + assert_eq!(lora.lora_int_id, 1);
149 + assert_eq!(lora.lora_path, "org/adapter-a");
150 +}
151 +
152 fn sse_data_payloads(text: &str) -> Vec<&str> {
153 text.lines().filter_map(|line| line.strip_prefix("data: ")).collect()
154 }
@@ -822,12 +830,13 @@ where
830
831 let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
832 (
825 - build_router_with_dev_mode(
833 + build_router_with_dev_mode_and_lora(
834 Arc::new(AppState::new(
835 vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
836 chat,
837 )),
838 true,
839 + true,
840 ),
841 engine_task,
842 )
@@ -1078,6 +1087,367 @@ async fn version_returns_engine_vllm_version() {
1087 );
1088 }
1089
1090 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1091 +#[serial]
1092 +async fn load_lora_adapter_registers_model_and_forwards_lora_request() {
1093 + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
1094 + boxed_test_future(async move {
1095 + let utility = recv_engine_message(dealer).await;
1096 + assert_eq!(utility[0].as_ref(), &[0x03]);
1097 +
1098 + let payload = decode_value(&utility[1]).expect("decode utility payload");
1099 + let array = payload.as_array().expect("utility payload array");
1100 + let call_id = array[1].as_u64().expect("call id");
1101 + assert_eq!(array[2], Value::from("add_lora"));
1102 +
1103 + let args = array[3].as_array().expect("utility args");
1104 + let lora = args[0].as_array().expect("lora request tuple");
1105 + assert_eq!(lora[0], Value::from("adapter-a"));
1106 + assert_eq!(lora[1], Value::from(1));
1107 + assert_eq!(lora[2], Value::from("org/adapter-a"));
1108 +
1109 + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
1110 +
1111 + let add = recv_engine_message(dealer).await;
1112 + assert_eq!(add[0].as_ref(), &[0x00]);
1113 + let request: EngineCoreRequest =
1114 + rmp_serde::from_slice(&add[1]).expect("decode engine request");
1115 + assert_adapter_a_lora_request(&request);
1116 +
1117 + send_outputs(
1118 + push,
1119 + engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
1120 + )
1121 + .await;
1122 +
1123 + let add = recv_engine_message(dealer).await;
1124 + assert_eq!(add[0].as_ref(), &[0x00]);
1125 + let request: EngineCoreRequest =
1126 + rmp_serde::from_slice(&add[1]).expect("decode engine request");
1127 + assert_adapter_a_lora_request(&request);
1128 +
1129 + send_outputs(
1130 + push,
1131 + engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
1132 + )
1133 + .await;
1134 +
1135 + let add = recv_engine_message(dealer).await;
1136 + assert_eq!(add[0].as_ref(), &[0x00]);
1137 + let request: EngineCoreRequest =
1138 + rmp_serde::from_slice(&add[1]).expect("decode engine request");
1139 + assert_eq!(request.prompt_token_ids.as_deref(), Some(&[11, 22][..]));
1140 + assert_adapter_a_lora_request(&request);
1141 +
1142 + send_outputs(
1143 + push,
1144 + engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
1145 + )
1146 + .await;
1147 +
1148 + let utility = recv_engine_message(dealer).await;
1149 + assert_eq!(utility[0].as_ref(), &[0x03]);
1150 +
1151 + let payload = decode_value(&utility[1]).expect("decode utility payload");
1152 + let array = payload.as_array().expect("utility payload array");
1153 + let call_id = array[1].as_u64().expect("call id");
1154 + assert_eq!(array[2], Value::from("remove_lora"));
1155 +
1156 + let args = array[3].as_array().expect("utility args");
1157 + assert_eq!(args[0], Value::from(1));
1158 +
1159 + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
1160 + })
1161 + })
1162 + .await;
1163 +
1164 + let response = app
1165 + .call(
1166 + Request::builder()
1167 + .method("POST")
1168 + .uri("/v1/load_lora_adapter")
1169 + .header("content-type", "application/json")
1170 + .body(Body::from(
1171 + json!({
1172 + "lora_name": "adapter-a",
1173 + "lora_path": "org/adapter-a"
1174 + })
1175 + .to_string(),
1176 + ))
1177 + .expect("build request"),
1178 + )
1179 + .await
1180 + .expect("call app");
1181 + assert_eq!(response.status(), StatusCode::OK);
1182 +
1183 + let models = app
1184 + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
1185 + .await
1186 + .expect("call app");
1187 + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
1188 + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1189 + assert_eq!(json["data"][1]["id"], "adapter-a");
1190 +
1191 + let response = app
1192 + .call(
1193 + Request::builder()
1194 + .method("POST")
1195 + .uri("/v1/completions")
1196 + .header("content-type", "application/json")
1197 + .body(Body::from(
1198 + json!({
1199 + "model": "adapter-a",
1200 + "prompt": "hello",
1201 + "max_tokens": 2
1202 + })
1203 + .to_string(),
1204 + ))
1205 + .expect("build request"),
1206 + )
1207 + .await
1208 + .expect("call app");
1209 + assert_eq!(response.status(), StatusCode::OK);
1210 +
1211 + let response = app
1212 + .call(
1213 + Request::builder()
1214 + .method("POST")
1215 + .uri("/v1/chat/completions")
1216 + .header("content-type", "application/json")
1217 + .body(Body::from(
1218 + json!({
1219 + "model": "adapter-a",
1220 + "stream": false,
1221 + "messages": [{"role": "user", "content": "hello"}]
1222 + })
1223 + .to_string(),
1224 + ))
1225 + .expect("build request"),
1226 + )
1227 + .await
1228 + .expect("call app");
1229 + assert_eq!(response.status(), StatusCode::OK);
1230 +
1231 + let response = app
1232 + .call(
1233 + Request::builder()
1234 + .method("POST")
1235 + .uri("/inference/v1/generate")
1236 + .header("content-type", "application/json")
1237 + .body(Body::from(
1238 + json!({
1239 + "model": "adapter-a",
1240 + "token_ids": [11, 22],
1241 + "stream": false,
1242 + "sampling_params": {
1243 + "max_tokens": 2
1244 + }
1245 + })
1246 + .to_string(),
1247 + ))
1248 + .expect("build request"),
1249 + )
1250 + .await
1251 + .expect("call app");
1252 + assert_eq!(response.status(), StatusCode::OK);
1253 +
1254 + let response = app
1255 + .call(
1256 + Request::builder()
1257 + .method("POST")
1258 + .uri("/v1/unload_lora_adapter")
1259 + .header("content-type", "application/json")
1260 + .body(Body::from(
1261 + json!({
1262 + "lora_name": "adapter-a",
1263 + "lora_int_id": 1
1264 + })
1265 + .to_string(),
1266 + ))
1267 + .expect("build request"),
1268 + )
1269 + .await
1270 + .expect("call app");
1271 + assert_eq!(response.status(), StatusCode::OK);
1272 +
1273 + let models = app
1274 + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
1275 + .await
1276 + .expect("call app");
1277 + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
1278 + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1279 + assert_eq!(json["data"].as_array().expect("model data").len(), 1);
1280 +
1281 + let response = app
1282 + .call(
1283 + Request::builder()
1284 + .method("POST")
1285 + .uri("/v1/completions")
1286 + .header("content-type", "application/json")
1287 + .body(Body::from(
1288 + json!({
1289 + "model": "adapter-a",
1290 + "prompt": "hello",
1291 + "max_tokens": 2
1292 + })
1293 + .to_string(),
1294 + ))
1295 + .expect("build request"),
1296 + )
1297 + .await
1298 + .expect("call app");
1299 + assert_eq!(response.status(), StatusCode::NOT_FOUND);
1300 +
1301 + drop(app);
1302 + engine_task.finish().await;
1303 +}
1304 +
1305 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1306 +#[serial]
1307 +async fn unload_lora_adapter_rejects_mismatched_lora_int_id() {
1308 + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
1309 + boxed_test_future(async move {
1310 + let utility = recv_engine_message(dealer).await;
1311 + assert_eq!(utility[0].as_ref(), &[0x03]);
1312 +
1313 + let payload = decode_value(&utility[1]).expect("decode utility payload");
1314 + let array = payload.as_array().expect("utility payload array");
1315 + let call_id = array[1].as_u64().expect("call id");
1316 + assert_eq!(array[2], Value::from("add_lora"));
1317 +
1318 + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
1319 + })
1320 + })
1321 + .await;
1322 +
1323 + let response = app
1324 + .call(
1325 + Request::builder()
1326 + .method("POST")
1327 + .uri("/v1/load_lora_adapter")
1328 + .header("content-type", "application/json")
1329 + .body(Body::from(
1330 + json!({
1331 + "lora_name": "adapter-a",
1332 + "lora_path": "org/adapter-a"
1333 + })
1334 + .to_string(),
1335 + ))
1336 + .expect("build request"),
1337 + )
1338 + .await
1339 + .expect("call app");
1340 + assert_eq!(response.status(), StatusCode::OK);
1341 +
1342 + let response = app
1343 + .call(
1344 + Request::builder()
1345 + .method("POST")
1346 + .uri("/v1/unload_lora_adapter")
1347 + .header("content-type", "application/json")
1348 + .body(Body::from(
1349 + json!({
1350 + "lora_name": "adapter-a",
1351 + "lora_int_id": 99
1352 + })
1353 + .to_string(),
1354 + ))
1355 + .expect("build request"),
1356 + )
1357 + .await
1358 + .expect("call app");
1359 + assert_eq!(response.status(), StatusCode::BAD_REQUEST);
1360 +
1361 + let models = app
1362 + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
1363 + .await
1364 + .expect("call app");
1365 + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
1366 + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1367 + assert_eq!(json["data"][1]["id"], "adapter-a");
1368 +
1369 + drop(app);
1370 + engine_task.finish().await;
1371 +}
1372 +
1373 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1374 +#[serial]
1375 +async fn load_lora_adapter_rejects_engine_false_result() {
1376 + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
1377 + boxed_test_future(async move {
1378 + let utility = recv_engine_message(dealer).await;
1379 + assert_eq!(utility[0].as_ref(), &[0x03]);
1380 +
1381 + let payload = decode_value(&utility[1]).expect("decode utility payload");
1382 + let array = payload.as_array().expect("utility payload array");
1383 + let call_id = array[1].as_u64().expect("call id");
1384 + assert_eq!(array[2], Value::from("add_lora"));
1385 +
1386 + send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await;
1387 + })
1388 + })
1389 + .await;
1390 +
1391 + let response = app
1392 + .call(
1393 + Request::builder()
1394 + .method("POST")
1395 + .uri("/v1/load_lora_adapter")
1396 + .header("content-type", "application/json")
1397 + .body(Body::from(
1398 + json!({
1399 + "lora_name": "adapter-a",
1400 + "lora_path": "org/adapter-a"
1401 + })
1402 + .to_string(),
1403 + ))
1404 + .expect("build request"),
1405 + )
1406 + .await
1407 + .expect("call app");
1408 + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
1409 +
1410 + let models = app
1411 + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
1412 + .await
1413 + .expect("call app");
1414 + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
1415 + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1416 + assert_eq!(json["data"].as_array().expect("model data").len(), 1);
1417 +
1418 + drop(app);
1419 + engine_task.finish().await;
1420 +}
1421 +
1422 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1423 +#[serial]
1424 +async fn load_lora_adapter_rejects_base_model_name_collision() {
1425 + let (mut app, engine_task) =
1426 + test_admin_app_with_engine_script(|_, _| boxed_test_future(async move {})).await;
1427 +
1428 + let response = app
1429 + .call(
1430 + Request::builder()
1431 + .method("POST")
1432 + .uri("/v1/load_lora_adapter")
1433 + .header("content-type", "application/json")
1434 + .body(Body::from(
1435 + json!({
1436 + "lora_name": "Qwen/Qwen1.5-0.5B-Chat",
1437 + "lora_path": "org/adapter-a"
1438 + })
1439 + .to_string(),
1440 + ))
1441 + .expect("build request"),
1442 + )
1443 + .await
1444 + .expect("call app");
1445 + assert_eq!(response.status(), StatusCode::BAD_REQUEST);
1446 +
1447 + drop(app);
1448 + engine_task.finish().await;
1449 +}
1450 +
1451 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1452 #[serial]
1453 async fn http_metrics_record_list_models_requests() {
@@ -3000,7 +3370,10 @@ async fn prepared_openai_request_streams_text_events() {
3370 "messages": [{"role": "user", "content": "hello"}]
3371 }))
3372 .expect("decode request"),
3003 - &["Qwen/Qwen1.5-0.5B-Chat".to_string()],
3373 + &LoraModelResolution {
3374 + model_names: vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
3375 + lora_request: None,
3376 + },
3377 crate::utils::ResolvedRequestContext::default(),
3378 )
3379 .expect("prepare request");
rust/src/server/src/state.rs
+49
@@ -5,6 +5,9 @@ use tokio::time::{Duration, Instant, sleep_until};
5 use tracing::warn;
6 use vllm_chat::ChatLlm;
7 use vllm_engine_core_client::EngineCoreClient;
8 +use vllm_engine_core_client::protocol::lora::LoraRequest;
9 +
10 +use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
11
12 const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
13
@@ -21,6 +24,8 @@ pub struct AppState {
24 pub enable_request_id_headers: bool,
25 /// Number of in-flight inference requests currently owned by this frontend.
26 server_load: AtomicU64,
27 + /// Dynamic LoRA adapter registry.
28 + lora_manager: LoraManager,
29 }
30
31 impl AppState {
@@ -43,6 +48,7 @@ impl AppState {
48 enable_log_requests: false,
49 enable_request_id_headers: false,
50 server_load: AtomicU64::new(0),
51 + lora_manager: LoraManager::new(),
52 }
53 }
54
@@ -69,6 +75,49 @@ impl AppState {
75 &self.served_model_names
76 }
77
78 + /// Return base served model names plus dynamically loaded LoRA adapter
79 + /// names.
80 + pub async fn served_model_names_with_loras(&self) -> Vec<String> {
81 + self.lora_manager.served_model_names(&self.served_model_names).await
82 + }
83 +
84 + /// Resolve the requested model against one dynamic LoRA registry snapshot.
85 + pub async fn resolve_model_with_loras(&self, model_name: Option<&str>) -> LoraModelResolution {
86 + self.lora_manager.resolve_model(&self.served_model_names, model_name).await
87 + }
88 +
89 + /// Load one dynamic LoRA adapter and register it as a public model name.
90 + pub async fn load_lora(
91 + &self,
92 + lora_name: String,
93 + lora_path: String,
94 + load_inplace: bool,
95 + is_3d_lora_weight: bool,
96 + ) -> Result<LoraRequest, LoadLoraError> {
97 + self.lora_manager
98 + .load_lora(
99 + self.engine_core_client(),
100 + &self.served_model_names,
101 + lora_name,
102 + lora_path,
103 + load_inplace,
104 + is_3d_lora_weight,
105 + )
106 + .await
107 + }
108 +
109 + /// Remove one dynamic LoRA adapter from the engine and public model
110 + /// registry.
111 + pub async fn unload_lora(
112 + &self,
113 + lora_name: &str,
114 + lora_int_id: Option<u64>,
115 + ) -> Result<LoraRequest, UnloadLoraError> {
116 + self.lora_manager
117 + .unload_lora(self.engine_core_client(), lora_name, lora_int_id)
118 + .await
119 + }
120 +
121 /// Return a reference to the underlying engine core client for utility
122 /// calls.
123 pub(crate) fn engine_core_client(&self) -> &EngineCoreClient {
rust/src/text/src/lower.rs
+1 -1
@@ -40,11 +40,11 @@ pub fn lower_text_request(
40 cache_salt: request.cache_salt.clone(),
41 priority: request.priority,
42 data_parallel_rank: request.data_parallel_rank,
43 + lora_request: request.lora_request.clone(),
44 // Fields below are currently placeholders.
45 arrival_time: None,
46 trace_headers: None,
47 reasoning_ended: None,
47 - lora_request: None,
48 };
49
50 Ok(PreparedTextRequest {
rust/src/text/src/request.rs
+5
@@ -4,6 +4,7 @@ use enum_as_inner::EnumAsInner;
4 use serde::{Deserialize, Serialize};
5 use serde_json::Value;
6 use vllm_engine_core_client::protocol::StructuredOutputsParams;
7 +use vllm_engine_core_client::protocol::lora::LoraRequest;
8 use vllm_engine_core_client::protocol::multimodal::MmFeatures;
9
10 use crate::error::{Error, Result};
@@ -166,6 +167,9 @@ pub struct TextRequest {
167 /// Override data parallel rank.
168 #[serde(default)]
169 pub data_parallel_rank: Option<u32>,
170 + /// LoRA adapter selected for this request.
171 + #[serde(default)]
172 + pub lora_request: Option<LoraRequest>,
173 }
174
175 impl TextRequest {
@@ -182,6 +186,7 @@ impl TextRequest {
186 cache_salt: None,
187 add_special_tokens: false,
188 data_parallel_rank: None,
189 + lora_request: None,
190 }
191 }
192