@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 2db1c4dc31

[Rust Frontend] Support `--lora-modules` for static adapter loading (#54837)

Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Will Eaton <weaton@redhat.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Will Eaton committed Sep 3, 2026 at 15:12 UTC 2db1c4dc31c58bc4fd1f5fc48dbfceebc598e502
12 files changed +497 -99
rust/src/cmd/src/cli.rs
+11 -2
@@ -29,8 +29,8 @@ use vllm_managed_engine::ManagedEngineConfig;
29 use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
30 use vllm_server::{
31 ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
32 - DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RenderConfig, RendererSelection,
33 - TlsConfig,
32 + DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
33 + RendererSelection, TlsConfig,
34 };
35
36 use crate::cli::unsupported::UnsupportedArgs;
@@ -269,6 +269,13 @@ pub struct SharedRuntimeArgs {
269 #[serde(default)]
270 pub limit_mm_per_prompt: MmLimitPerPrompt,
271
272 + /// LoRA adapters to load before serving, each as `name=path` or a JSON
273 + /// object: `{"name": "name", "path": "lora_path", "base_model_name": "id"}`.
274 + /// Requires `--enable-lora`; startup fails if any adapter cannot be loaded.
275 + #[arg(long, num_args = 1.., value_name = "MODULE")]
276 + #[serde(default)]
277 + pub lora_modules: Vec<LoraModulePath>,
278 +
279 /// The format to render message content within a chat template.
280 ///
281 /// * "auto" detects the format from the template
@@ -498,6 +505,7 @@ impl SharedRuntimeArgs {
505 chat_template: self.chat_template,
506 default_chat_template_kwargs: self.default_chat_template_kwargs,
507 limit_mm_per_prompt: self.limit_mm_per_prompt,
508 + lora_modules: self.lora_modules,
509 chat_template_content_format: self.chat_template_content_format,
510 max_logprobs: self.max_logprobs,
511 api_server_options,
@@ -552,6 +560,7 @@ impl SharedRuntimeArgs {
560 chat_template: self.chat_template,
561 default_chat_template_kwargs: self.default_chat_template_kwargs,
562 limit_mm_per_prompt: self.limit_mm_per_prompt,
563 + lora_modules: self.lora_modules,
564 chat_template_content_format: self.chat_template_content_format,
565 max_logprobs: self.max_logprobs,
566 api_server_options,
rust/src/cmd/src/cli/tests.rs
+105 -1
@@ -4,7 +4,8 @@
4 use expect_test::expect;
5 use vllm_engine_core_client::TransportMode;
6 use vllm_server::{
7 - Config, GenerationConfigMode, HttpListenerMode, ParserSelection, RendererSelection,
7 + Config, GenerationConfigMode, HttpListenerMode, LoraModulePath, ParserSelection,
8 + RendererSelection,
9 };
10
11 use super::{BenchCommand, Cli, Command};
@@ -103,6 +104,7 @@ fn serve_args_forward_python_flags_with_separator() {
104 chat_template: None,
105 default_chat_template_kwargs: None,
106 limit_mm_per_prompt: {},
107 + lora_modules: [],
108 chat_template_content_format: Auto,
109 enable_log_requests: false,
110 enable_prompt_tokens_details: false,
@@ -623,6 +625,103 @@ fn frontend_args_json_passes_enable_request_id_headers_into_config() {
625 assert!(config.api_server_options.enable_request_id_headers);
626 }
627
628 +#[test]
629 +fn serve_args_parse_lora_modules_in_both_forms() {
630 + let cli = Cli::try_parse_from([
631 + "vllm-rs",
632 + "serve",
633 + "Qwen/Qwen3-0.6B",
634 + "--enable-lora",
635 + "--lora-modules",
636 + "alice=charent/self_cognition_Alice",
637 + r#"{"name": "bob", "path": "/adapters/bob", "base_model_name": "base"}"#,
638 + "--lora-modules",
639 + "carol=org/carol",
640 + ])
641 + .unwrap();
642 +
643 + let Command::Serve(args) = cli.command else {
644 + panic!("expected serve args");
645 + };
646 + // Frontend-only flag: the managed Python engine must not see it.
647 + assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]);
648 + let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
649 + expect![[r#"
650 + [
651 + LoraModulePath {
652 + name: "alice",
653 + path: "charent/self_cognition_Alice",
654 + base_model_name: None,
655 + is_3d_lora_weight: false,
656 + },
657 + LoraModulePath {
658 + name: "bob",
659 + path: "/adapters/bob",
660 + base_model_name: Some(
661 + "base",
662 + ),
663 + is_3d_lora_weight: false,
664 + },
665 + LoraModulePath {
666 + name: "carol",
667 + path: "org/carol",
668 + base_model_name: None,
669 + is_3d_lora_weight: false,
670 + },
671 + ]
672 + "#]]
673 + .assert_debug_eq(&config.lora_modules);
674 +}
675 +
676 +#[test]
677 +fn serve_args_reject_malformed_lora_module() {
678 + let error = Cli::try_parse_from([
679 + "vllm-rs",
680 + "serve",
681 + "Qwen/Qwen3-0.6B",
682 + "--lora-modules",
683 + "alice",
684 + ])
685 + .unwrap_err();
686 +
687 + expect![[r#"
688 + error: invalid value 'alice' for '--lora-modules <MODULE>...': expected `name=path`, got `alice`
689 +
690 + For more information, try '--help'.
691 + "#]].assert_eq(&error.to_string());
692 +}
693 +
694 +#[test]
695 +fn frontend_args_json_passes_lora_modules_into_config() {
696 + let cli = Cli::try_parse_from([
697 + "vllm-rs",
698 + "frontend",
699 + "--listen-fd",
700 + "3",
701 + "--input-address",
702 + "ipc:///tmp/input.sock",
703 + "--output-address",
704 + "ipc:///tmp/output.sock",
705 + "--args-json",
706 + r#"{"model_tag":"Qwen/Qwen3-0.6B","lora_modules":[{"name":"alice","path":"org/alice","base_model_name":null,"is_3d_lora_weight":true}]}"#,
707 + ])
708 + .unwrap();
709 +
710 + let Command::Frontend(args) = cli.command else {
711 + panic!("expected frontend args");
712 + };
713 + let config = args.into_config();
714 + assert_eq!(
715 + config.lora_modules,
716 + vec![LoraModulePath {
717 + name: "alice".to_string(),
718 + path: "org/alice".to_string(),
719 + base_model_name: None,
720 + is_3d_lora_weight: true,
721 + }]
722 + );
723 +}
724 +
725 #[test]
726 fn serve_passes_api_keys_into_config() {
727 let cli = Cli::try_parse_from([
@@ -822,6 +921,7 @@ fn frontend_args_accept_json() {
921 chat_template: None,
922 default_chat_template_kwargs: None,
923 limit_mm_per_prompt: {},
924 + lora_modules: [],
925 chat_template_content_format: Auto,
926 enable_log_requests: false,
927 enable_prompt_tokens_details: false,
@@ -1416,6 +1516,7 @@ fn serve_args_accept_handshake_aliases() {
1516 chat_template: None,
1517 default_chat_template_kwargs: None,
1518 limit_mm_per_prompt: {},
1519 + lora_modules: [],
1520 chat_template_content_format: Auto,
1521 enable_log_requests: false,
1522 enable_prompt_tokens_details: false,
@@ -1561,6 +1662,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
1662 chat_template: None,
1663 default_chat_template_kwargs: None,
1664 limit_mm_per_prompt: {},
1665 + lora_modules: [],
1666 chat_template_content_format: Auto,
1667 max_logprobs: None,
1668 api_server_options: ApiServerOptions {
@@ -1647,6 +1749,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
1749 chat_template: None,
1750 default_chat_template_kwargs: None,
1751 limit_mm_per_prompt: {},
1752 + lora_modules: [],
1753 chat_template_content_format: Auto,
1754 max_logprobs: None,
1755 api_server_options: ApiServerOptions {
@@ -1754,6 +1857,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
1857 chat_template: None,
1858 default_chat_template_kwargs: None,
1859 limit_mm_per_prompt: {},
1860 + lora_modules: [],
1861 chat_template_content_format: Auto,
1862 max_logprobs: None,
1863 api_server_options: ApiServerOptions {
rust/src/cmd/src/cli/unsupported.rs
-7
@@ -353,13 +353,6 @@ pub struct EngineUnsupportedArgs {
353 #[serde_with::skip_serializing_none]
354 #[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
355 pub struct ServerUnsupportedArgs {
356 - /// LoRA modules configurations in either 'name=path' format or JSON format
357 - /// or JSON list format. Example (old format): `'name=path'` Example (new
358 - /// format): `{"name": "name", "path": "lora_path",
359 - /// "base_model_name": "id"}`
360 - #[arg(long)]
361 - pub lora_modules: Option<Unsupported>,
362 -
356 /// Whether to trust the chat template provided in the request. If False,
357 /// the server will always use the chat template specified by
358 /// `--chat-template` or the ones from tokenizer.
rust/src/engine-core-client/src/protocol/lora.rs
-41
@@ -2,7 +2,6 @@
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 use serde_tuple::{Deserialize_tuple, Serialize_tuple};
5 -use thiserror::Error;
5
6 use crate::protocol::OpaqueValue;
7
@@ -24,43 +23,3 @@ pub struct LoraRequest {
23 #[serde(default)]
24 pub is_3d_lora_weight: bool,
25 }
27 -
28 -#[derive(Debug, Error, PartialEq, Eq)]
29 -pub enum LoraRequestError {
30 - #[error("lora_name must not be empty")]
31 - EmptyName,
32 - #[error("lora_int_id must be nonzero")]
33 - ZeroId,
34 - #[error("lora_path must not be empty")]
35 - EmptyPath,
36 -}
37 -
38 -impl LoraRequest {
39 - pub fn new(
40 - lora_name: String,
41 - lora_int_id: u64,
42 - lora_path: String,
43 - load_inplace: bool,
44 - is_3d_lora_weight: bool,
45 - ) -> Result<Self, LoraRequestError> {
46 - if lora_name.trim().is_empty() {
47 - return Err(LoraRequestError::EmptyName);
48 - }
49 - if lora_int_id == 0 {
50 - return Err(LoraRequestError::ZeroId);
51 - }
52 - if lora_path.trim().is_empty() {
53 - return Err(LoraRequestError::EmptyPath);
54 - }
55 -
56 - Ok(Self {
57 - lora_name,
58 - lora_int_id,
59 - lora_path,
60 - base_model_name: None,
61 - tensorizer_config_dict: None,
62 - load_inplace,
63 - is_3d_lora_weight,
64 - })
65 - }
66 -}
rust/src/server/examples/external_engine_openai_qwen.rs
+1
@@ -73,6 +73,7 @@ async fn main() -> Result<()> {
73 chat_template: None,
74 default_chat_template_kwargs: None,
75 limit_mm_per_prompt: HashMap::new(),
76 + lora_modules: Vec::new(),
77 chat_template_content_format: ChatTemplateContentFormatOption::Auto,
78 max_logprobs: None,
79 api_server_options: ApiServerOptions::default(),
rust/src/server/src/config.rs
+110 -1
@@ -3,12 +3,13 @@
3
4 use std::collections::HashMap;
5 use std::fmt;
6 +use std::str::FromStr;
7 use std::time::Duration;
8
9 use anyhow::{Result, bail};
10 use axum::http::{HeaderName, HeaderValue, Method};
11 use educe::Educe;
11 -use serde::Serialize;
12 +use serde::{Deserialize, Serialize};
13 use serde_json::Value;
14 use vllm_chat::multimodal::MmLimitPerPrompt;
15 use vllm_chat::{
@@ -157,6 +158,48 @@ impl TlsConfig {
158 }
159 }
160
161 +/// One LoRA adapter to load before the server accepts traffic.
162 +///
163 +/// Mirrors `LoRAModulePath` in vllm/entrypoints/openai/models/protocol.py,
164 +/// which is also the JSON shape the Python supervisor forwards in
165 +/// `--args-json`.
166 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
167 +pub struct LoraModulePath {
168 + /// Public model id the adapter is served under.
169 + pub name: String,
170 + /// Local path or Hugging Face repo id of the adapter.
171 + pub path: String,
172 + /// Base model reported as `parent` in `/v1/models`; defaults to the
173 + /// primary served model name.
174 + #[serde(default)]
175 + pub base_model_name: Option<String>,
176 + #[serde(default)]
177 + pub is_3d_lora_weight: bool,
178 +}
179 +
180 +impl FromStr for LoraModulePath {
181 + type Err = String;
182 +
183 + /// Accept the two CLI forms Python's `LoRAParserAction` accepts:
184 + /// `name=path`, or a JSON object with the struct's fields.
185 + fn from_str(value: &str) -> Result<Self, Self::Err> {
186 + if !value.trim_start().starts_with('{') {
187 + let (name, path) = value
188 + .split_once('=')
189 + .filter(|(name, path)| !name.is_empty() && !path.is_empty())
190 + .ok_or_else(|| format!("expected `name=path`, got `{value}`"))?;
191 + return Ok(Self {
192 + name: name.to_string(),
193 + path: path.to_string(),
194 + base_model_name: None,
195 + is_3d_lora_weight: false,
196 + });
197 + }
198 + serde_json::from_str(value)
199 + .map_err(|e| format!("expected `name=path` or a JSON object: {e}"))
200 + }
201 +}
202 +
203 /// Normalized runtime configuration for the minimal OpenAI-compatible server.
204 #[derive(Educe, Clone, PartialEq, Eq, Serialize)]
205 #[educe(Debug)]
@@ -192,6 +235,9 @@ pub struct Config {
235 /// Maximum number of input items allowed per prompt for each modality.
236 /// Unspecified modalities are unlimited.
237 pub limit_mm_per_prompt: MmLimitPerPrompt,
238 + /// LoRA adapters loaded at startup, in order. Startup fails if any of
239 + /// them cannot be loaded.
240 + pub lora_modules: Vec<LoraModulePath>,
241 /// How to serialize `message.content` for chat-template rendering.
242 pub chat_template_content_format: ChatTemplateContentFormatOption,
243 /// Optional maximum number of top log probabilities accepted by the
@@ -290,3 +336,66 @@ impl fmt::Debug for RedactedApiKeys<'_> {
336 fn fmt_redacted_api_keys(api_keys: &[String], f: &mut fmt::Formatter<'_>) -> fmt::Result {
337 fmt::Debug::fmt(&RedactedApiKeys(api_keys), f)
338 }
339 +
340 +#[cfg(test)]
341 +mod lora_module_path_tests {
342 + use super::LoraModulePath;
343 +
344 + #[test]
345 + fn parses_name_equals_path() {
346 + let module: LoraModulePath = "alice=charent/self_cognition_Alice".parse().unwrap();
347 + assert_eq!(
348 + module,
349 + LoraModulePath {
350 + name: "alice".to_string(),
351 + path: "charent/self_cognition_Alice".to_string(),
352 + base_model_name: None,
353 + is_3d_lora_weight: false,
354 + }
355 + );
356 + }
357 +
358 + #[test]
359 + fn parses_json_object() {
360 + let module: LoraModulePath =
361 + r#"{"name": "alice", "path": "/adapters/alice", "base_model_name": "base", "is_3d_lora_weight": true}"#
362 + .parse()
363 + .unwrap();
364 + assert_eq!(
365 + module,
366 + LoraModulePath {
367 + name: "alice".to_string(),
368 + path: "/adapters/alice".to_string(),
369 + base_model_name: Some("base".to_string()),
370 + is_3d_lora_weight: true,
371 + }
372 + );
373 + }
374 +
375 + #[test]
376 + fn parses_name_equals_path_with_comma() {
377 + let module: LoraModulePath = "alice=/adapters/a,b".parse().unwrap();
378 + assert_eq!(module.name, "alice");
379 + assert_eq!(module.path, "/adapters/a,b");
380 + }
381 +
382 + #[test]
383 + fn json_object_optional_fields_default() {
384 + let module: LoraModulePath = r#"{"name": "a", "path": "org/a"}"#.parse().unwrap();
385 + assert_eq!(module.base_model_name, None);
386 + assert!(!module.is_3d_lora_weight);
387 + }
388 +
389 + #[test]
390 + fn rejects_empty_name_or_path() {
391 + assert!("=org/a".parse::<LoraModulePath>().is_err());
392 + assert!("a=".parse::<LoraModulePath>().is_err());
393 + }
394 +
395 + #[test]
396 + fn rejects_bare_path_and_bad_json() {
397 + assert!("org/a".parse::<LoraModulePath>().is_err());
398 + assert!(r#"{"name": "a"}"#.parse::<LoraModulePath>().is_err());
399 + assert!(r#"[{"name": "a", "path": "org/a"}]"#.parse::<LoraModulePath>().is_err());
400 + }
401 +}
rust/src/server/src/grpc/control.rs
+9 -6
@@ -13,6 +13,7 @@ use vllm_engine_core_client::protocol::lora::LoraRequest;
13 use vllm_engine_core_client::protocol::utility::PauseMode as EnginePauseMode;
14
15 use super::{ControlServer, pb};
16 +use crate::config::LoraModulePath;
17 use crate::lora::{LoadLoraError, LoraDisabledError, LoraPathAccessError, UnloadLoraError};
18 use crate::state::AppState;
19
@@ -147,7 +148,7 @@ fn lora_to_proto(adapter: &LoraRequest) -> pb::LoraAdapter {
148 fn load_lora_status(error: LoadLoraError) -> Status {
149 let code = match &error {
150 LoadLoraError::Disabled(_) => Code::FailedPrecondition,
150 - LoadLoraError::InvalidRequest(_) => Code::InvalidArgument,
151 + LoadLoraError::InvalidAdapter { .. } => Code::InvalidArgument,
152 LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => Code::InvalidArgument,
153 LoadLoraError::PathAccess(LoraPathAccessError::InvalidConfiguration { .. }) => {
154 Code::Internal
@@ -247,11 +248,13 @@ impl pb::control_server::Control for ControlServiceImpl {
248 request: Request<pb::LoadLoraRequest>,
249 ) -> Result<Response<pb::LoadLoraResponse>, Status> {
250 let request = request.into_inner();
250 - let adapter = self
251 - .state
252 - .load_lora(request.lora_name, request.source_path, false, false)
253 - .await
254 - .map_err(load_lora_status)?;
251 + let module = LoraModulePath {
252 + name: request.lora_name,
253 + path: request.source_path,
254 + base_model_name: None,
255 + is_3d_lora_weight: false,
256 + };
257 + let adapter = self.state.load_lora(module, false).await.map_err(load_lora_status)?;
258 Ok(Response::new(pb::LoadLoraResponse {
259 adapter: Some(lora_to_proto(&adapter)),
260 }))
rust/src/server/src/lib.rs
+30 -5
@@ -29,7 +29,7 @@ use axum::body::Body;
29 use axum::http::Request;
30 pub use config::{
31 ApiServerOptions, Config, CoordinatorMode, CorsConfig, DEFAULT_KEEP_ALIVE_TIMEOUT,
32 - HttpListenerMode, TlsConfig,
32 + HttpListenerMode, LoraModulePath, TlsConfig,
33 };
34 use hyper::body::Incoming;
35 use hyper::server::conn::http1;
@@ -86,7 +86,7 @@ fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str {
86 }
87
88 /// Build the shared application state for one configured model and one engine
89 -/// client.
89 +/// client, including any static LoRA adapters from `--lora-modules`.
90 async fn build_state(config: &Config) -> Result<Arc<AppState>> {
91 // If no served names are specified, fall back to the backend model path so
92 // that the API always has at least one valid model ID. Use the same primary
@@ -139,7 +139,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
139 .with_tool_call_parser(config.tool_call_parser.clone())
140 .with_reasoning_parser(config.reasoning_parser.clone());
141
142 - Ok(Arc::new(
142 + let state = Arc::new(
143 AppState::new(served_model_names, chat)
144 .with_model_path(config.model.clone())
145 .with_api_server_options(config.api_server_options)
@@ -147,7 +147,31 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
147 .with_api_keys(config.api_keys.clone())
148 .with_cors(config.cors.clone())
149 .with_profiler(config.profiler.clone()),
150 - ))
150 + );
151 +
152 + // Load operator-configured static LoRA adapters before serving, failing
153 + // startup on any error like Python's `init_static_loras` does.
154 + load_static_loras(&state, &config.lora_modules).await?;
155 +
156 + Ok(state)
157 +}
158 +
159 +async fn load_static_loras(state: &AppState, modules: &[LoraModulePath]) -> Result<()> {
160 + for module in modules {
161 + let request = state.load_static_lora(module).await.with_context(|| {
162 + format!(
163 + "failed to load LoRA adapter `{}` from --lora-modules",
164 + module.name
165 + )
166 + })?;
167 + info!(
168 + lora_name = %request.lora_name,
169 + lora_int_id = request.lora_int_id,
170 + lora_path = %request.lora_path,
171 + "loaded static LoRA adapter"
172 + );
173 + }
174 + Ok(())
175 }
176
177 /// Run the OpenAI-compatible HTTP server until the supplied shutdown token is
@@ -182,7 +206,8 @@ where
206 .transpose()
207 .context("invalid TLS configuration")?;
208
185 - // Also check shutdown during the (potentially long) startup handshake.
209 + // Also check shutdown during the (potentially long) startup handshake and
210 + // static LoRA loading.
211 let state = tokio::select! {
212 result = build_state(&config) => result?,
213 _ = shutdown.cancelled() => return Ok(()),
rust/src/server/src/lora.rs
+51 -13
@@ -9,7 +9,9 @@ use thiserror::Error;
9 use thiserror_ext::Macro;
10 use tokio::sync::{Mutex, RwLock};
11 use vllm_engine_core_client::EngineCoreClient;
12 -use vllm_engine_core_client::protocol::lora::{LoraRequest, LoraRequestError};
12 +use vllm_engine_core_client::protocol::lora::LoraRequest;
13 +
14 +use crate::config::LoraModulePath;
15
16 const RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV: &str = "VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES";
17
@@ -103,12 +105,12 @@ pub(crate) struct LoraManager {
105 #[error("engine was not started with LoRA enabled")]
106 pub(crate) struct LoraDisabledError;
107
106 -#[derive(Debug, Error)]
108 +#[derive(Debug, Error, Macro)]
109 pub(crate) enum LoadLoraError {
110 #[error(transparent)]
111 Disabled(#[from] LoraDisabledError),
110 - #[error(transparent)]
111 - InvalidRequest(#[from] LoraRequestError),
112 + #[error("{message}")]
113 + InvalidAdapter { message: String },
114 #[error(transparent)]
115 PathAccess(#[from] LoraPathAccessError),
116 #[error("LoRA adapter `{lora_name}` is already loaded")]
@@ -186,14 +188,49 @@ impl LoraManager {
188 &self,
189 engine_core_client: &EngineCoreClient,
190 base_model_names: &[String],
189 - lora_name: String,
190 - lora_path: String,
191 + mut module: LoraModulePath,
192 load_inplace: bool,
192 - is_3d_lora_weight: bool,
193 ) -> Result<LoraRequest, LoadLoraError> {
194 let allowed_prefixes = runtime_lora_allowed_path_prefixes();
195 - let lora_path = validate_lora_path_access(&lora_path, allowed_prefixes.as_deref())?
196 - .unwrap_or(lora_path);
195 + if let Some(canonical_path) =
196 + validate_lora_path_access(&module.path, allowed_prefixes.as_deref())?
197 + {
198 + module.path = canonical_path;
199 + }
200 + self.register(engine_core_client, base_model_names, module, load_inplace).await
201 + }
202 +
203 + /// Load an operator-configured adapter (`--lora-modules`). Unlike the
204 + /// runtime endpoint, the path is trusted as given: no allowed-prefix
205 + /// check applies.
206 + pub async fn load_static_lora(
207 + &self,
208 + engine_core_client: &EngineCoreClient,
209 + base_model_names: &[String],
210 + module: &LoraModulePath,
211 + ) -> Result<LoraRequest, LoadLoraError> {
212 + self.register(engine_core_client, base_model_names, module.clone(), false).await
213 + }
214 +
215 + async fn register(
216 + &self,
217 + engine_core_client: &EngineCoreClient,
218 + base_model_names: &[String],
219 + module: LoraModulePath,
220 + load_inplace: bool,
221 + ) -> Result<LoraRequest, LoadLoraError> {
222 + let LoraModulePath {
223 + name: lora_name,
224 + path: lora_path,
225 + base_model_name,
226 + is_3d_lora_weight,
227 + } = module;
228 + if lora_name.trim().is_empty() {
229 + bail_invalid_adapter!("lora_name must not be empty");
230 + }
231 + if lora_path.trim().is_empty() {
232 + bail_invalid_adapter!("lora_path must not be empty");
233 + }
234 let _guard = self.update_lock.lock().await;
235 if base_model_names.iter().any(|name| name == &lora_name) {
236 return Err(LoadLoraError::BaseModelName { lora_name });
@@ -206,14 +243,15 @@ impl LoraManager {
243
244 let lora_int_id = existing_lora_int_id
245 .unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1);
209 - let lora_request = LoraRequest::new(
210 - lora_name.clone(),
246 + let lora_request = LoraRequest {
247 + lora_name: lora_name.clone(),
248 lora_int_id,
249 lora_path,
250 + base_model_name,
251 + tensorizer_config_dict: None,
252 load_inplace,
253 is_3d_lora_weight,
215 - )
216 - .map_err(LoadLoraError::InvalidRequest)?;
254 + };
255 drop(requests);
256
257 let loaded = engine_core_client.add_lora(&lora_request).await.map_err(|source| {
rust/src/server/src/routes/lora.rs
+9 -7
@@ -8,6 +8,7 @@ use serde::Deserialize;
8 use thiserror_ext::AsReport;
9 use validator::Validate;
10
11 +use crate::config::LoraModulePath;
12 use crate::error::ApiError;
13 use crate::lora::{LoadLoraError, LoraPathAccessError, UnloadLoraError};
14 use crate::routes::openai::utils::types::Normalizable;
@@ -39,7 +40,7 @@ fn load_lora_api_error(error: LoadLoraError) -> ApiError {
40 let message = error.to_report_string();
41 match error {
42 LoadLoraError::Disabled(_) => ApiError::invalid_request(message, None),
42 - LoadLoraError::InvalidRequest(_) => ApiError::invalid_request(message, None),
43 + LoadLoraError::InvalidAdapter { .. } => ApiError::invalid_request(message, None),
44 LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => {
45 ApiError::invalid_request(message, Some("lora_path"))
46 }
@@ -81,13 +82,14 @@ pub async fn load_lora_adapter(
82 ValidatedJson(request): ValidatedJson<LoadLoraAdapterRequest>,
83 ) -> Result<String, ApiError> {
84 let lora_name = request.lora_name;
85 + let module = LoraModulePath {
86 + name: lora_name.clone(),
87 + path: request.lora_path,
88 + base_model_name: None,
89 + is_3d_lora_weight: request.is_3d_lora_weight,
90 + };
91 state
85 - .load_lora(
86 - lora_name.clone(),
87 - request.lora_path,
88 - request.load_inplace,
89 - request.is_3d_lora_weight,
90 - )
92 + .load_lora(module, request.load_inplace)
93 .await
94 .map_err(load_lora_api_error)?;
95
rust/src/server/src/routes/tests.rs
+154 -9
@@ -57,7 +57,8 @@ use super::{
57 build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
58 render::build_router as build_render_router,
59 };
60 -use crate::config::{ApiServerOptions, CorsConfig};
60 +use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath};
61 +use crate::lora::LoadLoraError;
62 use crate::render::RenderState;
63 use crate::state::AppState;
64
@@ -941,6 +942,20 @@ async fn test_admin_app_with_ready_and_engine_script<F>(
942 ready: EngineCoreReadyResponse,
943 script: F,
944 ) -> (axum::Router, MockEngineTask)
945 +where
946 + F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
947 +{
948 + let (state, engine_task) = test_admin_state_with_ready_and_engine_script(ready, script).await;
949 + (
950 + build_router_with_dev_mode_and_lora(state, true, true),
951 + engine_task,
952 + )
953 +}
954 +
955 +async fn test_admin_state_with_ready_and_engine_script<F>(
956 + ready: EngineCoreReadyResponse,
957 + script: F,
958 +) -> (Arc<AppState>, MockEngineTask)
959 where
960 F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
961 {
@@ -968,18 +983,148 @@ where
983
984 let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
985 (
971 - build_router_with_dev_mode_and_lora(
972 - Arc::new(AppState::new(
973 - vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
974 - chat,
975 - )),
976 - true,
977 - true,
978 - ),
986 + Arc::new(AppState::new(
987 + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
988 + chat,
989 + )),
990 engine_task,
991 )
992 }
993
994 +/// Engine script that answers one `add_lora` utility call with `loaded` and
995 +/// hands the decoded LoRA request tuple to `check`.
996 +fn add_lora_script(
997 + loaded: bool,
998 + check: impl FnOnce(&[Value]) + Send + 'static,
999 +) -> impl for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static
1000 +{
1001 + move |dealer, push| {
1002 + boxed_test_future(async move {
1003 + let utility = recv_engine_message(dealer).await;
1004 + assert_eq!(utility[0].as_ref(), &[0x03]);
1005 + let payload = decode_value(&utility[1]).expect("decode utility payload");
1006 + let array = payload.as_array().expect("utility payload array");
1007 + let call_id = array[1].as_u64().expect("call id");
1008 + assert_eq!(array[2], Value::from("add_lora"));
1009 + let args = array[3].as_array().expect("utility args");
1010 + check(args[0].as_array().expect("lora request tuple"));
1011 + send_outputs(push, utility_outputs(call_id, utility_result_value(loaded))).await;
1012 + })
1013 + }
1014 +}
1015 +
1016 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1017 +#[serial]
1018 +async fn static_lora_module_loads_and_lists_with_parent() {
1019 + // An absolute local path: the runtime endpoint would reject it without
1020 + // VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES, static config trusts it.
1021 + let mut ready = default_ready_response();
1022 + ready.supports_lora = true;
1023 + ready.max_loras = 4;
1024 + let (state, _engine_task) = test_admin_state_with_ready_and_engine_script(
1025 + ready,
1026 + add_lora_script(true, |lora| {
1027 + assert_eq!(lora[0], Value::from("alice"));
1028 + assert_eq!(lora[1], Value::from(1));
1029 + assert_eq!(lora[2], Value::from("/adapters/alice"));
1030 + assert_eq!(lora[3], Value::from("base-model"));
1031 + }),
1032 + )
1033 + .await;
1034 +
1035 + let module = LoraModulePath {
1036 + name: "alice".to_string(),
1037 + path: "/adapters/alice".to_string(),
1038 + base_model_name: Some("base-model".to_string()),
1039 + is_3d_lora_weight: false,
1040 + };
1041 + let request = state.load_static_lora(&module).await.expect("load static lora");
1042 + assert_eq!(request.lora_name, "alice");
1043 + assert_eq!(request.lora_int_id, 1);
1044 +
1045 + let mut app = build_router_with_dev_mode_and_lora(state, true, false);
1046 + let models = app
1047 + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
1048 + .await
1049 + .expect("call app");
1050 + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
1051 + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1052 + assert_eq!(json["data"][1]["id"], "alice");
1053 + assert_eq!(json["data"][1]["root"], "/adapters/alice");
1054 + assert_eq!(json["data"][1]["parent"], "base-model");
1055 +}
1056 +
1057 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1058 +#[serial]
1059 +async fn static_lora_module_requires_lora_enabled_engine() {
1060 + let (state, _engine_task) = test_admin_state_with_ready_and_engine_script(
1061 + default_ready_response(),
1062 + |_dealer, _push| boxed_test_future(async {}),
1063 + )
1064 + .await;
1065 +
1066 + let module = LoraModulePath {
1067 + name: "alice".to_string(),
1068 + path: "org/alice".to_string(),
1069 + base_model_name: None,
1070 + is_3d_lora_weight: false,
1071 + };
1072 + let error = state.load_static_lora(&module).await.expect_err("engine has no lora");
1073 + assert!(matches!(error, LoadLoraError::Disabled(_)), "{error:?}");
1074 +}
1075 +
1076 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1077 +#[serial]
1078 +async fn static_lora_module_fails_when_engine_rejects_it() {
1079 + let mut ready = default_ready_response();
1080 + ready.supports_lora = true;
1081 + ready.max_loras = 4;
1082 + let (state, _engine_task) =
1083 + test_admin_state_with_ready_and_engine_script(ready, add_lora_script(false, |_| {})).await;
1084 +
1085 + let module = LoraModulePath {
1086 + name: "alice".to_string(),
1087 + path: "org/alice".to_string(),
1088 + base_model_name: None,
1089 + is_3d_lora_weight: false,
1090 + };
1091 + let error = state.load_static_lora(&module).await.expect_err("engine rejected");
1092 + assert!(
1093 + matches!(error, LoadLoraError::NotLoaded { .. }),
1094 + "{error:?}"
1095 + );
1096 + assert!(state.served_lora_requests().await.is_empty());
1097 +}
1098 +
1099 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1100 +#[serial]
1101 +async fn static_lora_module_rejects_empty_name_or_path() {
1102 + // The JSON form of `--lora-modules` is not validated at parse time; the
1103 + // manager rejects empty fields before any engine RPC.
1104 + let mut ready = default_ready_response();
1105 + ready.supports_lora = true;
1106 + ready.max_loras = 4;
1107 + let (state, _engine_task) =
1108 + test_admin_state_with_ready_and_engine_script(ready, |_dealer, _push| {
1109 + boxed_test_future(async {})
1110 + })
1111 + .await;
1112 +
1113 + for (name, path) in [("", "org/alice"), ("alice", "")] {
1114 + let module = LoraModulePath {
1115 + name: name.to_string(),
1116 + path: path.to_string(),
1117 + base_model_name: None,
1118 + is_3d_lora_weight: false,
1119 + };
1120 + let error = state.load_static_lora(&module).await.expect_err("empty field");
1121 + assert!(
1122 + matches!(error, LoadLoraError::InvalidAdapter { .. }),
1123 + "{error:?}"
1124 + );
1125 + }
1126 +}
1127 +
1128 async fn test_app_with_engine_handle() -> (axum::Router, MockEngineTask) {
1129 test_app_with_stream_output_specs(default_stream_output_specs()).await
1130 }
rust/src/server/src/state.rs
+17 -7
@@ -14,7 +14,7 @@ use vllm_engine_core_client::EngineCoreClient;
14 use vllm_engine_core_client::protocol::lora::LoraRequest;
15 use vllm_engine_core_client::runtime::BackgroundShutdownRuntime;
16
17 -use crate::config::{ApiServerOptions, CorsConfig};
17 +use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath};
18 use crate::lora::{
19 LoadLoraError, LoraDisabledError, LoraManager, LoraModelResolution, UnloadLoraError,
20 };
@@ -181,24 +181,34 @@ impl AppState {
181 /// Returns error if the engine was started without LoRA support.
182 pub async fn load_lora(
183 &self,
184 - lora_name: String,
185 - lora_path: String,
184 + module: LoraModulePath,
185 load_inplace: bool,
187 - is_3d_lora_weight: bool,
186 ) -> Result<LoraRequest, LoadLoraError> {
187 self.ensure_lora_enabled()?;
188 self.lora_manager
189 .load_lora(
190 self.engine_core_client(),
191 &self.served_model_names,
194 - lora_name,
195 - lora_path,
192 + module,
193 load_inplace,
197 - is_3d_lora_weight,
194 )
195 .await
196 }
197
198 + /// Load one adapter from `--lora-modules` and register it as a public
199 + /// model name.
200 + ///
201 + /// Returns error if the engine was started without LoRA support.
202 + pub async fn load_static_lora(
203 + &self,
204 + module: &LoraModulePath,
205 + ) -> Result<LoraRequest, LoadLoraError> {
206 + self.ensure_lora_enabled()?;
207 + self.lora_manager
208 + .load_static_lora(self.engine_core_client(), &self.served_model_names, module)
209 + .await
210 + }
211 +
212 /// Remove one dynamic LoRA adapter from the engine and public model
213 /// registry.
214 ///