@tduncan / Nvidia-Optimization-Stack-VLLM / commits / bfd713bf8b

[Rust Frontend] Support HF config overrides `--hf-overrides` (#56931)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Sep 16, 2026 at 10:52 UTC bfd713bf8b3869a7b65a8f3198095c3d73c88875
16 files changed +298 -19
rust/Cargo.lock
+1
@@ -5813,6 +5813,7 @@ dependencies = [
5813 "vllm-engine-core-client",
5814 "vllm-managed-engine",
5815 "vllm-server",
5816 + "vllm-text",
5817 "vllm-tracing",
5818 ]
5819
rust/src/chat/src/backend/hf.rs
+9 -13
@@ -129,7 +129,8 @@ pub(super) async fn load_model_backends(
129 model_id: &str,
130 options: LoadModelBackendsOptions,
131 ) -> Result<LoadedModelBackends> {
132 - let files = ResolvedModelFiles::new(model_id, options.revision.as_deref()).await?;
132 + let mut files = ResolvedModelFiles::new(model_id, options.revision.as_deref()).await?;
133 + files.apply_overrides(&options.hf_overrides)?;
134 let text_backend = HfTextBackend::from_resolved_model_files(
135 files.clone(),
136 model_id.to_string(),
@@ -166,13 +167,11 @@ fn resolve_multimodal_render_info(
167 #[cfg(test)]
168 mod tests {
169 use std::collections::HashMap;
169 - use std::path::PathBuf;
170 use std::sync::Arc;
171
172 use tempfile::tempdir;
173 use thiserror_ext::AsReport as _;
174 use vllm_text::Prompt;
175 - use vllm_text::backend::hf::TokenizerSource;
175 use vllm_text::tokenizer::DynTokenizer;
176 use vllm_tokenizer::test_utils::TestTokenizer;
177
@@ -206,16 +205,12 @@ mod tests {
205 write_json(&config_path, config_json);
206 write_json(&tokenizer_config_path, tokenizer_config_json);
207
209 - vllm_text::backend::hf::ResolvedModelFiles {
210 - tokenizer: TokenizerSource::HuggingFace(PathBuf::from("/tmp/unused-tokenizer.json")),
211 - tokenizer_config_path: Some(tokenizer_config_path),
212 - generation_config_path: None,
213 - preprocessor_config_path: None,
214 - video_preprocessor_config_path: None,
215 - processor_config_path: None,
216 - chat_template_path: None,
217 - config_path: Some(config_path),
218 - }
208 + write_json(&root.join("tokenizer.json"), "{}");
209 + futures::executor::block_on(vllm_text::backend::hf::ResolvedModelFiles::new(
210 + root.to_str().unwrap(),
211 + None,
212 + ))
213 + .unwrap()
214 }
215
216 fn test_tokenizer() -> DynTokenizer {
@@ -232,6 +227,7 @@ mod tests {
227 "test-model".to_string(),
228 LoadModelBackendsOptions {
229 revision: None,
230 + hf_overrides: Default::default(),
231 generation_config: Default::default(),
232 renderer,
233 language_model_only: false,
rust/src/chat/src/backend/mod.rs
+3
@@ -5,6 +5,7 @@ use std::collections::HashMap;
5 use std::sync::Arc;
6
7 use serde_json::Value;
8 +use vllm_text::backend::hf::HfOverrides;
9 use vllm_text::{DynTextBackend, GenerationConfigMode, TextBackend};
10
11 use crate::error::Result;
@@ -63,6 +64,8 @@ pub type DynChatTextBackend = Arc<dyn ChatTextBackend>;
64 pub struct LoadModelBackendsOptions {
65 /// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
66 pub revision: Option<String>,
67 + /// JSON Merge Patch applied to the model config before loading any backend.
68 + pub hf_overrides: HfOverrides,
69 /// Which generation-config sampling defaults to inherit.
70 pub generation_config: GenerationConfigMode,
71 /// Which chat renderer implementation to use.
rust/src/cmd/Cargo.toml
+1
@@ -33,6 +33,7 @@ vllm-chat.workspace = true
33 vllm-engine-core-client.workspace = true
34 vllm-managed-engine.workspace = true
35 vllm-server.workspace = true
36 +vllm-text.workspace = true
37 vllm-tracing.workspace = true
38
39 [dev-dependencies]
rust/src/cmd/src/cli.rs
+17
@@ -33,6 +33,7 @@ use vllm_server::{
33 DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
34 RendererSelection,
35 };
36 +use vllm_text::backend::hf::HfOverrides;
37
38 use crate::cli::ssl::SslArgs;
39 use crate::cli::unsupported::UnsupportedArgs;
@@ -113,6 +114,9 @@ pub struct RenderArgs {
114 /// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
115 #[arg(long)]
116 revision: Option<String>,
117 + /// JSON Merge Patch (RFC 7396) for config.json; null removes a field.
118 + #[arg(long, value_parser = parse_json::<HfOverrides>, default_value = "{}", value_name = "JSON")]
119 + hf_overrides: HfOverrides,
120 /// HTTP bind host.
121 #[arg(long, default_value = "127.0.0.1")]
122 host: String,
@@ -161,6 +165,7 @@ impl RenderArgs {
165 RenderConfig {
166 model: self.model,
167 revision: self.revision,
168 + hf_overrides: self.hf_overrides,
169 served_model_name: self.served_model_name,
170 host: self.host,
171 port: self.port,
@@ -204,6 +209,12 @@ pub struct SharedRuntimeArgs {
209 #[serde(default)]
210 pub revision: Option<String>,
211
212 + /// JSON Merge Patch (RFC 7396) for config.json; null removes a field.
213 + /// Objects merge recursively and arrays/scalars replace existing values.
214 + #[arg(long, value_parser = parse_json::<HfOverrides>, default_value = "{}", value_name = "JSON")]
215 + #[serde(default)]
216 + pub hf_overrides: HfOverrides,
217 +
218 /// The source of generation-config sampling defaults. `"auto"` loads the
219 /// model's defaults, while `"vllm"` uses vLLM's neutral defaults.
220 #[arg(long, default_value_t)]
@@ -500,6 +511,7 @@ impl SharedRuntimeArgs {
511 },
512 model: self.model,
513 revision: self.revision,
514 + hf_overrides: self.hf_overrides,
515 generation_config: self.generation_config,
516 served_model_name: self.served_model_name,
517 listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
@@ -556,6 +568,7 @@ impl SharedRuntimeArgs {
568 coordinator_mode: CoordinatorMode::MaybeInProc,
569 model: self.model,
570 revision: self.revision,
571 + hf_overrides: self.hf_overrides,
572 generation_config: self.generation_config,
573 served_model_name: self.served_model_name,
574 listener_mode,
@@ -761,6 +774,9 @@ impl ServeArgs {
774 let reasoning_parser =
775 effective_engine_reasoning_parser(&self.runtime.reasoning_parser, &self.runtime.model);
776 let profiler_config = self.runtime.profiler_config_json();
777 + let hf_overrides = (!self.runtime.hf_overrides.is_empty()).then(|| {
778 + serde_json::to_string(&self.runtime.hf_overrides).expect("JSON object serializes")
779 + });
780
781 self.managed_engine.clone().into_config(
782 self.runtime.model.clone(),
@@ -773,6 +789,7 @@ impl ServeArgs {
789 self.runtime.shutdown_timeout,
790 handshake_port,
791 self.runtime.limit_mm_per_prompt_json(),
792 + hf_overrides,
793 )
794 }
795 }
rust/src/cmd/src/cli/tests.rs
+85
@@ -177,6 +177,9 @@ fn serve_args_forward_python_flags_with_separator() {
177 runtime: SharedRuntimeArgs {
178 model: "Qwen/Qwen3-0.6B",
179 revision: None,
180 + hf_overrides: HfOverrides(
181 + {},
182 + ),
183 generation_config: Auto,
184 engine_ready_timeout_secs: 600,
185 tool_call_parser: Auto,
@@ -1013,6 +1016,9 @@ fn frontend_args_accept_json() {
1016 runtime: SharedRuntimeArgs {
1017 model: "Qwen/Qwen3-0.6B",
1018 revision: None,
1019 + hf_overrides: HfOverrides(
1020 + {},
1021 + ),
1022 generation_config: Auto,
1023 engine_ready_timeout_secs: 600,
1024 tool_call_parser: None,
@@ -1120,6 +1126,73 @@ fn frontend_args_json_ignores_engine_owned_max_model_len() {
1126 assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
1127 }
1128
1129 +#[test]
1130 +fn hf_overrides_preserve_merge_patch_in_cli_and_python_bootstrap() {
1131 + let patch = r#"{"text_config":{"rope_parameters":{"factor":4}},"sliding_window":null}"#;
1132 + let cli = Cli::try_parse_from([
1133 + "vllm-rs",
1134 + "serve",
1135 + "Qwen/Qwen3-0.6B",
1136 + "--hf-overrides",
1137 + patch,
1138 + ])
1139 + .unwrap();
1140 + let Command::Serve(args) = cli.command else {
1141 + panic!("expected serve args")
1142 + };
1143 + let expected: serde_json::Value = serde_json::from_str(patch).unwrap();
1144 + assert_eq!(
1145 + serde_json::to_value(&args.runtime.hf_overrides).unwrap(),
1146 + expected
1147 + );
1148 + let engine = args.to_managed_engine_config(1234);
1149 + let index = engine.python_args.iter().position(|arg| arg == "--hf-overrides").unwrap();
1150 + assert_eq!(
1151 + serde_json::from_str::<serde_json::Value>(&engine.python_args[index + 1]).unwrap(),
1152 + expected
1153 + );
1154 + assert!(!engine.python_args.iter().any(|arg| arg == "--hf-config-path"));
1155 + let config = args.to_frontend_config("tcp://127.0.0.1:1234".to_string());
1156 + assert_eq!(serde_json::to_value(config.hf_overrides).unwrap(), expected);
1157 +
1158 + let payload = serde_json::json!({"model_tag":"Qwen/Qwen3-0.6B", "hf_overrides":expected});
1159 + let cli = Cli::try_parse_from([
1160 + "vllm-rs",
1161 + "frontend",
1162 + "--listen-fd",
1163 + "3",
1164 + "--input-address",
1165 + "ipc:///tmp/input.sock",
1166 + "--output-address",
1167 + "ipc:///tmp/output.sock",
1168 + "--args-json",
1169 + &payload.to_string(),
1170 + ])
1171 + .unwrap();
1172 + let Command::Frontend(args) = cli.command else {
1173 + panic!("expected frontend args")
1174 + };
1175 + assert_eq!(
1176 + serde_json::to_value(args.into_config().hf_overrides).unwrap(),
1177 + expected
1178 + );
1179 +}
1180 +
1181 +#[test]
1182 +fn hf_overrides_reject_non_object_inputs() {
1183 + for patch in ["null", "[]", "1", r#""callable""#] {
1184 + let error = Cli::try_parse_from([
1185 + "vllm-rs",
1186 + "serve",
1187 + "Qwen/Qwen3-0.6B",
1188 + "--hf-overrides",
1189 + patch,
1190 + ])
1191 + .unwrap_err();
1192 + assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation);
1193 + }
1194 +}
1195 +
1196 #[test]
1197 fn frontend_args_json_accepts_supported_non_default_fields() {
1198 let cli = Cli::try_parse_from([
@@ -1613,6 +1686,9 @@ fn serve_args_accept_handshake_aliases() {
1686 runtime: SharedRuntimeArgs {
1687 model: "Qwen/Qwen3-0.6B",
1688 revision: None,
1689 + hf_overrides: HfOverrides(
1690 + {},
1691 + ),
1692 generation_config: Auto,
1693 engine_ready_timeout_secs: 600,
1694 tool_call_parser: Auto,
@@ -1763,6 +1839,9 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
1839 coordinator_mode: MaybeInProc,
1840 model: "Qwen/Qwen3-0.6B",
1841 revision: None,
1842 + hf_overrides: HfOverrides(
1843 + {},
1844 + ),
1845 generation_config: Auto,
1846 served_model_name: [],
1847 listener_mode: BindTcp {
@@ -1852,6 +1931,9 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
1931 coordinator_mode: MaybeInProc,
1932 model: "Qwen/Qwen3-0.6B",
1933 revision: None,
1934 + hf_overrides: HfOverrides(
1935 + {},
1936 + ),
1937 generation_config: Auto,
1938 served_model_name: [],
1939 listener_mode: BindTcp {
@@ -1963,6 +2045,9 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
2045 },
2046 model: "Qwen/Qwen3-0.6B",
2047 revision: None,
2048 + hf_overrides: HfOverrides(
2049 + {},
2050 + ),
2051 generation_config: Auto,
2052 served_model_name: [],
2053 listener_mode: InheritedFd {
rust/src/cmd/src/cli/unsupported.rs
-5
@@ -235,11 +235,6 @@ pub struct EngineUnsupportedArgs {
235 #[arg(long, default_missing_value = "true", num_args = 0..=1)]
236 pub hf_token: Option<Unsupported>,
237
238 - /// If a dictionary, contains arguments to be forwarded to the Hugging Face
239 - /// config. If a callable, it is called to update the HuggingFace config.
240 - #[arg(long)]
241 - pub hf_overrides: Option<Unsupported>,
242 -
238 /// Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If
239 /// used with `--generation-config auto`, the override parameters will be
240 /// merged with the default config from the model. If used with
rust/src/managed-engine/src/cli.rs
+5
@@ -93,6 +93,7 @@ impl ManagedEngineArgs {
93 shutdown_timeout: u64,
94 handshake_port: u16,
95 limit_mm_per_prompt: Option<String>,
96 + hf_overrides: Option<String>,
97 ) -> ManagedEngineConfig {
98 let mut python_args = self.python_args;
99 // Manually forward some args to the Python engine.
@@ -136,6 +137,10 @@ impl ManagedEngineArgs {
137 python_args.push("--limit-mm-per-prompt".to_string());
138 python_args.push(limit_mm_per_prompt);
139 }
140 + if let Some(hf_overrides) = hf_overrides {
141 + python_args.push("--hf-overrides".to_string());
142 + python_args.push(hf_overrides);
143 + }
144
145 ManagedEngineConfig {
146 python: self.python,
rust/src/server/examples/external_engine_openai_qwen.rs
+1
@@ -61,6 +61,7 @@ async fn main() -> Result<()> {
61 coordinator_mode: CoordinatorMode::MaybeInProc,
62 model: args.model,
63 revision: None,
64 + hf_overrides: Default::default(),
65 generation_config: Default::default(),
66 served_model_name: vec![],
67 listener_mode: HttpListenerMode::BindTcp {
rust/src/server/src/config.rs
+3
@@ -16,6 +16,7 @@ use vllm_chat::{
16 ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
17 };
18 use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
19 +use vllm_text::backend::hf::HfOverrides;
20
21 /// Default keep-alive idle timeout (seconds); also the head-read bound
22 /// when keep-alive is disabled (`0`).
@@ -214,6 +215,8 @@ pub struct Config {
215 pub model: String,
216 /// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
217 pub revision: Option<String>,
218 + /// JSON Merge Patch applied to the model config before backend initialization.
219 + pub hf_overrides: HfOverrides,
220 /// Which generation-config sampling defaults to inherit.
221 pub generation_config: GenerationConfigMode,
222 /// Model name(s) exposed to clients via the OpenAI API. When non-empty,
rust/src/server/src/lib.rs
+1
@@ -99,6 +99,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
99 &config.model,
100 LoadModelBackendsOptions {
101 revision: config.revision.clone(),
102 + hf_overrides: config.hf_overrides.clone(),
103 generation_config: config.generation_config,
104 renderer: config.renderer,
105 language_model_only: config.language_model_only,
rust/src/server/src/render.rs
+4
@@ -13,6 +13,7 @@ use vllm_chat::{
13 ParserSelection, RendererSelection, load_model_backends,
14 };
15 use vllm_text::TextRequestProcessor;
16 +use vllm_text::backend::hf::HfOverrides;
17
18 use crate::{
19 HttpListenerMode, TlsConfig,
@@ -25,6 +26,7 @@ use crate::{
26 pub struct RenderConfig {
27 pub model: String,
28 pub revision: Option<String>,
29 + pub hf_overrides: HfOverrides,
30 pub served_model_name: Vec<String>,
31 pub host: String,
32 pub port: u16,
@@ -69,6 +71,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
71 &config.model,
72 LoadModelBackendsOptions {
73 revision: config.revision.clone(),
74 + hf_overrides: config.hf_overrides.clone(),
75 generation_config: Default::default(),
76 renderer: config.renderer,
77 language_model_only: true,
@@ -153,6 +156,7 @@ mod tests {
156 RenderConfig {
157 model: "test-model".to_string(),
158 revision: None,
159 + hf_overrides: Default::default(),
160 served_model_name: Vec::new(),
161 host: "127.0.0.1".to_string(),
162 port: 8000,
rust/src/text/Cargo.toml
+1 -1
@@ -16,6 +16,7 @@ reqwest.workspace = true
16 serde.workspace = true
17 serde_json.workspace = true
18 serde_with.workspace = true
19 +tempfile.workspace = true
20 thiserror.workspace = true
21 thiserror-ext.workspace = true
22 tracing.workspace = true
@@ -28,7 +29,6 @@ vllm-tokenizer.workspace = true
29 expect-test.workspace = true
30 futures.workspace = true
31 serial_test.workspace = true
31 -tempfile.workspace = true
32 tokio.workspace = true
33 vllm-llm = { workspace = true, features = ["test-util"] }
34 vllm-tokenizer = { workspace = true, features = ["test-utils"] }
rust/src/text/src/backend/hf/mod.rs
+2
@@ -3,6 +3,7 @@
3
4 mod config;
5 mod model_files;
6 +mod overrides;
7
8 use std::collections::BTreeSet;
9 use std::sync::Arc;
@@ -18,6 +19,7 @@ pub use self::config::{
19 load_tokenizer_config,
20 };
21 pub use self::model_files::{ResolvedModelFiles, TokenizerSource};
22 +pub use self::overrides::HfOverrides;
23 use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend};
24 use crate::error::Result;
25
rust/src/text/src/backend/hf/model_files.rs
+72
@@ -2,11 +2,13 @@
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 use std::path::{Path, PathBuf};
5 +use std::sync::Arc;
6
7 use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo};
8 use hf_hub::{Cache, Repo, RepoType};
9 use thiserror_ext::AsReport as _;
10
11 +use super::HfOverrides;
12 use super::config::{HfTokenizerConfig, load_tokenizer_config};
13 use crate::error::{Error, Result};
14
@@ -50,9 +52,27 @@ pub struct ResolvedModelFiles {
52 pub processor_config_path: Option<PathBuf>,
53 pub chat_template_path: Option<PathBuf>,
54 pub config_path: Option<PathBuf>,
55 + /// Keep a patched config alive across clones and backend ownership.
56 + config_temp_path: Option<Arc<tempfile::TempPath>>,
57 }
58
59 impl ResolvedModelFiles {
60 + /// Materialize a merge-patched config owned by these resolved model files.
61 + pub fn apply_overrides(&mut self, overrides: &HfOverrides) -> Result<()> {
62 + if overrides.is_empty() {
63 + return Ok(());
64 + }
65 + let path = materialize_config(self.config_path.as_deref(), overrides).map_err(|error| {
66 + Error::Tokenizer(format!(
67 + "failed to materialize HF overrides: {}",
68 + error.as_report()
69 + ))
70 + })?;
71 + self.config_path = Some(path.to_path_buf());
72 + self.config_temp_path = Some(Arc::new(path));
73 + Ok(())
74 + }
75 +
76 /// Resolve tokenizer/config files from a local model directory first when
77 /// `model_id` points to one, otherwise consult the local HF cache and
78 /// finally the Hub.
@@ -84,6 +104,24 @@ impl ResolvedModelFiles {
104 }
105 }
106
107 +fn materialize_config(
108 + path: Option<&Path>,
109 + overrides: &HfOverrides,
110 +) -> anyhow::Result<tempfile::TempPath> {
111 + let mut config: serde_json::Value = match path {
112 + Some(path) => serde_json::from_slice(&std::fs::read(path)?)?,
113 + None => serde_json::json!({}),
114 + };
115 + anyhow::ensure!(config.is_object(), "HF model config must be a JSON object");
116 + overrides.apply(&mut config);
117 + let file = tempfile::Builder::new()
118 + .prefix("vllm-rs-hf-config-")
119 + .suffix(".json")
120 + .tempfile()?;
121 + serde_json::to_writer(file.as_file(), &config)?;
122 + Ok(file.into_temp_path())
123 +}
124 +
125 fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
126 let tokenizer_config_path = local_file_if_exists(model_dir, "tokenizer_config.json");
127 let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
@@ -101,6 +139,7 @@ fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
139 processor_config_path: local_file_if_exists(model_dir, "processor_config.json"),
140 chat_template_path: discover_chat_template_in_dir(model_dir),
141 config_path: local_file_if_exists(model_dir, "config.json"),
142 + config_temp_path: None,
143 })
144 }
145
@@ -164,6 +203,7 @@ async fn resolve_remote_model_files(
203 processor_config_path,
204 chat_template_path,
205 config_path,
206 + config_temp_path: None,
207 })
208 }
209
@@ -204,6 +244,7 @@ fn resolve_cached_model_files(cache: &Cache, repo: &Repo) -> Result<Option<Resol
244 processor_config_path,
245 chat_template_path,
246 config_path,
247 + config_temp_path: None,
248 }))
249 }
250
@@ -434,6 +475,37 @@ mod tests {
475
476 use super::{ResolvedModelFiles, TokenizerSource};
477
478 + #[tokio::test]
479 + async fn overrides_keep_configs_alive_across_clones_and_isolate_instances() {
480 + let source = tempdir().unwrap();
481 + let original = r#"{"vocab_size":10}"#;
482 + fs::write(source.path().join("config.json"), original).unwrap();
483 + fs::write(source.path().join("tokenizer.json"), "{}").unwrap();
484 + let mut first =
485 + ResolvedModelFiles::new(source.path().to_str().unwrap(), None).await.unwrap();
486 + let patch = serde_json::from_value(serde_json::json!({"vocab_size":20})).unwrap();
487 + first.apply_overrides(&patch).unwrap();
488 + let first_path = first.config_path.clone().unwrap();
489 +
490 + let mut second = first.clone();
491 + second.apply_overrides(&patch).unwrap();
492 + let second_path = second.config_path.clone().unwrap();
493 + assert_ne!(first_path, second_path);
494 +
495 + let retained = first.clone();
496 + drop(first);
497 + assert!(first_path.exists());
498 + drop(second);
499 + assert!(!second_path.exists());
500 + assert!(first_path.exists());
501 + drop(retained);
502 + assert!(!first_path.exists());
503 + assert_eq!(
504 + fs::read_to_string(source.path().join("config.json")).unwrap(),
505 + original
506 + );
507 + }
508 +
509 #[tokio::test]
510 async fn resolved_model_files_prefers_absolute_local_model_dir() {
511 let dir = tempdir().expect("create temp dir");
rust/src/text/src/backend/hf/overrides.rs new
+93
@@ -0,0 +1,93 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use serde::{Deserialize, Serialize};
5 +use serde_json::{Map, Value};
6 +
7 +/// An object-shaped JSON Merge Patch (RFC 7396) for the model's HF config.
8 +/// Objects merge recursively, arrays/scalars replace, and null removes a key.
9 +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)]
10 +#[serde(transparent)]
11 +pub struct HfOverrides(pub Map<String, Value>);
12 +
13 +impl HfOverrides {
14 + pub fn is_empty(&self) -> bool {
15 + self.0.is_empty()
16 + }
17 +
18 + /// Apply the patch to an in-memory config, preserving the source file.
19 + pub fn apply(&self, config: &mut Value) {
20 + merge_object(config, &self.0);
21 + }
22 +}
23 +
24 +/// Apply RFC 7396 object-member updates at every nesting level.
25 +fn merge_object(target: &mut Value, patch: &Map<String, Value>) {
26 + if !target.is_object() {
27 + *target = Value::Object(Map::new());
28 + }
29 + let target = target.as_object_mut().unwrap();
30 + for (key, value) in patch {
31 + match value {
32 + Value::Null => {
33 + target.remove(key);
34 + }
35 + Value::Object(object) => {
36 + merge_object(target.entry(key.clone()).or_insert(Value::Null), object)
37 + }
38 + _ => {
39 + target.insert(key.clone(), value.clone());
40 + }
41 + }
42 + }
43 +}
44 +
45 +#[cfg(test)]
46 +mod tests {
47 + use serde_json::json;
48 +
49 + use super::HfOverrides;
50 +
51 + #[test]
52 + fn merge_patch_preserves_siblings_replaces_arrays_and_deletes_null_members() {
53 + let mut config = json!({
54 + "rope_parameters": {"rope_type": "yarn", "factor": 2},
55 + "text_config": {"rope_parameters": {"rope_type": "yarn", "factor": 2}},
56 + "architectures": ["Old", "Other"],
57 + "sliding_window": 4096
58 + });
59 + let patch: HfOverrides = serde_json::from_value(json!({
60 + "rope_parameters": {"factor": 4},
61 + "text_config": {"rope_parameters": {"factor": 4}},
62 + "architectures": ["New"],
63 + "sliding_window": null,
64 + "missing": null
65 + }))
66 + .unwrap();
67 + patch.apply(&mut config);
68 + assert_eq!(
69 + config,
70 + json!({
71 + "rope_parameters": {"rope_type": "yarn", "factor": 4},
72 + "text_config": {"rope_parameters": {"rope_type": "yarn", "factor": 4}},
73 + "architectures": ["New"]
74 + })
75 + );
76 + }
77 +
78 + #[test]
79 + fn merge_patch_creates_objects_and_preserves_nulls_inside_replacement_arrays() {
80 + let mut config = json!({"a": 1, "b": [1, 2]});
81 + let patch: HfOverrides = serde_json::from_value(json!({
82 + "a": {"nested": {"removed": null}},
83 + "b": {},
84 + "c": [null, {"literal": null}]
85 + }))
86 + .unwrap();
87 + patch.apply(&mut config);
88 + assert_eq!(
89 + config,
90 + json!({"a": {"nested": {}}, "b": {}, "c": [null, {"literal": null}]})
91 + );
92 + }
93 +}