@tduncan / Nvidia-Optimization-Stack-VLLM / commits / f32b17b6d6

[Rust Frontend] Support `--generation-config vllm` (#53044)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Aug 21, 2026 at 10:31 UTC f32b17b6d616605eb5ed498777f150242bd1bc30

14 files changed +244 -41

rust/src/chat/src/backend/hf.rs

+6 -2

@@ -129,8 +129,11 @@ pub(super) async fn load_model_backends(

129 options: LoadModelBackendsOptions,

130 ) -> Result<LoadedModelBackends> {

131 let files = ResolvedModelFiles::new(model_id).await?;

132 - let text_backend =

133 - HfTextBackend::from_resolved_model_files(files.clone(), model_id.to_string())?;

132 + let text_backend = HfTextBackend::from_resolved_model_files(

133 + files.clone(),

134 + model_id.to_string(),

135 + options.generation_config,

136 + )?;

137 let tokenizer = text_backend.tokenizer();

138 let text_backend: DynTextBackend = Arc::new(text_backend);

139

@@ -227,6 +230,7 @@ mod tests {

230 resolved_files(config_json, tokenizer_config_json),

231 "test-model".to_string(),

232 LoadModelBackendsOptions {

233 + generation_config: Default::default(),

234 renderer,

235 language_model_only: false,

236 chat_template_content_format: Default::default(),

rust/src/chat/src/backend/mod.rs

+3 -1

@@ -5,7 +5,7 @@ use std::collections::HashMap;

5 use std::sync::Arc;

7 use serde_json::Value;

8 -use vllm_text::{DynTextBackend, TextBackend};

8 +use vllm_text::{DynTextBackend, GenerationConfigMode, TextBackend};

10 use crate::error::Result;

11 use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};

@@ -61,6 +61,8 @@ pub type DynChatTextBackend = Arc<dyn ChatTextBackend>;

61 /// Frontend-side chat backend loading options.

62 #[derive(Debug, Clone, PartialEq, Eq, Default)]

63 pub struct LoadModelBackendsOptions {

64 + /// Which generation-config sampling defaults to inherit.

65 + pub generation_config: GenerationConfigMode,

66 /// Which chat renderer implementation to use.

67 pub renderer: RendererSelection,

68 /// Disable frontend-side multimodal preprocessing and render the model as

rust/src/chat/src/lib.rs

@@ -44,6 +44,7 @@ pub use request::{

44 pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};

45 pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;

46 pub use vllm_llm::FinishReason;

47 +pub use vllm_text::GenerationConfigMode;

49 mod backend;

50 mod error;

rust/src/cmd/src/cli.rs

+9 -1

@@ -22,8 +22,8 @@ use serde_json::Value;

22 use serde_with::{DefaultOnNull, OneOrMany, serde_as};

23 use thiserror_ext::AsReport as _;

24 use uuid::Uuid;

25 -use vllm_chat::ReasoningParserFactory;

25 use vllm_chat::multimodal::MmLimitPerPrompt;

26 +use vllm_chat::{GenerationConfigMode, ReasoningParserFactory};

27 use vllm_engine_core_client::TransportMode;

28 use vllm_managed_engine::ManagedEngineConfig;

29 use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};

@@ -186,6 +186,12 @@ pub struct SharedRuntimeArgs {

186 /// public model ID.

187 pub model: String,

188

189 + /// The source of generation-config sampling defaults. `"auto"` loads the

190 + /// model's defaults, while `"vllm"` uses vLLM's neutral defaults.

191 + #[arg(long, default_value_t)]

192 + #[serde(default)]

193 + pub generation_config: GenerationConfigMode,

194 +

195 /// Maximum time to wait for the expected engines to register on the

196 /// frontend transport.

197 #[arg(

@@ -482,6 +488,7 @@ impl SharedRuntimeArgs {

488 None => CoordinatorMode::None,

489 },

490 model: self.model,

491 + generation_config: self.generation_config,

492 served_model_name: self.served_model_name,

493 listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },

494 tool_call_parser: self.tool_call_parser,

@@ -535,6 +542,7 @@ impl SharedRuntimeArgs {

542 },

543 coordinator_mode: CoordinatorMode::MaybeInProc,

544 model: self.model,

545 + generation_config: self.generation_config,

546 served_model_name: self.served_model_name,

547 listener_mode,

548 tool_call_parser: self.tool_call_parser,

rust/src/cmd/src/cli/tests.rs

+30 -2

@@ -3,7 +3,9 @@

4 use expect_test::expect;

5 use vllm_engine_core_client::TransportMode;

6 -use vllm_server::{Config, HttpListenerMode, ParserSelection, RendererSelection};

6 +use vllm_server::{

7 + Config, GenerationConfigMode, HttpListenerMode, ParserSelection, RendererSelection,

8 +};

10 use super::{BenchCommand, Cli, Command};

@@ -88,6 +90,7 @@ fn serve_args_forward_python_flags_with_separator() {

90 uds: None,

91 runtime: SharedRuntimeArgs {

92 model: "Qwen/Qwen3-0.6B",

93 + generation_config: Auto,

94 engine_ready_timeout_secs: 600,

95 tool_call_parser: Auto,

96 reasoning_parser: Auto,

@@ -732,6 +735,24 @@ fn serve_args_reject_unsupported_flag_arg() {

735 "#]].assert_eq(&error.to_string());

736 }

737

738 +#[test]

739 +fn serve_args_reject_custom_generation_config_source() {

740 + let error = Cli::try_parse_from([

741 + "vllm-rs",

742 + "serve",

743 + "Qwen/Qwen3-0.6B",

744 + "--generation-config",

745 + "/tmp/custom-config",

746 + ])

747 + .unwrap_err();

748 +

749 + expect![[r#"

750 + error: invalid value '/tmp/custom-config' for '--generation-config <GENERATION_CONFIG>': generation config source `/tmp/custom-config` is not implemented yet (expected one of: auto, vllm)

751 +

752 + For more information, try '--help'.

753 + "#]].assert_eq(&error.to_string());

754 +}

755 +

756 #[test]

757 fn serve_args_reject_unsupported_no_flag_alias() {

758 let error = Cli::try_parse_from([

@@ -788,6 +809,7 @@ fn frontend_args_accept_json() {

809 data_parallel_size: None,

810 runtime: SharedRuntimeArgs {

811 model: "Qwen/Qwen3-0.6B",

812 + generation_config: Auto,

813 engine_ready_timeout_secs: 600,

814 tool_call_parser: None,

815 reasoning_parser: None,

@@ -858,6 +880,7 @@ fn frontend_args_json_applies_defaults() {

880 panic!("expected frontend args");

881 };

882 assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");

883 + assert_eq!(args.runtime.generation_config, GenerationConfigMode::Auto);

884 assert_eq!(args.runtime.engine_ready_timeout_secs, 600);

885 assert_eq!(args.runtime.tool_call_parser, ParserSelection::None);

886 assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);

@@ -901,7 +924,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {

924 "--output-address",

925 "ipc:///tmp/output.sock",

926 "--args-json",

904 - r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#,

927 + r#"{"model_tag":"Qwen/Qwen3-0.6B","generation_config":"vllm","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#,

928 ])

929 .unwrap();

930

@@ -909,6 +932,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {

932 panic!("expected frontend args");

933 };

934 assert_eq!(args.runtime.engine_ready_timeout_secs, 42);

935 + assert_eq!(args.runtime.generation_config, GenerationConfigMode::Vllm);

936 assert_eq!(

937 args.runtime.tool_call_parser,

938 ParserSelection::Explicit("hermes".to_string())

@@ -1379,6 +1403,7 @@ fn serve_args_accept_handshake_aliases() {

1403 uds: None,

1404 runtime: SharedRuntimeArgs {

1405 model: "Qwen/Qwen3-0.6B",

1406 + generation_config: Auto,

1407 engine_ready_timeout_secs: 600,

1408 tool_call_parser: Auto,

1409 reasoning_parser: Auto,

@@ -1523,6 +1548,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {

1548 },

1549 coordinator_mode: MaybeInProc,

1550 model: "Qwen/Qwen3-0.6B",

1551 + generation_config: Auto,

1552 served_model_name: [],

1553 listener_mode: BindTcp {

1554 host: "127.0.0.1",

@@ -1608,6 +1634,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {

1634 },

1635 coordinator_mode: MaybeInProc,

1636 model: "Qwen/Qwen3-0.6B",

1637 + generation_config: Auto,

1638 served_model_name: [],

1639 listener_mode: BindTcp {

1640 host: "127.0.0.1",

@@ -1715,6 +1742,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present

1742 address: "tcp://127.0.0.1:7000",

1743 },

1744 model: "Qwen/Qwen3-0.6B",

1745 + generation_config: Auto,

1746 served_model_name: [],

1747 listener_mode: InheritedFd {

1748 fd: 3,

rust/src/cmd/src/cli/unsupported.rs

+5 -8

@@ -240,15 +240,12 @@ pub struct EngineUnsupportedArgs {

240 #[arg(long)]

241 pub hf_overrides: Option<Unsupported>,

242

243 - /// The folder path to the generation config. Defaults to `"auto"`, the

244 - /// generation config will be loaded from model path. If set to `"vllm"`, no

245 - /// generation config is loaded, vLLM defaults will be used. If set to a

246 - /// folder path, the generation config will be loaded from the specified

247 - /// folder path. If `max_new_tokens` is specified in generation config,

248 - /// then it sets a server-wide limit on the number of output tokens for

249 - /// all requests.

243 + /// Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If

244 + /// used with `--generation-config auto`, the override parameters will be

245 + /// merged with the default config from the model. If used with

246 + /// `--generation-config vllm`, only the override parameters are used.

247 #[arg(long)]

251 - pub generation_config: Option<Unsupported>,

248 + pub override_generation_config: Option<Unsupported>,

249

250 /// IOProcessor plugin name to load at model startup

251 #[arg(long)]

rust/src/server/examples/external_engine_openai_qwen.rs

@@ -60,6 +60,7 @@ async fn main() -> Result<()> {

60 },

61 coordinator_mode: CoordinatorMode::MaybeInProc,

62 model: args.model,

63 + generation_config: Default::default(),

64 served_model_name: vec![],

65 listener_mode: HttpListenerMode::BindTcp {

66 host: "127.0.0.1".to_string(),

rust/src/server/src/config.rs

+5 -1

@@ -11,7 +11,9 @@ use educe::Educe;

11 use serde::Serialize;

12 use serde_json::Value;

13 use vllm_chat::multimodal::MmLimitPerPrompt;

14 -use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};

14 +use vllm_chat::{

15 + ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,

16 +};

17 use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};

19 /// Default keep-alive idle timeout (seconds); also the head-read bound

@@ -165,6 +167,8 @@ pub struct Config {

167 pub coordinator_mode: CoordinatorMode,

168 /// Backend model identifier used for engine-core loading.

169 pub model: String,

170 + /// Which generation-config sampling defaults to inherit.

171 + pub generation_config: GenerationConfigMode,

172 /// Model name(s) exposed to clients via the OpenAI API. When non-empty,

173 /// the first entry is used as the primary ID in responses and all entries

174 /// are accepted in requests. When empty, falls back to `model`.

rust/src/server/src/lib.rs

+4 -1

@@ -45,7 +45,9 @@ use tonic_health::server::health_reporter;

45 use tower::ServiceExt as _;

46 use tracing::{info, trace, warn};

47 use vllm_chat::{ChatLlm, LoadModelBackendsOptions, load_model_backends};

48 -pub use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};

48 +pub use vllm_chat::{

49 + ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,

50 +};

51 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};

52 use vllm_llm::Llm;

53 use vllm_text::TextLlm;

@@ -96,6 +98,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {

98 let loaded = load_model_backends(

99 &config.model,

100 LoadModelBackendsOptions {

101 + generation_config: config.generation_config,

102 renderer: config.renderer,

103 language_model_only: config.language_model_only,

104 chat_template: config.chat_template.clone(),

rust/src/server/src/render.rs

@@ -56,6 +56,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {

56 let loaded = load_model_backends(

57 &config.model,

58 LoadModelBackendsOptions {

59 + generation_config: Default::default(),

60 renderer: config.renderer,

61 language_model_only: true,

62 chat_template: config.chat_template.clone(),

rust/src/text/src/backend/hf/mod.rs

+122 -19

@@ -18,7 +18,7 @@ pub use self::config::{

18 load_tokenizer_config,

19 };

20 pub use self::model_files::{ResolvedModelFiles, TokenizerSource};

21 -use crate::backend::{SamplingHints, TextBackend};

21 +use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend};

22 use crate::error::Result;

24 fn load_tokenizer(tokenizer: &TokenizerSource) -> Result<DynTokenizer> {

@@ -41,6 +41,7 @@ pub struct HfTextBackend {

41 /// Generation-config for sampling defaults that may be inherited when the

42 /// user does not explicitly override them.

43 generation_config: GenerationConfig,

44 + generation_config_mode: GenerationConfigMode,

45 /// Model vocabulary size from the selected text config.

46 model_vocab_size: usize,

47 /// Model config (`config.json`).

@@ -48,14 +49,12 @@ pub struct HfTextBackend {

49 }

51 impl HfTextBackend {

51 - /// Load the text backend with the given model id.

52 - pub async fn from_model(model_id: &str) -> Result<Self> {

53 - let files = ResolvedModelFiles::new(model_id).await?;

54 - Self::from_resolved_model_files(files, model_id.to_string())

55 - }

56 -

52 /// Load the text backend from resolved Hugging Face model files.

58 - pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {

53 + pub fn from_resolved_model_files(

54 + files: ResolvedModelFiles,

55 + model_id: String,

56 + generation_config_mode: GenerationConfigMode,

57 + ) -> Result<Self> {

58 let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;

59 let tokenizer = load_tokenizer(&files.tokenizer)?;

60 let model_config = load_model_config(files.config_path.as_deref())?;

@@ -80,6 +79,7 @@ impl HfTextBackend {

79 primary_eos_token_id,

80 extra_eos_token_ids,

81 generation_config,

82 + generation_config_mode,

83 model_vocab_size,

84 model_config,

85 })

@@ -146,16 +146,38 @@ impl TextBackend for HfTextBackend {

146 }

147

148 fn sampling_hints(&self) -> Result<SamplingHints> {

149 - Ok(SamplingHints {

150 - primary_eos_token_id: self.primary_eos_token_id,

151 - extra_eos_token_ids: self.extra_eos_token_ids.clone(),

152 - default_temperature: self.generation_config.temperature,

153 - default_top_p: self.generation_config.top_p,

154 - default_top_k: self.generation_config.top_k,

155 - default_min_p: self.generation_config.min_p,

156 - default_repetition_penalty: self.generation_config.repetition_penalty,

157 - default_max_tokens: self.generation_config.max_new_tokens,

158 - })

149 + Ok(build_sampling_hints(

150 + self.generation_config_mode,

151 + &self.generation_config,

152 + self.primary_eos_token_id,

153 + self.extra_eos_token_ids.clone(),

154 + ))

155 + }

156 +}

157 +

158 +fn build_sampling_hints(

159 + mode: GenerationConfigMode,

160 + generation_config: &GenerationConfig,

161 + primary_eos_token_id: Option<u32>,

162 + extra_eos_token_ids: BTreeSet<u32>,

163 +) -> SamplingHints {

164 + let sampling_config = match mode {

165 + // Auto inherits sampling defaults from the model's generation config.

166 + GenerationConfigMode::Auto => Some(generation_config),

167 + // Vllm skips model-provided sampling defaults. `lower_sampling_params`

168 + // applies vLLM fallback values; separately resolved EOS tokens remain.

169 + GenerationConfigMode::Vllm => None,

170 + };

171 +

172 + SamplingHints {

173 + primary_eos_token_id,

174 + extra_eos_token_ids,

175 + default_temperature: sampling_config.and_then(|config| config.temperature),

176 + default_top_p: sampling_config.and_then(|config| config.top_p),

177 + default_top_k: sampling_config.and_then(|config| config.top_k),

178 + default_min_p: sampling_config.and_then(|config| config.min_p),

179 + default_repetition_penalty: sampling_config.and_then(|config| config.repetition_penalty),

180 + default_max_tokens: sampling_config.and_then(|config| config.max_new_tokens),

181 }

182 }

183

@@ -163,7 +185,10 @@ impl TextBackend for HfTextBackend {

185 mod tests {

186 use std::collections::BTreeSet;

187

166 - use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids};

188 + use super::{

189 + GenerationConfig, GenerationConfigMode, HfTokenizerConfig, ModelConfig,

190 + build_sampling_hints, resolve_eos_token_ids,

191 + };

192 use vllm_tokenizer::Tokenizer;

193

194 struct FakeTokenizer;

@@ -235,4 +260,82 @@ mod tests {

260 assert_eq!(primary, Some(2));

261 assert_eq!(extra, BTreeSet::from([200006, 200010]));

262 }

263 +

264 + #[test]

265 + fn vllm_generation_config_keeps_eos_and_uses_neutral_sampling_defaults() {

266 + let generation_config: GenerationConfig = serde_json::from_str(

267 + r#"{

268 + "eos_token_id": [2, 3],

269 + "temperature": 0.6,

270 + "top_p": 0.95,

271 + "top_k": 20,

272 + "min_p": 0.1,

273 + "repetition_penalty": 1.1,

274 + "max_new_tokens": 4096

275 + }"#,

276 + )

277 + .unwrap();

278 + let extra_eos_token_ids = BTreeSet::from([3]);

279 +

280 + let hints = (

281 + build_sampling_hints(

282 + GenerationConfigMode::Auto,

283 + &generation_config,

284 + Some(2),

285 + extra_eos_token_ids.clone(),

286 + ),

287 + build_sampling_hints(

288 + GenerationConfigMode::Vllm,

289 + &generation_config,

290 + Some(2),

291 + extra_eos_token_ids,

292 + ),

293 + );

294 +

295 + expect_test::expect![[r#"

296 + (

297 + SamplingHints {

298 + primary_eos_token_id: Some(

299 + 2,

300 + ),

301 + extra_eos_token_ids: {

302 + 3,

303 + },

304 + default_temperature: Some(

305 + 0.6,

306 + ),

307 + default_top_p: Some(

308 + 0.95,

309 + ),

310 + default_top_k: Some(

311 + 20,

312 + ),

313 + default_min_p: Some(

314 + 0.1,

315 + ),

316 + default_repetition_penalty: Some(

317 + 1.1,

318 + ),

319 + default_max_tokens: Some(

320 + 4096,

321 + ),

322 + },

323 + SamplingHints {

324 + primary_eos_token_id: Some(

325 + 2,

326 + ),

327 + extra_eos_token_ids: {

328 + 3,

329 + },

330 + default_temperature: None,

331 + default_top_p: None,

332 + default_top_k: None,

333 + default_min_p: None,

334 + default_repetition_penalty: None,

335 + default_max_tokens: None,

336 + },

337 + )

338 + "#]]

339 + .assert_debug_eq(&hints);

340 + }

341 }

rust/src/text/src/backend/mod.rs

+44

@@ -3,12 +3,56 @@

4 pub mod hf;

6 +use std::fmt;

7 +use std::str::FromStr;

8 use std::sync::Arc;

10 +use serde_with::{DeserializeFromStr, SerializeDisplay};

11 use vllm_tokenizer::DynTokenizer;

13 use crate::error::Result;

15 +/// Select which sampling defaults to inherit from generation config.

16 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]

17 +pub enum GenerationConfigMode {

18 + /// Inherit sampling defaults from the model's generation config.

19 + #[default]

20 + Auto,

21 + /// Use vLLM's neutral sampling defaults.

22 + Vllm,

23 +}

24 +

25 +impl GenerationConfigMode {

26 + pub const AUTO_LITERAL: &str = "auto";

27 + pub const VLLM_LITERAL: &str = "vllm";

28 +}

29 +

30 +impl FromStr for GenerationConfigMode {

31 + type Err = String;

32 +

33 + fn from_str(value: &str) -> std::result::Result<Self, Self::Err> {

34 + if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {

35 + Ok(Self::Auto)

36 + } else if value.eq_ignore_ascii_case(Self::VLLM_LITERAL) {

37 + Ok(Self::Vllm)

38 + } else {

39 + Err(format!(

40 + "generation config source `{value}` is not implemented yet \

41 + (expected one of: auto, vllm)"

42 + ))

43 + }

44 + }

45 +}

46 +

47 +impl fmt::Display for GenerationConfigMode {

48 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

49 + match self {

50 + Self::Auto => f.write_str(Self::AUTO_LITERAL),

51 + Self::Vllm => f.write_str(Self::VLLM_LITERAL),

52 + }

53 + }

54 +}

55 +

56 /// Tokenizer/model-derived defaults used to enrich text-generation requests

57 /// before they are lowered into engine-core.

58 #[derive(Debug, Clone, Default, PartialEq)]

rust/src/text/src/lib.rs

+3 -1

@@ -9,7 +9,9 @@

10 use std::mem::take;

12 -pub use backend::{DynTextBackend, SamplingHints, SamplingLimits, TextBackend};

12 +pub use backend::{

13 + DynTextBackend, GenerationConfigMode, SamplingHints, SamplingLimits, TextBackend,

14 +};

15 pub use error::{Error, LogprobsError, Result, SamplingParamsError, TokenIdsError};

16 use futures::Stream;

17 pub use lower::{

rust/src/text/src/lower.rs

+10 -5

@@ -322,8 +322,8 @@ mod tests {

322 use vllm_tokenizer::test_utils::TestTokenizer;

323

324 use super::*;

325 - use crate::backend::hf::HfTextBackend;

326 - use crate::backend::{SamplingHints, TextBackend as _};

325 + use crate::backend::hf::{HfTextBackend, ResolvedModelFiles};

326 + use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend as _};

327 use crate::error::{LogprobsError, SamplingParamsError, TokenIdsError};

328 use crate::request::{Prompt, TextRequest};

329

@@ -785,9 +785,14 @@ mod tests {

785 #[tokio::test]

786 #[file_serial(hf_qwen3)]

787 async fn lower_text_request_uses_real_qwen_generation_defaults() {

788 - let backend = HfTextBackend::from_model("Qwen/Qwen3-0.6B")

789 - .await

790 - .expect("load qwen tokenizer and generation config");

788 + let model_id = "Qwen/Qwen3-0.6B";

789 + let files = ResolvedModelFiles::new(model_id).await.expect("resolve qwen model files");

790 + let backend = HfTextBackend::from_resolved_model_files(

791 + files,

792 + model_id.to_string(),

793 + GenerationConfigMode::Auto,

794 + )

795 + .expect("load qwen tokenizer and generation config");

796 let hints = backend.sampling_hints().expect("collect sampling hints");

797

798 expect_test::expect![[r#"