@tduncan / Nvidia-Optimization-Stack-VLLM / commits / b47b01cf33

[Model][Frontend] Support DeepSeek-V4.1-Flash in Rust and Python frontends (#56208)

Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>

Bugen Zhao committed Sep 10, 2026 at 18:15 UTC b47b01cf3383d50a5fae8569cadcbd0736076234
52 files changed +2986 -113
rust/Cargo.lock
+1 -1
@@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
2290 [[package]]
2291 name = "llm-multimodal"
2292 version = "1.7.1"
2293 -source = "git+https://github.com/smg-project/llm-multimodal?rev=74f64753e040ade87c9d7f4bb0ad2f4dd7764652#74f64753e040ade87c9d7f4bb0ad2f4dd7764652"
2293 +source = "git+https://github.com/smg-project/llm-multimodal?rev=24c676dd49d73f05cb4068aade2e60397a5c0afe#24c676dd49d73f05cb4068aade2e60397a5c0afe"
2294 dependencies = [
2295 "anyhow",
2296 "base64 0.22.1",
rust/Cargo.toml
+1 -1
@@ -60,7 +60,7 @@ indexmap = "2.13.0"
60 indicatif = "0.18.4"
61 itertools = "0.14.0"
62 libc = "0.2.177"
63 -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "74f64753e040ade87c9d7f4bb0ad2f4dd7764652", default-features = false, features = ["native-tls"] }
63 +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "24c676dd49d73f05cb4068aade2e60397a5c0afe", default-features = false, features = ["native-tls"] }
64 mimalloc = "0.1.52"
65 minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
66 minijinja-contrib = { version = "2.24", features = ["pycompat"] }
rust/src/chat/src/backend/hf.rs
+3 -2
@@ -19,8 +19,8 @@ use crate::output::{
19 };
20 use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
21 use crate::renderer::{
22 - DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
23 - InklingChatRenderer, KimiK3ChatRenderer,
22 + DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, DynChatRenderer,
23 + HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer,
24 };
25 use crate::request::ChatRequest;
26 use crate::{DynChatOutputProcessor, RendererSelection};
@@ -72,6 +72,7 @@ impl HfChatBackend {
72 )?),
73 RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
74 RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
75 + RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new()),
76 RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
77 RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
78 RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
rust/src/chat/src/error.rs
+3
@@ -17,6 +17,8 @@ pub enum Error {
17 MissingChatTemplate,
18 #[error("chat template error: {0}")]
19 ChatTemplate(String),
20 + #[error("{0}")]
21 + InvalidReasoningEffort(String),
22 #[error("multimodal input is not supported by this chat renderer")]
23 UnsupportedMultimodalRenderer,
24 #[error("unsupported multimodal content: {0}")]
@@ -96,6 +98,7 @@ impl Error {
98 pub fn is_request_validation_error(&self) -> bool {
99 match self {
100 Self::PromptTooLong { .. }
101 + | Self::InvalidReasoningEffort(_)
102 | Self::DuplicateToolName { .. }
103 | Self::ToolChoiceRequiresTools
104 | Self::ToolChoiceFunctionNotFound { .. } => true,
rust/src/chat/src/lib.rs
+4 -4
@@ -33,8 +33,8 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
33 pub use parser::{ParserSelection, validate_parser_overrides};
34 pub use renderer::hf::ChatTemplateContentFormatOption;
35 pub use renderer::{
36 - ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
37 - HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
36 + ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer,
37 + DynChatRenderer, HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
38 RendererSelection,
39 };
40 pub use request::{
@@ -363,7 +363,7 @@ mod tests {
363 )
364 .unwrap_err();
365
366 - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
366 + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
367 }
368
369 #[test]
@@ -374,6 +374,6 @@ mod tests {
374 )
375 .unwrap_err();
376
377 - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
377 + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
378 }
379 }
rust/src/chat/src/multimodal.rs
+39 -1
@@ -153,6 +153,10 @@ struct ResolvedMultimodalSpec {
153 modality: Modality,
154 field_layouts: EncoderFieldLayouts,
155 keep_on_cpu_keys: HashSet<String>,
156 + /// Spec-declared wire key for the primary encoder input tensor, when it
157 + /// differs from the per-modality default (e.g. `"patches"` for
158 + /// DeepSeek-V4.1 images).
159 + encoder_input_key: Option<String>,
160 }
161
162 impl ResolvedMultimodalSpec {
@@ -162,10 +166,14 @@ impl ResolvedMultimodalSpec {
166 modality,
167 field_layouts: raw.encoder_field_layouts_for(modality),
168 keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(),
169 + encoder_input_key: raw.encoder_input_key_for(modality),
170 }
171 }
172
168 - fn primary_key(&self) -> &'static str {
173 + fn primary_key(&self) -> &str {
174 + if let Some(key) = &self.encoder_input_key {
175 + return key;
176 + }
177 match self.modality {
178 Modality::Image => image::IMAGE_PRIMARY_KEY,
179 Modality::Video => video::VIDEO_PRIMARY_KEY,
@@ -841,6 +849,9 @@ mod tests {
849 pub(super) const QWEN3_IMAGE_PAD_ID: u32 = 151655;
850 pub(super) const QWEN3_VIDEO_PAD_ID: u32 = 151656;
851
852 + pub(super) const DEEPSEEK_V41_IMAGE_ID: u32 = 129264;
853 + pub(super) const DEEPSEEK_V41_IMAGE_PAD_ID: u32 = 129265;
854 +
855 fn llama4_tokenizer() -> TestTokenizer {
856 TestTokenizer::new()
857 .with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)
@@ -954,6 +965,33 @@ mod tests {
965 assert_eq!(info.placeholder_token(Modality::Video), None);
966 }
967
968 + fn deepseek_v41_info() -> MultimodalModelInfo {
969 + let config = serde_json::json!({
970 + "model_type": "deepseek_v41",
971 + "image_token_id": DEEPSEEK_V41_IMAGE_ID,
972 + });
973 + let tokenizer = TestTokenizer::new()
974 + .with_regular_token("<｜deepseek_image｜>", DEEPSEEK_V41_IMAGE_ID)
975 + .with_regular_token("<|place_holder_mm_span_0436|>", DEEPSEEK_V41_IMAGE_PAD_ID);
976 + test_info("deepseek_v41", config, tokenizer)
977 + }
978 +
979 + #[test]
980 + fn deepseek_v41_resolves_image_support_only() {
981 + let info = deepseek_v41_info();
982 +
983 + assert_eq!(
984 + info.placeholder_token(Modality::Image),
985 + Some("<｜deepseek_image｜>")
986 + );
987 + assert_eq!(info.placeholder_token(Modality::Video), None);
988 + let image = info.image.as_ref().expect("image support");
989 + assert_eq!(image.placeholder.marker_token_id, DEEPSEEK_V41_IMAGE_ID);
990 + assert_eq!(image.placeholder.embed_token_id, DEEPSEEK_V41_IMAGE_ID);
991 + // The engine's forward kwargs pop `patches` (not `pixel_values`).
992 + assert_eq!(image.spec.primary_key(), "patches");
993 + }
994 +
995 #[test]
996 fn input_audio_uses_connector_data_urls() {
997 assert_eq!(
rust/src/chat/src/multimodal/expand.rs
+136 -8
@@ -45,6 +45,14 @@ impl<'a> ExpansionLane<'a> {
45 /// matching the original media-part order within that modality; markers of
46 /// different modalities may interleave freely.
47 ///
48 +/// Replacements may carry an [`AlignmentPad`](llm_multimodal::AlignmentPad)
49 +/// (e.g. DeepSeek-V4.1's compressor-alignment pad): `period - 1 - offset %
50 +/// period` pad tokens are
51 +/// prepended at splice time, where `offset` is the position the replacement
52 +/// lands at in the expanded prompt. The pads are part of the reported range
53 +/// and count as embed positions only if the pad token id equals the
54 +/// modality's embed token id.
55 +///
56 /// The returned ranges point into the already-expanded prompt, grouped per
57 /// modality in item order.
58 pub(super) fn expand_prompt_token_ids(
@@ -56,12 +64,14 @@ pub(super) fn expand_prompt_token_ids(
64 return Ok(HashMap::new());
65 }
66
59 - let replacement_growth = lanes
60 - .iter()
61 - .flat_map(|lane| lane.replacements.iter())
62 - .fold(0usize, |total, replacement| {
63 - total.saturating_add(replacement.tokens.len().saturating_sub(1))
64 - });
67 + let replacement_growth = lanes.iter().flat_map(|lane| lane.replacements.iter()).fold(
68 + 0usize,
69 + |total, replacement| {
70 + // Alignment pads are position-dependent; reserve the worst case.
71 + let max_pads = replacement.alignment_pad.map_or(0, |pad| pad.period - 1);
72 + total.saturating_add(max_pads + replacement.tokens.len().saturating_sub(1))
73 + },
74 + );
75 let expanded_len = prompt_token_ids.len().saturating_add(replacement_growth);
76
77 let mut expanded = Vec::with_capacity(expanded_len);
@@ -95,10 +105,20 @@ pub(super) fn expand_prompt_token_ids(
105 expanded.len()
106 );
107 }
98 - let range_len = replacement_len.saturating_add(structural_prefix);
108 + // Position-dependent alignment padding, computed from the offset the
109 + // replacement block lands at in the expanded prompt (mirroring
110 + // vLLM's `_apply_token_matches_with_placeholders` splice-time pad).
111 + let pad_count = replacement.alignment_pad.map_or(0, |pad| pad.count_at(expanded.len()));
112 + let range_len = structural_prefix + pad_count + replacement_len;
113 let is_embed = {
114 let mut mask = Vec::with_capacity(range_len);
115 mask.extend(std::iter::repeat_n(false, structural_prefix));
116 + if let Some(pad) = &replacement.alignment_pad {
117 + mask.extend(std::iter::repeat_n(
118 + pad.token_id as u32 == lane.embed_token_id,
119 + pad_count,
120 + ));
121 + }
122 mask.extend(
123 replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
124 );
@@ -106,6 +126,9 @@ pub(super) fn expand_prompt_token_ids(
126 };
127
128 let expanded_offset = expanded.len() - structural_prefix;
129 + if let Some(pad) = &replacement.alignment_pad {
130 + expanded.extend(std::iter::repeat_n(pad.token_id as u32, pad_count));
131 + }
132 expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
133 ranges.entry(lane.modality).or_default().push(PlaceholderRange {
134 offset: expanded_offset,
@@ -454,7 +477,6 @@ mod tests {
477 }
478
479 const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
457 -
480 fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
481 prepared_media(
482 Modality::Audio,
@@ -542,4 +564,110 @@ mod tests {
564 ));
565 assert_eq!(prompt_token_ids, original_prompt_token_ids);
566 }
567 +
568 + // DeepSeek-V4.1 image spans: every span position carries the image token
569 + // id; a compressor-alignment pad (id 129265, period 2) is prepended at
570 + // splice time so spans always start at the same compressor phase.
571 + const DS_IMAGE_ID: u32 = 129_264;
572 + const DS_IMAGE_PAD_ID: u32 = 129_265;
573 +
574 + fn deepseek_prepared(span_lens: Vec<usize>) -> PreparedMedia {
575 + prepared_media(
576 + Modality::Image,
577 + "<｜deepseek_image｜>",
578 + DS_IMAGE_ID,
579 + DS_IMAGE_ID,
580 + span_lens
581 + .into_iter()
582 + .map(|len| {
583 + PromptReplacement::repeated(
584 + Modality::Image,
585 + "<｜deepseek_image｜>",
586 + DS_IMAGE_ID as TokenId,
587 + len,
588 + )
589 + .with_alignment_pad(DS_IMAGE_PAD_ID as TokenId, 2)
590 + })
591 + .collect(),
592 + )
593 + }
594 +
595 + #[test]
596 + fn expand_prompt_tokens_skips_alignment_pad_at_odd_offset() {
597 + // Odd block start: already at the target compressor phase, no pad.
598 + let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2];
599 + let prepared = vec![deepseek_prepared(vec![3])];
600 +
601 + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
602 + let ranges = &ranges[&Modality::Image];
603 +
604 + assert_eq!(
605 + prompt_token_ids,
606 + vec![1, DS_IMAGE_ID, DS_IMAGE_ID, DS_IMAGE_ID, 2]
607 + );
608 + assert_eq!(ranges[0].offset, 1);
609 + assert_eq!(ranges[0].length, 3);
610 + assert_bool_mask(&ranges[0], &[true, true, true]);
611 + }
612 +
613 + #[test]
614 + fn expand_prompt_tokens_prepends_alignment_pad_at_even_offset() {
615 + // Even block start: one pad token aligns the span to phase 1.
616 + let mut prompt_token_ids = vec![1, 3, DS_IMAGE_ID, 2];
617 + let prepared = vec![deepseek_prepared(vec![3])];
618 +
619 + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
620 + let ranges = &ranges[&Modality::Image];
621 +
622 + assert_eq!(
623 + prompt_token_ids,
624 + vec![
625 + 1,
626 + 3,
627 + DS_IMAGE_PAD_ID,
628 + DS_IMAGE_ID,
629 + DS_IMAGE_ID,
630 + DS_IMAGE_ID,
631 + 2
632 + ]
633 + );
634 + // The range starts at the pad; pads are not embed positions.
635 + assert_eq!(ranges[0].offset, 2);
636 + assert_eq!(ranges[0].length, 4);
637 + assert_bool_mask(&ranges[0], &[false, true, true, true]);
638 + }
639 +
640 + #[test]
641 + fn expand_prompt_tokens_alignment_pad_realigns_each_span() {
642 + // Every image span must start at phase 1 (odd offset) regardless of
643 + // where the previous span landed.
644 + let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2, DS_IMAGE_ID, 3];
645 + let prepared = vec![deepseek_prepared(vec![2, 2])];
646 +
647 + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
648 + let ranges = &ranges[&Modality::Image];
649 +
650 + // First span at odd offset 1: no pad (block [1,3)).
651 + // Second marker originally at index 3; after the first expansion the
652 + // block starts at offset 4 (even): one pad, span at odd offset 5.
653 + assert_eq!(
654 + prompt_token_ids,
655 + vec![
656 + 1,
657 + DS_IMAGE_ID,
658 + DS_IMAGE_ID,
659 + 2,
660 + DS_IMAGE_PAD_ID,
661 + DS_IMAGE_ID,
662 + DS_IMAGE_ID,
663 + 3,
664 + ]
665 + );
666 + assert_eq!((ranges[0].offset, ranges[0].length), (1, 2));
667 + assert_bool_mask(&ranges[0], &[true, true]);
668 + assert_eq!((ranges[1].offset, ranges[1].length), (4, 3));
669 + assert_bool_mask(&ranges[1], &[false, true, true]);
670 + // Both spans (past the pad) start at odd offsets.
671 + assert_eq!((ranges[1].offset + 1) % 2, 1);
672 + }
673 }
rust/src/chat/src/parser/reasoning/mod.rs
+8 -4
@@ -7,10 +7,11 @@ use std::sync::{Arc, LazyLock};
7
8 pub use vllm_parser::reasoning::{
9 CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
10 - DeepSeekV4ReasoningParser, Glm45ReasoningParser, Glm47ReasoningParser, KimiK2ReasoningParser,
11 - KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
12 - NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError,
13 - ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser,
10 + DeepSeekV4ReasoningParser, DeepSeekV41ReasoningParser, Glm45ReasoningParser,
11 + Glm47ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser, MiniMaxM2ReasoningParser,
12 + MiniMaxM3ReasoningParser, NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta,
13 + ReasoningError, ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser,
14 + Step3p5ReasoningParser,
15 };
16 use vllm_tokenizer::DynTokenizer;
17
@@ -22,6 +23,7 @@ pub mod names {
23 pub const DEEPSEEK_R1: &str = "deepseek_r1";
24 pub const DEEPSEEK_V3: &str = "deepseek_v3";
25 pub const DEEPSEEK_V4: &str = "deepseek_v4";
26 + pub const DEEPSEEK_V41: &str = "deepseek_v41";
27 pub const GLM45: &str = "glm45";
28 pub const GLM47: &str = "glm47";
29 pub const KIMI: &str = "kimi";
@@ -62,6 +64,7 @@ impl ReasoningParserFactory {
64 .register_parser::<DeepSeekR1ReasoningParser>(names::DEEPSEEK_R1)
65 .register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
66 .register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
67 + .register_parser::<DeepSeekV41ReasoningParser>(names::DEEPSEEK_V41)
68 .register_parser::<Glm45ReasoningParser>(names::GLM45)
69 .register_parser::<Glm47ReasoningParser>(names::GLM47)
70 .register_parser::<KimiReasoningParser>(names::KIMI)
@@ -76,6 +79,7 @@ impl ReasoningParserFactory {
79
80 factory
81 .register_pattern("deepseek-r1", names::DEEPSEEK_R1)
82 + .register_pattern("deepseek-v4.1", names::DEEPSEEK_V41)
83 .register_pattern("deepseek-v4", names::DEEPSEEK_V4)
84 .register_pattern("deepseek_v4", names::DEEPSEEK_V4)
85 .register_pattern("deepseek-v3", names::DEEPSEEK_V3)
rust/src/chat/src/parser/reasoning/tests.rs
+4
@@ -33,6 +33,10 @@ fn factory_resolves_deepseek_v4() {
33 factory.resolve_name_for_model("deepseek_v4"),
34 Some(names::DEEPSEEK_V4)
35 );
36 + assert_eq!(
37 + factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"),
38 + Some(names::DEEPSEEK_V41)
39 + );
40 }
41
42 #[test]
rust/src/chat/src/parser/tool/mod.rs
+7 -4
@@ -7,10 +7,10 @@ use std::sync::{Arc, LazyLock};
7
8 pub use vllm_parser::tool::{
9 DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
10 - Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
11 - Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser,
12 - MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser,
13 - Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
10 + DeepSeekV41ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser,
11 + HermesToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
12 + MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
13 + Qwen3CoderToolParser, Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
14 };
15
16 use crate::parser::ParserFactory;
@@ -22,6 +22,7 @@ pub mod names {
22 pub const DEEPSEEK_V31: &str = "deepseek_v31";
23 pub const DEEPSEEK_V32: &str = "deepseek_v32";
24 pub const DEEPSEEK_V4: &str = "deepseek_v4";
25 + pub const DEEPSEEK_V41: &str = "deepseek_v41";
26 pub const GLM45: &str = "glm45";
27 pub const GLM47: &str = "glm47";
28 pub const GRANITE4: &str = "granite4";
@@ -66,6 +67,7 @@ impl ToolParserFactory {
67 .register_parser::<DeepSeekV31ToolParser>(names::DEEPSEEK_V31)
68 .register_parser::<DeepSeekV32ToolParser>(names::DEEPSEEK_V32)
69 .register_parser::<DeepSeekV4ToolParser>(names::DEEPSEEK_V4)
70 + .register_parser::<DeepSeekV41ToolParser>(names::DEEPSEEK_V41)
71 .register_parser::<Glm45MoeToolParser>(names::GLM45)
72 .register_parser::<Glm47MoeToolParser>(names::GLM47)
73 .register_parser::<Granite4ToolParser>(names::GRANITE4)
@@ -101,6 +103,7 @@ impl ToolParserFactory {
103 .register_pattern("llama-3.2", names::LLAMA3_JSON)
104 .register_pattern("llama-3.1", names::LLAMA3_JSON)
105 .register_pattern("deepseek-r1", names::DEEPSEEK_V3)
106 + .register_pattern("deepseek-v4.1", names::DEEPSEEK_V41)
107 .register_pattern("deepseek-v4", names::DEEPSEEK_V4)
108 .register_pattern("deepseek_v4", names::DEEPSEEK_V4)
109 .register_pattern("deepseek-v3.2", names::DEEPSEEK_V32)
rust/src/chat/src/parser/tool/tests.rs
+4
@@ -147,6 +147,10 @@ fn factory_new_resolves_default_patterns() {
147 factory.resolve_name_for_model("deepseek_v4"),
148 Some(names::DEEPSEEK_V4)
149 );
150 + assert_eq!(
151 + factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"),
152 + Some(names::DEEPSEEK_V41)
153 + );
154 assert_eq!(
155 factory.resolve_name_for_model("deepseek-ai/DeepSeek-R1-0528"),
156 Some(names::DEEPSEEK_V3)
rust/src/chat/src/renderer/deepseek.rs renamed
+230 -59
@@ -1,11 +1,12 @@
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -//! DeepSeek V4 prompt renderer.
4 +//! Shared DeepSeek V4 and V4.1 prompt rendering.
5 //!
6 //! Official Python reference:
7 //! <https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/encoding/encoding_dsv4.py>
8
9 +use std::borrow::Cow;
10 use std::collections::HashMap;
11 use std::fmt::Write as _;
12
@@ -13,8 +14,12 @@ use serde::Serialize;
14 use serde_json::Value;
15 use serde_json_fmt::JsonFormat;
16
17 +use llm_multimodal::DEEPSEEK_V41_IMAGE_PLACEHOLDER;
18 +
19 use crate::error::{Error, Result};
17 -use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, ReasoningEffort};
20 +use crate::request::{
21 + ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ReasoningEffort,
22 +};
23 use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
24
25 const BOS_TOKEN: &str = "<｜begin▁of▁sentence｜>";
@@ -23,6 +28,7 @@ const THINKING_START_TOKEN: &str = "<think>";
28 const THINKING_END_TOKEN: &str = "</think>";
29 const DSML_TOKEN: &str = "｜DSML｜";
30 const USER_SP_TOKEN: &str = "<｜User｜>";
31 +const SYSTEM_SP_TOKEN: &str = "<｜System｜>";
32 const ASSISTANT_SP_TOKEN: &str = "<｜Assistant｜>";
33 const REASONING_EFFORT_HIGH: &str = concat!(
34 "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n",
@@ -41,6 +47,35 @@ enum ThinkingMode {
47 Thinking,
48 }
49
50 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51 +pub(super) enum DsDialect {
52 + V4,
53 + V41,
54 +}
55 +
56 +impl DsDialect {
57 + fn tool_calls_tag(self) -> &'static str {
58 + match self {
59 + Self::V4 => "tool_calls",
60 + Self::V41 => " calls",
61 + }
62 + }
63 +
64 + fn invoke_tag(self) -> &'static str {
65 + match self {
66 + Self::V4 => "invoke",
67 + Self::V41 => " invoke",
68 + }
69 + }
70 +
71 + fn parameter_tag(self) -> &'static str {
72 + match self {
73 + Self::V4 => "parameter",
74 + Self::V41 => " parameter",
75 + }
76 + }
77 +}
78 +
79 #[serde_with::skip_serializing_none]
80 #[derive(Debug, Serialize)]
81 struct RenderedToolSchema<'a> {
@@ -51,25 +86,47 @@ struct RenderedToolSchema<'a> {
86 }
87
88 /// Render one chat request into the final prompt string.
54 -pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
55 - let (thinking_mode, reasoning_effort_prompt) = resolve_thinking_options(request)?;
89 +pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Result<String> {
90 + let (thinking_mode, reasoning_effort_prompt) = match dialect {
91 + DsDialect::V4 => {
92 + resolve_thinking_options(request).map(|(mode, prompt)| (mode, Cow::Borrowed(prompt)))?
93 + }
94 + DsDialect::V41 => resolve_v41_thinking_options(request)?,
95 + };
96 let request_tools = request_tools(request);
97 let synthetic_tool_system = needs_synthetic_tool_system(request, request_tools);
98 let drop_thinking = request.parse_template_bool("drop_thinking")?.unwrap_or(true)
99 && !rendered_tools_present(request, request_tools);
100 let drop_historical_developers = thinking_mode == ThinkingMode::Thinking && drop_thinking;
61 - let last_user_like_message_index = request.messages.iter().rposition(is_user_like_entry);
62 - let last_user_render_index =
63 - find_last_user_render_index(request.messages.as_slice(), synthetic_tool_system);
101 + let last_user_like_message_index = request
102 + .messages
103 + .iter()
104 + .enumerate()
105 + .rfind(|(index, message)| is_user_like_entry(message, *index, dialect))
106 + .map(|(index, _)| index);
107 + let last_user_render_index = find_last_user_render_index(
108 + request.messages.as_slice(),
109 + synthetic_tool_system,
110 + drop_historical_developers,
111 + last_user_like_message_index,
112 + dialect,
113 + );
114 let mut out = String::from(BOS_TOKEN);
115 + if dialect == DsDialect::V41
116 + && (thinking_mode == ThinkingMode::Thinking
117 + || synthetic_tool_system
118 + || matches!(request.messages.first(), Some(ChatMessage::System { .. })))
119 + {
120 + out.push_str(SYSTEM_SP_TOKEN);
121 + }
122 if thinking_mode == ThinkingMode::Thinking {
66 - out.push_str(reasoning_effort_prompt);
123 + out.push_str(&reasoning_effort_prompt);
124 }
125
126 let mut request_tools_attached = false;
127 let mut render_index = 0isize;
128 if synthetic_tool_system {
72 - render_system_message(&mut out, None, request_tools)?;
129 + render_system_message(&mut out, None, request_tools, dialect)?;
130 request_tools_attached = true;
131 render_index += 1;
132 }
@@ -80,27 +137,35 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
137 continue;
138 }
139
83 - let current_render_index = render_index;
84 - render_index += 1;
85 -
140 if drop_historical_developers
141 && is_historical_developer(message, message_index, last_user_like_message_index)
142 {
143 continue;
144 }
145
146 + let current_render_index = render_index;
147 + render_index += 1;
148 +
149 match message {
150 ChatMessage::System { content } => {
151 + if dialect == DsDialect::V41 && current_render_index > 0 {
152 + out.push_str(SYSTEM_SP_TOKEN);
153 + }
154 let tools = if !request_tools_attached {
155 request_tools_attached = true;
156 request_tools
157 } else {
158 &[]
159 };
100 - render_system_message(&mut out, Some(content), tools)?;
160 + render_system_message(&mut out, Some(content), tools, dialect)?;
161 }
162 ChatMessage::Developer { content, tools } => {
103 - render_developer_message(&mut out, content, tools.as_deref().unwrap_or(&[]))?;
163 + render_developer_message(
164 + &mut out,
165 + content,
166 + tools.as_deref().unwrap_or(&[]),
167 + dialect,
168 + )?;
169 }
170 ChatMessage::User { .. } | ChatMessage::ToolResponse { .. } => {
171 render_user_content_block(
@@ -108,6 +173,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
173 request.messages.as_slice(),
174 message_index,
175 &last_tool_call_order,
176 + dialect,
177 )?;
178 }
179 ChatMessage::Assistant { content } => {
@@ -119,7 +185,13 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
185 && (!drop_thinking || current_render_index > last_user_render_index);
186 let append_eos = !(message_index + 1 == request.messages.len()
187 && request.chat_options.continue_final_message());
122 - render_assistant_message(&mut out, emit_thinking_block, append_eos, content)?;
188 + render_assistant_message(
189 + &mut out,
190 + emit_thinking_block,
191 + append_eos,
192 + content,
193 + dialect,
194 + )?;
195
196 if content.has_tool_calls() {
197 last_tool_call_order.clear();
@@ -133,7 +205,8 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
205 }
206 }
207
136 - if (is_user_like_entry(message) || matches!(message, ChatMessage::System { .. }))
208 + if (is_user_like_entry(message, current_render_index as usize, dialect)
209 + || (dialect == DsDialect::V4 && matches!(message, ChatMessage::System { .. })))
210 && next_rendered_entry_is_assistant_or_end(
211 request.messages.as_slice(),
212 message_index,
@@ -181,6 +254,57 @@ fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'st
254 Ok((thinking_mode, reasoning_effort_prompt))
255 }
256
257 +/// Resolve V4.1's numeric reasoning effort using the top-level value before template kwargs.
258 +fn resolve_v41_thinking_options(
259 + request: &ChatRequest,
260 +) -> Result<(ThinkingMode, Cow<'static, str>)> {
261 + let mut thinking = request.enable_thinking()?.unwrap_or(true);
262 + let budget = match request.chat_options.reasoning_effort {
263 + Some(ReasoningEffort::None) => {
264 + thinking = false;
265 + 50
266 + }
267 + Some(ReasoningEffort::Low) => 25,
268 + Some(ReasoningEffort::High) => 50,
269 + Some(ReasoningEffort::XHigh) => 75,
270 + Some(ReasoningEffort::Max) => 100,
271 + Some(ReasoningEffort::Minimal | ReasoningEffort::Medium) => {
272 + return Err(invalid_v41_reasoning_effort());
273 + }
274 + None => match request.chat_options.template_kwargs.get("reasoning_effort") {
275 + None | Some(Value::Null) => 50,
276 + Some(Value::String(effort)) => match effort.as_str() {
277 + "low" => 25,
278 + "high" => 50,
279 + "xhigh" => 75,
280 + "max" => 100,
281 + _ => return Err(invalid_v41_reasoning_effort()),
282 + },
283 + Some(Value::Number(budget)) => budget
284 + .as_u64()
285 + .filter(|budget| (1..=100).contains(budget))
286 + .ok_or_else(invalid_v41_reasoning_effort)?,
287 + Some(_) => return Err(invalid_v41_reasoning_effort()),
288 + },
289 + };
290 + if thinking {
291 + Ok((
292 + ThinkingMode::Thinking,
293 + Cow::Owned(format!(
294 + "Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n"
295 + )),
296 + ))
297 + } else {
298 + Ok((ThinkingMode::Chat, Cow::Borrowed("")))
299 + }
300 +}
301 +
302 +fn invalid_v41_reasoning_effort() -> Error {
303 + Error::InvalidReasoningEffort(
304 + "DeepSeek V4.1 reasoning_effort must be low, high, xhigh, max, or an integer within [1, 100] in chat_template_kwargs".to_string(),
305 + )
306 +}
307 +
308 /// Return request-level tools only when native tool parsing is enabled.
309 fn request_tools(request: &ChatRequest) -> &[ChatTool] {
310 if request.tool_parsing_enabled() {
@@ -214,16 +338,25 @@ fn rendered_tools_present(request: &ChatRequest, request_tools: &[ChatTool]) ->
338 }
339
340 /// Find the last user-like turn after inline tool-response merging.
217 -fn find_last_user_render_index(messages: &[ChatMessage], synthetic_tool_system: bool) -> isize {
341 +fn find_last_user_render_index(
342 + messages: &[ChatMessage],
343 + synthetic_tool_system: bool,
344 + drop_historical_developers: bool,
345 + last_user_like_message_index: Option<usize>,
346 + dialect: DsDialect,
347 +) -> isize {
348 let mut render_index = isize::from(synthetic_tool_system);
349 let mut last_user_index = -1;
350
351 for (message_index, message) in messages.iter().enumerate() {
222 - if is_following_user_content(messages, message_index) {
352 + if is_following_user_content(messages, message_index)
353 + || (drop_historical_developers
354 + && is_historical_developer(message, message_index, last_user_like_message_index))
355 + {
356 continue;
357 }
358
226 - if is_user_like_entry(message) {
359 + if is_user_like_entry(message, render_index as usize, dialect) {
360 last_user_index = render_index;
361 }
362 render_index += 1;
@@ -249,11 +382,13 @@ fn is_user_content_entry(message: &ChatMessage) -> bool {
382 }
383
384 /// Return whether one rendered entry should be treated as user-like.
252 -fn is_user_like_entry(message: &ChatMessage) -> bool {
385 +fn is_user_like_entry(message: &ChatMessage, message_index: usize, dialect: DsDialect) -> bool {
386 matches!(
387 message,
388 ChatMessage::Developer { .. } | ChatMessage::User { .. } | ChatMessage::ToolResponse { .. }
256 - )
389 + ) || (dialect == DsDialect::V41
390 + && message_index > 0
391 + && matches!(message, ChatMessage::System { .. }))
392 }
393
394 /// Return whether a developer entry precedes another user-like turn.
@@ -293,22 +428,26 @@ fn next_rendered_entry_is_assistant_or_end(
428 .unwrap_or(true)
429 }
430
296 -/// Render the tool preamble shown to the model, V4 flavor.
297 -fn render_tools(out: &mut String, tools: &[ChatTool]) -> Result<()> {
298 - out.push_str(
431 +/// Render the tool preamble shown to the model for one DeepSeek dialect.
432 +fn render_tools(out: &mut String, tools: &[ChatTool], dialect: DsDialect) -> Result<()> {
433 + let tool_calls_tag = dialect.tool_calls_tag();
434 + let invoke_tag = dialect.invoke_tag();
435 + let parameter_tag = dialect.parameter_tag();
436 + write!(
437 + out,
438 r#"## Tools
439
301 -You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following:
440 +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜{tool_calls_tag}>" block like the following:
441
303 -<｜DSML｜tool_calls>
304 -<｜DSML｜invoke name="$TOOL_NAME">
305 -<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜parameter>
442 +<｜DSML｜{tool_calls_tag}>
443 +<｜DSML｜{invoke_tag} name="$TOOL_NAME">
444 +<｜DSML｜{parameter_tag} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜{parameter_tag}>
445 ...
307 -</｜DSML｜invoke>
308 -<｜DSML｜invoke name="$TOOL_NAME2">
446 +</｜DSML｜{invoke_tag}>
447 +<｜DSML｜{invoke_tag} name="$TOOL_NAME2">
448 ...
310 -</｜DSML｜invoke>
311 -</｜DSML｜tool_calls>
449 +</｜DSML｜{invoke_tag}>
450 +</｜DSML｜{tool_calls_tag}>
451
452 String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
453
@@ -319,7 +458,8 @@ Otherwise, output directly after </think> with tool calls or final response.
458 ### Available Tool Schemas
459
460 "#,
322 - );
461 + )
462 + .expect("writing to String cannot fail");
463
464 for (index, tool) in tools.iter().enumerate() {
465 if index > 0 {
@@ -350,13 +490,14 @@ fn render_system_message(
490 out: &mut String,
491 content: Option<&ChatContent>,
492 tools: &[ChatTool],
493 + dialect: DsDialect,
494 ) -> Result<()> {
495 if let Some(content) = content {
355 - write_chat_content(out, content)?;
496 + write_chat_content(out, content, dialect)?;
497 }
498 if !tools.is_empty() {
499 out.push_str("\n\n");
359 - render_tools(out, tools)?;
500 + render_tools(out, tools, dialect)?;
501 }
502 Ok(())
503 }
@@ -366,18 +507,19 @@ fn render_developer_message(
507 out: &mut String,
508 content: &ChatContent,
509 tools: &[ChatTool],
510 + dialect: DsDialect,
511 ) -> Result<()> {
512 if content.is_empty() {
513 return Err(Error::ChatTemplate(
372 - "invalid DeepSeek V4 developer message: empty content".to_string(),
514 + "invalid DeepSeek developer message: empty content".to_string(),
515 ));
516 }
517
518 out.push_str(USER_SP_TOKEN);
377 - write_chat_content(out, content)?;
519 + write_chat_content(out, content, dialect)?;
520 if !tools.is_empty() {
521 out.push_str("\n\n");
380 - render_tools(out, tools)?;
522 + render_tools(out, tools, dialect)?;
523 }
524 Ok(())
525 }
@@ -388,6 +530,7 @@ fn render_user_content_block(
530 messages: &[ChatMessage],
531 message_index: usize,
532 tool_call_order: &HashMap<String, usize>,
533 + dialect: DsDialect,
534 ) -> Result<()> {
535 let (block_start, block_end) = user_content_block_bounds(messages, message_index);
536 let mut sorted_tool_indices =
@@ -399,7 +542,7 @@ fn render_user_content_block(
542 out.push_str("\n\n");
543 }
544 match &messages[message_index] {
402 - ChatMessage::User { content } => write_chat_content(out, content)?,
545 + ChatMessage::User { content } => write_chat_content(out, content, dialect)?,
546 ChatMessage::ToolResponse { .. } => {
547 let sorted_index = sorted_tool_indices
548 .next()
@@ -407,7 +550,7 @@ fn render_user_content_block(
550 let ChatMessage::ToolResponse { content, .. } = &messages[sorted_index] else {
551 unreachable!("sorted tool response index should reference a tool message");
552 };
410 - write_tool_result(out, content)?;
553 + write_tool_result(out, content, dialect)?;
554 }
555 _ => unreachable!("user content block should only contain user content messages"),
556 }
@@ -454,9 +597,9 @@ fn sorted_tool_response_indices(
597 }
598
599 /// Render one tool response payload inside a V4 `<tool_result>` block.
457 -fn write_tool_result(out: &mut String, content: &ChatContent) -> Result<()> {
600 +fn write_tool_result(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
601 out.push_str("<tool_result>");
459 - write_chat_content(out, content)?;
602 + write_chat_content(out, content, dialect)?;
603 out.push_str("</tool_result>");
604 Ok(())
605 }
@@ -483,6 +626,7 @@ fn render_assistant_message(
626 emit_thinking_block: bool,
627 append_eos: bool,
628 content: &[AssistantContentBlock],
629 + dialect: DsDialect,
630 ) -> Result<()> {
631 let has_tool_calls = content.has_tool_calls();
632
@@ -496,14 +640,15 @@ fn render_assistant_message(
640 write_assistant_text(out, content);
641
642 if has_tool_calls {
499 - out.push_str("\n\n<｜DSML｜tool_calls>\n");
643 + let tool_calls_tag = dialect.tool_calls_tag();
644 + writeln!(out, "\n\n<{DSML_TOKEN}{tool_calls_tag}>").expect("writing to String cannot fail");
645 for (index, tool_call) in content.tool_calls().enumerate() {
646 if index > 0 {
647 out.push('\n');
648 }
504 - render_tool_call(out, tool_call)?;
649 + render_tool_call(out, tool_call, dialect)?;
650 }
506 - out.push_str("\n</｜DSML｜tool_calls>");
651 + write!(out, "\n</{DSML_TOKEN}{tool_calls_tag}>").expect("writing to String cannot fail");
652 }
653
654 if append_eos {
@@ -513,11 +658,20 @@ fn render_assistant_message(
658 }
659
660 /// Render one assistant tool call in DSML XML-like format.
516 -fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
517 - writeln!(out, "<{DSML_TOKEN}invoke name=\"{}\">", tool_call.name)
518 - .expect("writing to String cannot fail");
519 - encode_arguments_to_dsml(out, tool_call)?;
520 - write!(out, "\n</{DSML_TOKEN}invoke>").expect("writing to String cannot fail");
661 +fn render_tool_call(
662 + out: &mut String,
663 + tool_call: &AssistantToolCall,
664 + dialect: DsDialect,
665 +) -> Result<()> {
666 + let invoke_tag = dialect.invoke_tag();
667 + writeln!(
668 + out,
669 + "<{DSML_TOKEN}{invoke_tag} name=\"{}\">",
670 + tool_call.name
671 + )
672 + .expect("writing to String cannot fail");
673 + encode_arguments_to_dsml(out, tool_call, dialect)?;
674 + write!(out, "\n</{DSML_TOKEN}{invoke_tag}>").expect("writing to String cannot fail");
675 Ok(())
676 }
677
@@ -525,15 +679,20 @@ fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<(
679 ///
680 /// String values are emitted raw with `string="true"`, while all other JSON
681 /// values are rendered with JSON syntax and `string="false"`.
528 -fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
682 +fn encode_arguments_to_dsml(
683 + out: &mut String,
684 + tool_call: &AssistantToolCall,
685 + dialect: DsDialect,
686 +) -> Result<()> {
687 + let parameter_tag = dialect.parameter_tag();
688 let arguments: Value = serde_json::from_str(&tool_call.arguments).map_err(|error| {
689 Error::ChatTemplate(format!(
531 - "assistant tool call has invalid JSON arguments for DeepSeek V4: {error}"
690 + "assistant tool call has invalid JSON arguments for DeepSeek: {error}"
691 ))
692 })?;
693 let Some(arguments) = arguments.as_object() else {
694 return Err(Error::ChatTemplate(
536 - "assistant tool call arguments for DeepSeek V4 must be a JSON object".to_string(),
695 + "assistant tool call arguments for DeepSeek must be a JSON object".to_string(),
696 ));
697 };
698
@@ -546,7 +705,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
705 let is_string = matches!(value, Value::String(_));
706 write!(
707 out,
549 - "<{DSML_TOKEN}parameter name=\"{key}\" string=\"{}\">",
708 + "<{DSML_TOKEN}{parameter_tag} name=\"{key}\" string=\"{}\">",
709 if is_string { "true" } else { "false" }
710 )
711 .expect("writing to String cannot fail");
@@ -556,7 +715,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
715 value => out.push_str(&json_dumps(value)?),
716 }
717
559 - write!(out, "</{DSML_TOKEN}parameter>").expect("writing to String cannot fail");
718 + write!(out, "</{DSML_TOKEN}{parameter_tag}>").expect("writing to String cannot fail");
719 wrote_parameter = true;
720 }
721
@@ -565,12 +724,24 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
724
725 /// Write chat content directly into the destination buffer without flattening
726 /// it into an intermediate `String`.
568 -fn write_chat_content(out: &mut String, content: &ChatContent) -> Result<()> {
727 +///
728 +/// The V4.1 dialect inlines the image placeholder at each image part's
729 +/// position unconditionally (matching the Python encoding's
730 +/// `IMAGE_PLACEHOLDER`); other dialects reject multimodal parts.
731 +fn write_chat_content(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
732 match content {
733 ChatContent::Text(text) => out.push_str(text),
734 ChatContent::Parts(parts) => {
572 - for part in parts {
573 - out.push_str(part.as_text()?);
735 + for (index, part) in parts.iter().enumerate() {
736 + if index > 0 && dialect == DsDialect::V41 {
737 + out.push_str("\n\n");
738 + }
739 + match part {
740 + ChatContentPart::ImageUrl { .. } if dialect == DsDialect::V41 => {
741 + out.push_str(DEEPSEEK_V41_IMAGE_PLACEHOLDER);
742 + }
743 + _ => out.push_str(part.as_text()?),
744 + }
745 }
746 }
747 }
@@ -606,7 +777,7 @@ fn json_dumps<T: Serialize>(value: &T) -> Result<String> {
777 .format_to_string(value)
778 .map_err(|error| {
779 Error::ChatTemplate(format!(
609 - "failed to serialize DeepSeek V4 JSON payload: {error}"
780 + "failed to serialize DeepSeek JSON payload: {error}"
781 ))
782 })
783 }
rust/src/chat/src/renderer/deepseek_v4/mod.rs
+2 -3
@@ -1,10 +1,9 @@
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -mod encoding;
5 -
4 use vllm_text::Prompt;
5
6 +use super::deepseek::{self, DsDialect};
7 use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
8 use crate::Result;
9 use crate::request::ChatRequest;
@@ -24,7 +23,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {
23 request.validate()?;
24
25 Ok(RenderedPrompt {
27 - prompt: Prompt::Text(encoding::render_request(request)?),
26 + prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V4)?),
27 effective_template_kwargs: request_template_kwargs(request),
28 })
29 }
rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_1.txt new
+38
@@ -0,0 +1,38 @@
1 +<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
2 +
3 +You are a helpful assistant.
4 +
5 +## Tools
6 +
7 +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜ calls>" block like the following:
8 +
9 +<｜DSML｜ calls>
10 +<｜DSML｜ invoke name="$TOOL_NAME">
11 +<｜DSML｜ parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜ parameter>
12 +...
13 +</｜DSML｜ invoke>
14 +<｜DSML｜ invoke name="$TOOL_NAME2">
15 +...
16 +</｜DSML｜ invoke>
17 +</｜DSML｜ calls>
18 +
19 +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
20 +
21 +If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
22 +
23 +Otherwise, output directly after </think> with tool calls or final response.
24 +
25 +### Available Tool Schemas
26 +
27 +{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
28 +{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
29 +
30 +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
31 +<｜User｜>What's the weather in Beijing?<｜Assistant｜><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
32 +
33 +<｜DSML｜ calls>
34 +<｜DSML｜ invoke name="get_weather">
35 +<｜DSML｜ parameter name="location" string="true">Beijing</｜DSML｜ parameter>
36 +<｜DSML｜ parameter name="unit" string="true">celsius</｜DSML｜ parameter>
37 +</｜DSML｜ invoke>
38 +</｜DSML｜ calls><｜end▁of▁sentence｜><｜User｜><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><｜Assistant｜><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<｜end▁of▁sentence｜>
\ No newline at end of file
rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_2.txt new
+3
@@ -0,0 +1,3 @@
1 +<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
2 +
3 +You are a helpful assistant.<｜User｜>Hello<｜Assistant｜></think>Hi there! How can I help you?<｜end▁of▁sentence｜><｜User｜>What is the capital of France?<｜Assistant｜><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<｜end▁of▁sentence｜>
\ No newline at end of file
rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_developer_tools.txt new
+30
@@ -0,0 +1,30 @@
1 +<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
2 +
3 +<｜User｜>question<｜User｜>policy
4 +
5 +## Tools
6 +
7 +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜ calls>" block like the following:
8 +
9 +<｜DSML｜ calls>
10 +<｜DSML｜ invoke name="$TOOL_NAME">
11 +<｜DSML｜ parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜ parameter>
12 +...
13 +</｜DSML｜ invoke>
14 +<｜DSML｜ invoke name="$TOOL_NAME2">
15 +...
16 +</｜DSML｜ invoke>
17 +</｜DSML｜ calls>
18 +
19 +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
20 +
21 +If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
22 +
23 +Otherwise, output directly after </think> with tool calls or final response.
24 +
25 +### Available Tool Schemas
26 +
27 +{"name": "lookup", "parameters": {"type": "object"}}
28 +
29 +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
30 +<｜Assistant｜><think></think>answer<｜end▁of▁sentence｜>
\ No newline at end of file
rust/src/chat/src/renderer/deepseek_v41/mod.rs new
+33
@@ -0,0 +1,33 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use vllm_text::Prompt;
5 +
6 +use super::deepseek::{self, DsDialect};
7 +use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
8 +use crate::Result;
9 +use crate::request::ChatRequest;
10 +
11 +/// Dedicated DeepSeek V4.1 renderer.
12 +#[derive(Debug, Clone, Copy, Default)]
13 +pub struct DeepSeekV41ChatRenderer;
14 +
15 +impl DeepSeekV41ChatRenderer {
16 + pub fn new() -> Self {
17 + Self
18 + }
19 +}
20 +
21 +impl ChatRenderer for DeepSeekV41ChatRenderer {
22 + fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
23 + request.validate()?;
24 +
25 + Ok(RenderedPrompt {
26 + prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V41)?),
27 + effective_template_kwargs: request_template_kwargs(request),
28 + })
29 + }
30 +}
31 +
32 +#[cfg(test)]
33 +mod tests;
rust/src/chat/src/renderer/deepseek_v41/tests.rs new
+223
@@ -0,0 +1,223 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use std::path::PathBuf;
5 +
6 +use expect_test::{expect, expect_file};
7 +use serde_json::{Value, json};
8 +
9 +use super::DeepSeekV41ChatRenderer;
10 +use crate::ChatRenderer;
11 +use crate::event::AssistantContentBlock;
12 +use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
13 +use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ReasoningEffort};
14 +
15 +fn render(request: &ChatRequest) -> String {
16 + DeepSeekV41ChatRenderer::new()
17 + .render(request)
18 + .unwrap()
19 + .prompt
20 + .into_text()
21 + .unwrap()
22 +}
23 +
24 +fn request() -> ChatRequest {
25 + ChatRequest {
26 + messages: vec![ChatMessage::user("question")],
27 + ..ChatRequest::for_test()
28 + }
29 +}
30 +
31 +#[test]
32 +fn renders_shared_deepseek_fixtures_with_v41_reference_encoding() {
33 + let fixture_dir =
34 + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/renderer/deepseek_v4/fixtures");
35 + for (input, expected) in [
36 + (
37 + "test_input_1.json",
38 + expect_file!["fixtures/test_output_1.txt"],
39 + ),
40 + (
41 + "test_input_2.json",
42 + expect_file!["fixtures/test_output_2.txt"],
43 + ),
44 + (
45 + "test_input_developer_tools.json",
46 + expect_file!["fixtures/test_output_developer_tools.txt"],
47 + ),
48 + ] {
49 + let request = fixture_chat_request(
50 + &fixture_dir.join(input),
51 + FixtureRequestOptions {
52 + enable_thinking: Some(true),
53 + no_generation_prompt_when_last_assistant: true,
54 + },
55 + );
56 + expected.assert_eq(&render(&request));
57 + }
58 +}
59 +
60 +#[test]
61 +fn inlines_image_placeholder_at_image_part_positions() {
62 + // The placeholder is unconditional (matching the Python encoding's
63 + // `IMAGE_PLACEHOLDER`); requests to a multimodal-less backend fail later,
64 + // at media preparation.
65 + let renderer = DeepSeekV41ChatRenderer::new();
66 + let request = ChatRequest {
67 + messages: vec![ChatMessage::User {
68 + content: ChatContent::Parts(vec![
69 + ChatContentPart::image_url("https://example.com/carrots.jpeg"),
70 + ChatContentPart::text("what is in the image?"),
71 + ]),
72 + }],
73 + ..ChatRequest::for_test()
74 + };
75 +
76 + let rendered = renderer.render(&request).unwrap().prompt.into_text().unwrap();
77 +
78 + // Image parts inline the placeholder at their position; parts stay
79 + // separated by "\n\n" (the V4.1 reference encoding's part separator).
80 + assert!(
81 + rendered.contains("<｜User｜><｜deepseek_image｜>\n\nwhat is in the image?"),
82 + "unexpected prompt: {rendered:?}"
83 + );
84 +}
85 +
86 +#[test]
87 +fn maps_reasoning_effort_to_reference_numeric_budget() {
88 + for (effort, budget) in [
89 + (None, 50),
90 + (Some(ReasoningEffort::Low), 25),
91 + (Some(ReasoningEffort::High), 50),
92 + (Some(ReasoningEffort::XHigh), 75),
93 + (Some(ReasoningEffort::Max), 100),
94 + ] {
95 + let mut request = request();
96 + request.chat_options.reasoning_effort = effort;
97 + assert_eq!(
98 + render(&request),
99 + format!(
100 + "<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n<｜User｜>question<｜Assistant｜><think>"
101 + )
102 + );
103 + }
104 +}
105 +
106 +#[test]
107 +fn accepts_numeric_template_effort_with_top_level_precedence() {
108 + for (effort, budget) in [
109 + (json!(1), 1),
110 + (json!(42), 42),
111 + (json!(100), 100),
112 + (json!("low"), 25),
113 + ] {
114 + let mut request = request();
115 + request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
116 + assert!(render(&request).contains(&format!("Reasoning Effort: {budget} (range 1-100")));
117 +
118 + request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh);
119 + assert!(render(&request).contains("Reasoning Effort: 75 (range 1-100"));
120 + }
121 +}
122 +
123 +#[test]
124 +fn rejects_undefined_effort_names_and_invalid_numeric_budgets() {
125 + for effort in [
126 + json!("medium"),
127 + json!("minimal"),
128 + json!(-1),
129 + json!(0),
130 + json!(101),
131 + json!(true),
132 + json!(1.5),
133 + ] {
134 + let mut request = request();
135 + request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
136 + let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
137 + assert!(error.is_request_validation_error());
138 + }
139 + for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] {
140 + let mut request = request();
141 + request.chat_options.reasoning_effort = Some(effort);
142 + let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
143 + assert!(error.is_request_validation_error());
144 + }
145 +}
146 +
147 +#[test]
148 +fn chat_mode_and_none_effort_use_closed_thinking_prefix() {
149 + let mut request = request();
150 + request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
151 + let expected = expect!["<｜begin▁of▁sentence｜><｜User｜>question<｜Assistant｜></think>"];
152 + expected.assert_eq(&render(&request));
153 +
154 + request.chat_options.template_kwargs.insert("thinking".into(), json!(true));
155 + request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
156 + expected.assert_eq(&render(&request));
157 +}
158 +
159 +#[test]
160 +fn leading_system_in_chat_mode_uses_system_token() {
161 + let mut request = request();
162 + request.messages.insert(0, ChatMessage::system("policy"));
163 + request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
164 + expect!["<｜begin▁of▁sentence｜><｜System｜>policy<｜User｜>question<｜Assistant｜></think>"]
165 + .assert_eq(&render(&request));
166 +}
167 +
168 +#[test]
169 +fn system_after_dropped_developer_becomes_leading_system() {
170 + let mut request = ChatRequest {
171 + messages: vec![
172 + ChatMessage::developer("old policy", None),
173 + ChatMessage::system("sys"),
174 + ],
175 + ..ChatRequest::for_test()
176 + };
177 + expect!["<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys"]
178 + .assert_eq(&render(&request));
179 +
180 + request.messages.push(ChatMessage::user("question"));
181 + expect!["<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<｜User｜>question<｜Assistant｜><think>"]
182 + .assert_eq(&render(&request));
183 +}
184 +
185 +#[test]
186 +fn mid_system_advances_last_user_boundary_and_drops_old_reasoning() {
187 + let mut request = ChatRequest {
188 + messages: vec![
189 + ChatMessage::system("sys"),
190 + ChatMessage::user("q1"),
191 + ChatMessage::assistant_blocks(vec![
192 + AssistantContentBlock::Reasoning { text: "r1".into() },
193 + AssistantContentBlock::Text { text: "a1".into() },
194 + ]),
195 + ChatMessage::developer("old policy", None),
196 + ChatMessage::system("mid sys"),
197 + ],
198 + ..ChatRequest::for_test()
199 + };
200 + request
201 + .chat_options
202 + .template_kwargs
203 + .insert("reasoning_effort".into(), json!(88));
204 + expect!["<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 88 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<｜User｜>q1<｜Assistant｜></think>a1<｜end▁of▁sentence｜><｜System｜>mid sys<｜Assistant｜><think>"]
205 + .assert_eq(&render(&request));
206 +}
207 +
208 +#[test]
209 +fn joins_text_parts_with_reference_separator_and_preserves_literal_tags() {
210 + let mut request = ChatRequest {
211 + messages: vec![ChatMessage::user(ChatContent::Parts(vec![
212 + ChatContentPart::text("first"),
213 + ChatContentPart::text("<｜DSML｜invoke name=\"literal\">"),
214 + ]))],
215 + ..ChatRequest::for_test()
216 + };
217 + request
218 + .chat_options
219 + .template_kwargs
220 + .insert("thinking".into(), Value::Bool(false));
221 + expect!["<｜begin▁of▁sentence｜><｜User｜>first\n\n<｜DSML｜invoke name=\"literal\"><｜Assistant｜></think>"]
222 + .assert_eq(&render(&request));
223 +}
rust/src/chat/src/renderer/mod.rs
+3
@@ -10,8 +10,10 @@ use vllm_text::Prompt;
10 use crate::error::Result;
11 use crate::request::{ChatRequest, ReasoningEffort};
12
13 +mod deepseek;
14 pub mod deepseek_v32;
15 pub mod deepseek_v4;
16 +pub mod deepseek_v41;
17 pub mod harmony;
18 pub mod hf;
19 mod inkling;
@@ -22,6 +24,7 @@ mod test_utils;
24
25 pub use deepseek_v4::DeepSeekV4ChatRenderer;
26 pub use deepseek_v32::DeepSeekV32ChatRenderer;
27 +pub use deepseek_v41::DeepSeekV41ChatRenderer;
28 pub use harmony::HarmonyChatRenderer;
29 pub use inkling::InklingChatRenderer;
30 pub use kimi_k3::KimiK3ChatRenderer;
rust/src/chat/src/renderer/selection.rs
+8 -1
@@ -22,6 +22,8 @@ pub enum RendererSelection {
22 DeepSeekV32,
23 /// Force the DeepSeek V4 renderer.
24 DeepSeekV4,
25 + /// Force the DeepSeek V4.1 renderer.
26 + DeepSeekV41,
27 /// Force the GPT-OSS Harmony renderer.
28 Harmony,
29 /// Force the Inkling native token renderer.
@@ -34,6 +36,7 @@ impl RendererSelection {
36 pub const AUTO_LITERAL: &str = "auto";
37 pub const DEEPSEEK_V32_LITERAL: &str = "deepseek_v32";
38 pub const DEEPSEEK_V4_LITERAL: &str = "deepseek_v4";
39 + pub const DEEPSEEK_V41_LITERAL: &str = "deepseek_v41";
40 pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
41 pub const HARMONY_LITERAL: &str = "harmony";
42 pub const HF_LITERAL: &str = "hf";
@@ -48,6 +51,7 @@ impl RendererSelection {
51 Self::Auto => match model_type {
52 Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
53 Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
54 + Self::DEEPSEEK_V41_LITERAL => Self::DeepSeekV41,
55 Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
56 Self::INKLING_MODEL_TYPE => Self::Inkling,
57 Self::KIMI_K3_LITERAL => Self::KimiK3,
@@ -70,6 +74,8 @@ impl FromStr for RendererSelection {
74 Ok(Self::DeepSeekV32)
75 } else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V4_LITERAL) {
76 Ok(Self::DeepSeekV4)
77 + } else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V41_LITERAL) {
78 + Ok(Self::DeepSeekV41)
79 } else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
80 Ok(Self::Harmony)
81 } else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
@@ -92,6 +98,7 @@ impl fmt::Display for RendererSelection {
98 Self::Hf => f.write_str(Self::HF_LITERAL),
99 Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
100 Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
101 + Self::DeepSeekV41 => f.write_str(Self::DEEPSEEK_V41_LITERAL),
102 Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
103 Self::Inkling => f.write_str(Self::INKLING_LITERAL),
104 Self::KimiK3 => f.write_str(Self::KIMI_K3_LITERAL),
@@ -121,7 +128,7 @@ mod tests {
128 fn renderer_selection_expected_error_message() {
129 let err = RendererSelection::from_str("unknown").unwrap_err();
130 expect_test::expect![
124 - "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)"
131 + "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, deepseek_v41, harmony, inkling, kimi_k3)"
132 ]
133 .assert_eq(&err);
134 }
rust/src/chat/tests/roundtrip.rs
+15 -1
@@ -29,7 +29,7 @@ const TOKEN_COMPLETION_CHUNK_TOKENS: usize = 1;
29 /// One model/parser configuration used to run the fixed roundtrip fixtures.
30 #[derive(Clone)]
31 struct RoundtripCase {
32 - /// Hugging Face model id resolved through the production backend loader.
32 + /// Hugging Face model id or local checkpoint resolved through the production backend loader.
33 model_id: &'static str,
34 /// Final assistant-history suffix rendered by the chat template but not
35 /// generated by the model body consumed by the output processor.
@@ -180,6 +180,19 @@ impl RoundtripCase {
180 }
181 }
182
183 + /// DeepSeek V4.1 Flash DSML tool-call format.
184 + fn deepseek_v41() -> Self {
185 + Self {
186 + model_id: "deepseek-ai/DeepSeek-V4.1-Flash",
187 + assistant_stop_suffix: "<｜end▁of▁sentence｜>",
188 + tool_call_parser: ParserSelection::Auto,
189 + reasoning_parser: ParserSelection::Auto,
190 + thinking_behavior: ThinkingBehavior::Toggleable { default: true },
191 + json_fmt: compact_json_fmt(),
192 + sort_json_keys: false,
193 + }
194 + }
195 +
196 /// DeepSeek V3.2 DSML tool-call format.
197 fn deepseek_v32() -> Self {
198 Self {
@@ -385,6 +398,7 @@ roundtrip_tests! {
398 minimax_m25 => [reasoning_and_content, tool_call_mix],
399 minimax_m3 => [reasoning_and_content, tool_call_mix],
400 deepseek_v4 => [reasoning_and_content, tool_call_mix],
401 + deepseek_v41 => [reasoning_and_content, tool_call_mix],
402 deepseek_v32 => [tool_call_mix],
403 glm45 => [reasoning_and_content, tool_call_mix],
404 glm47 => [reasoning_and_content, tool_call_mix],
rust/src/cmd/src/cli/tests.rs
+1 -1
@@ -903,7 +903,7 @@ fn serve_args_reject_unknown_renderer_value() {
903 .unwrap_err();
904
905 expect![[r#"
906 - error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)
906 + error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, deepseek_v41, harmony, inkling, kimi_k3)
907
908 For more information, try '--help'.
909 "#]]
rust/src/parser/src/reasoning/mod.rs
+2
@@ -53,6 +53,8 @@ pub type Step3ReasoningParser = DeepSeekR1ReasoningParser;
53
54 /// DeepSeek V4 currently shares the DeepSeek V3 reasoning parser.
55 pub type DeepSeekV4ReasoningParser = DeepSeekV3ReasoningParser;
56 +/// DeepSeek V4.1 uses bare `<think>...</think>` delimiters.
57 +pub type DeepSeekV41ReasoningParser = DeepSeekV3ReasoningParser;
58 /// GLM-4.7 and GLM-5 use bare reasoning delimiters.
59 pub type Glm47ReasoningParser = DeepSeekV3ReasoningParser;
60 /// Kimi K2 currently shares the DeepSeek V3 reasoning parser.
rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs new
+47
@@ -0,0 +1,47 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use super::{DeepSeekDsmlToolParser, DsmlTokens};
5 +use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
6 +
7 +mod structural_tag;
8 +
9 +/// Tool parser for DeepSeek V4.1's spaced DSML tags.
10 +///
11 +/// Arguments are emitted only after a full `invoke` block is parsed.
12 +pub struct DeepSeekV41ToolParser(DeepSeekDsmlToolParser);
13 +
14 +impl ToolParser for DeepSeekV41ToolParser {
15 + fn create(tools: &[Tool]) -> Result<Box<dyn ToolParser>>
16 + where
17 + Self: Sized + 'static,
18 + {
19 + Ok(Box::new(Self(DeepSeekDsmlToolParser::new(
20 + tools,
21 + DsmlTokens::V41,
22 + ))))
23 + }
24 +
25 + fn preserve_special_tokens(&self) -> bool {
26 + true
27 + }
28 +
29 + fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
30 + Some(&structural_tag::DeepSeekV41StructuralTagBuilder)
31 + }
32 +
33 + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
34 + self.0.parse_into(chunk, output)
35 + }
36 +
37 + fn finish(&mut self) -> Result<ToolParserOutput> {
38 + self.0.finish()
39 + }
40 +
41 + fn reset(&mut self) -> String {
42 + self.0.reset()
43 + }
44 +}
45 +
46 +#[cfg(test)]
47 +mod tests;
rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs new
+108
@@ -0,0 +1,108 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Visible-text grammar for DeepSeek V4.1 tool calls.
5 +//!
6 +//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with
7 +//! V4.1's spaced DSML tags and schema-unconstrained parameter values.
8 +
9 +use serde_json::Value;
10 +use xgrammar_structural_tag::Result;
11 +use xgrammar_structural_tag::builders::{
12 + StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
13 +};
14 +use xgrammar_structural_tag::format::{
15 + Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat,
16 +};
17 +use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
18 +
19 +use super::DsmlTokens;
20 +
21 +const TOKENS: DsmlTokens = DsmlTokens::V41;
22 +
23 +pub(super) struct DeepSeekV41StructuralTagBuilder;
24 +
25 +impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder {
26 + fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
27 + let tags = ctx
28 + .function_tools
29 + .iter()
30 + .map(|tool| call_tag(tool, ctx.options))
31 + .collect::<Vec<_>>();
32 + let excludes = if ctx.options.exclude_special_tokens {
33 + &["<think>", "</think>"][..]
34 + } else {
35 + &[]
36 + };
37 + let format = match ctx.tool_choice {
38 + BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes),
39 + BuilderToolChoice::Auto => Format::TriggeredTags(
40 + TriggeredTagsFormat::new(
41 + &[TOKENS.tool_calls_start],
42 + vec![TagFormat::new(
43 + format!("{}\n", TOKENS.tool_calls_start),
44 + Format::tags_with_separator(tags, "", true, false),
45 + TOKENS.tool_calls_end,
46 + )],
47 + )
48 + .with_excludes(excludes),
49 + ),
50 + BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![
51 + Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)),
52 + Format::tags_with_separator(
53 + tags,
54 + "",
55 + true,
56 + ctx.tool_choice == BuilderToolChoice::Forced,
57 + ),
58 + Format::const_string(TOKENS.tool_calls_end),
59 + ]),
60 + };
61 + // Reasoning boundaries remain owned by the engine reasoner; serving
62 + // calls this visible-text builder with reasoning=false.
63 + Ok(StructuralTag::new(format))
64 + }
65 +}
66 +
67 +fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat {
68 + // TODO: enforce tool.parameters and strict schema semantics once V4.1
69 + // argument conversion is available. This builder intentionally ignores
70 + // parameters and strict: it constrains tool names and DSML/value syntax
71 + // only. Parameter names, presence, uniqueness and value schemas remain
72 + // unconstrained, including for strict=true. Serving still uses strict to
73 + // decide whether auto tool choice activates a grammar.
74 + TagFormat::new(
75 + format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name),
76 + Format::star(parameter(options)),
77 + format!("{}\n", TOKENS.invoke_end),
78 + )
79 +}
80 +
81 +fn parameter(options: StructuralTagOptions) -> Format {
82 + Format::tag(
83 + format!("{} name=\"", TOKENS.parameter_start),
84 + Format::sequence(vec![
85 + Format::regex(r#"[^"]+"#),
86 + Format::const_string("\" string=\""),
87 + Format::or(vec![
88 + Format::sequence(vec![
89 + Format::const_string("true\">"),
90 + Format::any_text_excluding(&[
91 + TOKENS.parameter_end,
92 + TOKENS.invoke_end,
93 + TOKENS.tool_calls_end,
94 + ]),
95 + ]),
96 + Format::sequence(vec![
97 + Format::const_string("false\">"),
98 + Format::JsonSchema(
99 + JsonSchemaFormat::new(Value::Bool(true))
100 + .with_any_order(options.any_order)
101 + .with_max_whitespace_cnt(options.max_whitespace_cnt),
102 + ),
103 + ]),
104 + ]),
105 + ]),
106 + format!("{}\n", TOKENS.parameter_end),
107 + )
108 +}
rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs new
+113
@@ -0,0 +1,113 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use expect_test::expect;
5 +use serde_json::json;
6 +use xgrammar_structural_tag::builders::StructuralTagOptions;
7 +use xgrammar_structural_tag::{
8 + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
9 +};
10 +
11 +use super::DeepSeekV41ToolParser;
12 +use crate::tool::test_utils::collect_stream;
13 +use crate::tool::{Tool, ToolParser};
14 +
15 +fn tools() -> Vec<Tool> {
16 + vec![Tool {
17 + name: "lookup".into(),
18 + description: None,
19 + parameters: json!({
20 + "type": "object",
21 + "properties": {
22 + "query": {"type": "string"},
23 + "limit": {"type": "integer"},
24 + "enabled": {"type": "boolean"},
25 + "data": {"type": "object"},
26 + "items": {"type": "array"},
27 + "label": {"type": "string"}
28 + }
29 + }),
30 + strict: None,
31 + }]
32 +}
33 +
34 +#[test]
35 +fn spaced_dsml_streaming_preserves_shared_schema_coercion() {
36 + let wire = concat!(
37 + "summary\n\n<｜DSML｜ calls>\n",
38 + "<｜DSML｜ invoke name=\"lookup\">\n",
39 + "<｜DSML｜ parameter name=\"query\" string=\"true\"> value\n</｜DSML｜ parameter>\n",
40 + "<｜DSML｜ parameter name=\"limit\" string=\"false\">2</｜DSML｜ parameter>\n",
41 + "<｜DSML｜ parameter name=\"enabled\" string=\"false\">true</｜DSML｜ parameter>\n",
42 + "<｜DSML｜ parameter name=\"data\" string=\"false\">{\"x\":1}</｜DSML｜ parameter>\n",
43 + "<｜DSML｜ parameter name=\"items\" string=\"false\">[1,2]</｜DSML｜ parameter>\n",
44 + "<｜DSML｜ parameter name=\"label\" string=\"false\">42</｜DSML｜ parameter>\n",
45 + "<｜DSML｜ parameter name=\"unknown\" string=\"false\">42</｜DSML｜ parameter>\n",
46 + "<｜DSML｜ parameter name=\"nullish\" string=\"false\">null</｜DSML｜ parameter>\n",
47 + "</｜DSML｜ invoke>\n",
48 + "<｜DSML｜ invoke name=\"lookup\">\n",
49 + "<｜DSML｜ parameter name=\"query\" string=\"true\">second</｜DSML｜ parameter>\n",
50 + "</｜DSML｜ invoke>\n</｜DSML｜ calls>",
51 + );
52 + let expected = [
53 + json!({"query":" value\n","limit":2,"enabled":true,"data":{"x":1},"items":[1,2],"label":"42","unknown":"42","nullish":null}),
54 + json!({"query":"second"}),
55 + ];
56 + for split in wire.char_indices().map(|(i, _)| i).chain(std::iter::once(wire.len())) {
57 + let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
58 + let output = collect_stream(parser.as_mut(), &[&wire[..split], &wire[split..]]);
59 + // Protocol framing follows the current shared DSML behavior.
60 + assert_eq!(output.normal_text(), "summary", "split {split}");
61 + assert_eq!(output.calls().len(), 2, "split {split}");
62 + for (index, call) in output.calls().iter().enumerate() {
63 + assert_eq!(call.tool_index, index, "split {split}");
64 + assert_eq!(call.name.as_deref(), Some("lookup"), "split {split}");
65 + assert_eq!(
66 + serde_json::from_str::<serde_json::Value>(&call.arguments).unwrap(),
67 + expected[index],
68 + "split {split}"
69 + );
70 + }
71 + }
72 +}
73 +
74 +#[test]
75 +fn v4_tags_remain_text_in_v41_dialect() {
76 + let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
77 + let output = collect_stream(parser.as_mut(), &["before<｜DSML｜tool_calls>after"]);
78 + expect!["before<｜DSML｜tool_calls>after"].assert_eq(&output.normal_text());
79 + assert!(output.calls().is_empty());
80 +}
81 +
82 +#[test]
83 +fn structural_tag_ignores_parameter_schemas_and_strict() {
84 + let parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
85 + let builder = parser.structural_tag_builder().unwrap();
86 + let options = StructuralTagOptions::default().with_reasoning(false);
87 + for choice in [
88 + ToolChoice::auto(),
89 + ToolChoice::required(),
90 + ToolChoice::function("lookup"),
91 + ] {
92 + let build = |function| {
93 + build_structural_tag(
94 + builder,
95 + &[ToolParam::Function(FunctionToolParam::new(function))],
96 + choice.clone(),
97 + options,
98 + )
99 + .unwrap()
100 + };
101 + let expected = build(FunctionDefinition::new("lookup"));
102 + for strict in [None, Some(false), Some(true)] {
103 + for parameters in [
104 + json!(false),
105 + json!({"type":"object", "properties":{"query":{"const":"fixed"}}, "required":["query"]}),
106 + ] {
107 + let mut function = FunctionDefinition::new("lookup").with_parameters(parameters);
108 + function.strict = strict;
109 + assert_eq!(build(function), expected);
110 + }
111 + }
112 + }
113 +}
rust/src/parser/src/tool/deepseek_dsml/mod.rs
+40 -16
@@ -14,14 +14,11 @@ use crate::tool::Tool;
14
15 mod deepseek_v32;
16 mod deepseek_v4;
17 +mod deepseek_v41;
18
19 pub use deepseek_v4::DeepSeekV4ToolParser;
20 pub use deepseek_v32::DeepSeekV32ToolParser;
20 -
21 -const INVOKE_START: &str = "<｜DSML｜invoke";
22 -const INVOKE_END: &str = "</｜DSML｜invoke>";
23 -const PARAMETER_START: &str = "<｜DSML｜parameter";
24 -const PARAMETER_END: &str = "</｜DSML｜parameter>";
21 +pub use deepseek_v41::DeepSeekV41ToolParser;
22
23 type DsmlInput<'i> = Partial<&'i str>;
24
@@ -30,6 +27,10 @@ struct DsmlTokens {
27 tool_calls_start: &'static str,
28 framed_tool_calls_start: &'static str,
29 tool_calls_end: &'static str,
30 + invoke_start: &'static str,
31 + invoke_end: &'static str,
32 + parameter_start: &'static str,
33 + parameter_end: &'static str,
34 }
35
36 impl DsmlTokens {
@@ -37,11 +38,25 @@ impl DsmlTokens {
38 tool_calls_start: "<｜DSML｜function_calls>",
39 framed_tool_calls_start: "\n\n<｜DSML｜function_calls>",
40 tool_calls_end: "</｜DSML｜function_calls>",
41 + invoke_start: "<｜DSML｜invoke",
42 + invoke_end: "</｜DSML｜invoke>",
43 + parameter_start: "<｜DSML｜parameter",
44 + parameter_end: "</｜DSML｜parameter>",
45 };
46 const V4: Self = Self {
47 tool_calls_start: "<｜DSML｜tool_calls>",
48 framed_tool_calls_start: "\n\n<｜DSML｜tool_calls>",
49 tool_calls_end: "</｜DSML｜tool_calls>",
50 + ..Self::V32
51 + };
52 + const V41: Self = Self {
53 + tool_calls_start: "<｜DSML｜ calls>",
54 + framed_tool_calls_start: "\n\n<｜DSML｜ calls>",
55 + tool_calls_end: "</｜DSML｜ calls>",
56 + invoke_start: "<｜DSML｜ invoke",
57 + invoke_end: "</｜DSML｜ invoke>",
58 + parameter_start: "<｜DSML｜ parameter",
59 + parameter_end: "</｜DSML｜ parameter>",
60 };
61 }
62
@@ -205,7 +220,7 @@ fn parse_tool_block_event(
220 ) -> ModalResult<DsmlEvent> {
221 ws0.void().parse_next(input)?;
222 alt((
208 - |input: &mut DsmlInput<'_>| invoke_event(input, invoke_end_scan),
223 + |input: &mut DsmlInput<'_>| invoke_event(input, tokens, invoke_end_scan),
224 |input: &mut DsmlInput<'_>| tool_calls_end_event(input, tokens),
225 ))
226 .parse_next(input)
@@ -243,19 +258,20 @@ fn safe_text_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult
258 /// Parse a DSML invoke block.
259 fn invoke_event(
260 input: &mut DsmlInput<'_>,
261 + tokens: DsmlTokens,
262 invoke_end_scan: &mut MarkerScanState,
263 ) -> ModalResult<DsmlEvent> {
264 let (name, body) = seq!(
249 - _: literal(INVOKE_START),
265 + _: literal(tokens.invoke_start),
266 _: ws1,
267 dsml_name_attr,
268 _: ws0,
269 _: ">",
254 - take_until_marker(INVOKE_END, invoke_end_scan),
255 - _: literal(INVOKE_END),
270 + take_until_marker(tokens.invoke_end, invoke_end_scan),
271 + _: literal(tokens.invoke_end),
272 )
273 .parse_next(input)?;
258 - let raw_params = parse_invoke_params(body)?;
274 + let raw_params = parse_invoke_params(body, tokens)?;
275 Ok(DsmlEvent::Invoke {
276 name: name.to_string(),
277 raw_params,
@@ -263,23 +279,31 @@ fn invoke_event(
279 }
280
281 /// Parse a DSML invoke body.
266 -fn parse_invoke_params(invoke_body: &str) -> ModalResult<Vec<DsmlParameter>> {
282 +fn parse_invoke_params(invoke_body: &str, tokens: DsmlTokens) -> ModalResult<Vec<DsmlParameter>> {
283 let mut input = invoke_body;
268 - delimited(ws0, repeat(0.., terminated(parse_parameter, ws0)), eof).parse_next(&mut input)
284 + delimited(
285 + ws0,
286 + repeat(
287 + 0..,
288 + terminated(|input: &mut &str| parse_parameter(input, tokens), ws0),
289 + ),
290 + eof,
291 + )
292 + .parse_next(&mut input)
293 }
294
295 /// Parse a DSML parameter block.
272 -fn parse_parameter(input: &mut &str) -> ModalResult<DsmlParameter> {
296 +fn parse_parameter(input: &mut &str, tokens: DsmlTokens) -> ModalResult<DsmlParameter> {
297 seq! {DsmlParameter {
274 - _: literal(PARAMETER_START),
298 + _: literal(tokens.parameter_start),
299 _: ws1,
300 name: name_attr.map(|name: &str| name.to_string()),
301 _: ws1,
302 is_string: string_attr.map(|value| value == "true"),
303 _: ws0,
304 _: ">",
281 - value: take_until(0.., PARAMETER_END).map(str::to_string),
282 - _: literal(PARAMETER_END),
305 + value: take_until(0.., tokens.parameter_end).map(str::to_string),
306 + _: literal(tokens.parameter_end),
307 }}
308 .parse_next(input)
309 }
rust/src/parser/src/tool/mod.rs
+1 -1
@@ -20,7 +20,7 @@ mod seed_oss;
20 pub mod test_utils;
21 use std::collections::{BTreeMap, btree_map};
22
23 -pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser};
23 +pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser, DeepSeekV41ToolParser};
24 pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser};
25 pub use error::{Result, ToolParserError};
26 pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser};
rust/src/server/src/error.rs
+13
@@ -167,6 +167,19 @@ mod tests {
167 assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
168 }
169
170 + #[test]
171 + fn invalid_reasoning_effort_maps_to_invalid_request() {
172 + let error = vllm_chat::Error::InvalidReasoningEffort(
173 + "DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(),
174 + );
175 + let api_error = chat_submit_error("failed to submit chat request", error);
176 + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
177 + assert_eq!(
178 + api_error.to_error_response().error.error_type,
179 + "invalid_request_error"
180 + );
181 + }
182 +
183 #[test]
184 fn llm_wrapped_empty_prompt_maps_to_invalid_request() {
185 let error = vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds {
tests/parser/engine/test_deepseek_v41.py new
+146
@@ -0,0 +1,146 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +
6 +import pytest
7 +
8 +from tests.parser.engine.replay_harness import (
9 + DUMMY_TOOLS,
10 + MockTokenizer,
11 + _test_request,
12 + collect_output,
13 + replay_streaming,
14 +)
15 +from vllm.parser.deepseek_v41 import deepseek_v41_config
16 +from vllm.parser.parser_manager import ParserManager
17 +
18 +CALLS = (
19 + '\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name="get_weather">\n'
20 + '<｜DSML｜ parameter name="city" string="true">杭州</｜DSML｜ parameter>\n'
21 + '<｜DSML｜ parameter name="count" string="false">42</｜DSML｜ parameter>\n'
22 + '</｜DSML｜ invoke>\n<｜DSML｜ invoke name="add">\n'
23 + '<｜DSML｜ parameter name="x" string="false">1.5</｜DSML｜ parameter>\n'
24 + '<｜DSML｜ parameter name="y" string="false">2.25</｜DSML｜ parameter>\n'
25 + "</｜DSML｜ invoke>\n</｜DSML｜ calls>"
26 +)
27 +
28 +
29 +def tokenizer_for(text, special_calls):
30 + vocab = {"<think>": 50, "</think>": 51}
31 + if special_calls:
32 + vocab |= {"<｜DSML｜ calls>": 52, "</｜DSML｜ calls>": 53}
33 + tokens: list[tuple[int, str]] = []
34 + while text:
35 + special = next((marker for marker in vocab if text.startswith(marker)), None)
36 + piece = special or text[0]
37 + tokens.append((vocab[piece] if special else 100 + len(tokens), piece))
38 + text = text[len(piece) :]
39 + return MockTokenizer(vocab, tokens), tokens
40 +
41 +
42 +def parser_for(tokenizer, controls):
43 + cls = ParserManager.get_parser(
44 + tool_parser_name="deepseek_v41",
45 + reasoning_parser_name="deepseek_v41",
46 + enable_auto_tools=True,
47 + )
48 + return cls(tokenizer, chat_template_kwargs=controls)
49 +
50 +
51 +@pytest.mark.parametrize("thinking", [False, True])
52 +@pytest.mark.parametrize("special_calls", [False, True])
53 +@pytest.mark.parametrize("chunk_size", [1, 7, 10000])
54 +def test_registered_adapters_parse_parallel_calls_across_chunks(
55 + thinking,
56 + special_calls,
57 + chunk_size,
58 +):
59 + text = ("Plan.</think>" if thinking else "") + "Checking." + CALLS
60 + tokenizer, tokens = tokenizer_for(text, special_calls)
61 + parser = parser_for(tokenizer, {"thinking": thinking})
62 + output = collect_output(
63 + replay_streaming(
64 + parser,
65 + tokens,
66 + chunk_size=chunk_size,
67 + finished_on_last=True,
68 + tools=DUMMY_TOOLS,
69 + prompt_token_ids=[50 if thinking else 51],
70 + )
71 + )
72 + assert output.reasoning == ("Plan." if thinking else "")
73 + assert output.content.strip() == "Checking."
74 + assert [call["name"] for call in output.tool_calls] == ["get_weather", "add"]
75 + assert [json.loads(call["arguments"]) for call in output.tool_calls] == [
76 + {"city": "杭州", "count": 42},
77 + {"x": 1.5, "y": 2.25},
78 + ]
79 +
80 +
81 +@pytest.mark.parametrize("thinking", [False, True])
82 +def test_registered_adapters_parse_complete_output(thinking):
83 + text = ("Plan.</think>" if thinking else "") + CALLS
84 + tokenizer, tokens = tokenizer_for(text, False)
85 + parser = parser_for(tokenizer, {"thinking": thinking})
86 + reasoning, content, calls = parser.parse(
87 + text,
88 + _test_request(tools=DUMMY_TOOLS),
89 + enable_auto_tools=True,
90 + model_output_token_ids=[tid for tid, _ in tokens],
91 + )
92 + assert (reasoning or "") == ("Plan." if thinking else "")
93 + assert not (content or "").strip()
94 + assert [call.name for call in calls] == ["get_weather", "add"]
95 + assert json.loads(calls[0].arguments) == {"city": "杭州", "count": 42}
96 +
97 +
98 +@pytest.mark.parametrize(
99 + ("controls", "text", "expected_reasoning", "reasoning_tokens"),
100 + [
101 + ({"thinking": False}, "12", "", 0),
102 + ({"enable_thinking": False}, "12", "", 0),
103 + ({"thinking": True, "reasoning_effort": "none"}, "12", "", 0),
104 + ({}, "</think>12", "", 0),
105 + ({}, "Plan.</think>12", "Plan.", 5),
106 + ],
107 +)
108 +def test_reasoning_adapter_controls_and_usage(
109 + controls,
110 + text,
111 + expected_reasoning,
112 + reasoning_tokens,
113 +):
114 + tokenizer, tokens = tokenizer_for(text, False)
115 + parser = parser_for(tokenizer, controls)
116 + output = collect_output(
117 + replay_streaming(
118 + parser,
119 + tokens,
120 + chunk_size=1,
121 + finished_on_last=True,
122 + )
123 + )
124 + assert output.reasoning == expected_reasoning
125 + assert output.content == "12"
126 + assert (
127 + parser.reasoning_parser.count_reasoning_tokens([tid for tid, _ in tokens])
128 + == reasoning_tokens
129 + )
130 + assert parser.is_reasoning_end([50, 100, 51])
131 + assert not parser.is_reasoning_end([51, 100, 50])
132 +
133 +
134 +def test_python_argument_conversion_and_partial_values():
135 + converter = deepseek_v41_config().arg_converter
136 + raw = (
137 + '<｜DSML｜ parameter name="object" string="false">'
138 + '{"a": [true, null]}</｜DSML｜ parameter>'
139 + '<｜DSML｜ parameter name="bad" string="false">[broken</｜DSML｜ parameter>'
140 + '<｜DSML｜ parameter name="text" string="true"> a<b'
141 + )
142 + assert json.loads(converter(raw, True)) == {
143 + "object": {"a": [True, None]},
144 + "bad": "[broken",
145 + "text": " a<b",
146 + }
tests/tokenizers_/fixtures/deepseek_v41/test_output_1.txt new
+38
@@ -0,0 +1,38 @@
1 +<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
2 +
3 +You are a helpful assistant.
4 +
5 +## Tools
6 +
7 +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜ calls>" block like the following:
8 +
9 +<｜DSML｜ calls>
10 +<｜DSML｜ invoke name="$TOOL_NAME">
11 +<｜DSML｜ parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜ parameter>
12 +...
13 +</｜DSML｜ invoke>
14 +<｜DSML｜ invoke name="$TOOL_NAME2">
15 +...
16 +</｜DSML｜ invoke>
17 +</｜DSML｜ calls>
18 +
19 +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
20 +
21 +If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
22 +
23 +Otherwise, output directly after </think> with tool calls or final response.
24 +
25 +### Available Tool Schemas
26 +
27 +{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
28 +{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
29 +
30 +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
31 +<｜User｜>What's the weather in Beijing?<｜Assistant｜><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
32 +
33 +<｜DSML｜ calls>
34 +<｜DSML｜ invoke name="get_weather">
35 +<｜DSML｜ parameter name="location" string="true">Beijing</｜DSML｜ parameter>
36 +<｜DSML｜ parameter name="unit" string="true">celsius</｜DSML｜ parameter>
37 +</｜DSML｜ invoke>
38 +</｜DSML｜ calls><｜end▁of▁sentence｜><｜User｜><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><｜Assistant｜><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<｜end▁of▁sentence｜>
\ No newline at end of file
tests/tokenizers_/fixtures/deepseek_v41/test_output_2.txt new
+3
@@ -0,0 +1,3 @@
1 +<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
2 +
3 +You are a helpful assistant.<｜User｜>Hello<｜Assistant｜></think>Hi there! How can I help you?<｜end▁of▁sentence｜><｜User｜>What is the capital of France?<｜Assistant｜><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<｜end▁of▁sentence｜>
\ No newline at end of file
tests/tokenizers_/test_deepseek_v41.py new
+169
@@ -0,0 +1,169 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import copy
5 +import json
6 +from pathlib import Path
7 +
8 +import pytest
9 +
10 +from tests.tokenizers_.test_deepseek_v4 import FakeHfTokenizer
11 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
12 +from vllm.tokenizers.deepseek_v41 import get_deepseek_v41_tokenizer
13 +
14 +FIXTURES = Path(__file__).parent / "fixtures"
15 +
16 +
17 +def render(messages, **kwargs):
18 + return get_deepseek_v41_tokenizer(FakeHfTokenizer()).apply_chat_template(
19 + messages, tokenize=False, **kwargs
20 + )
21 +
22 +
23 +@pytest.mark.parametrize("case_id", [1, 2])
24 +def test_reference_encoder_fixtures(case_id):
25 + # Expected prompts come from the DeepSeek V4.1 reference encoder.
26 + data = json.loads(
27 + (FIXTURES / "deepseek_v4" / f"test_input_{case_id}.json").read_text()
28 + )
29 + messages = data["messages"] if isinstance(data, dict) else data
30 + tools = data.get("tools") if isinstance(data, dict) else None
31 + expected = (FIXTURES / "deepseek_v41" / f"test_output_{case_id}.txt").read_text()
32 + assert render(messages, tools=tools) == expected
33 +
34 +
35 +@pytest.mark.parametrize(
36 + ("effort", "budget"),
37 + [
38 + (None, 50),
39 + ("low", 25),
40 + ("high", 50),
41 + ("xhigh", 75),
42 + ("max", 100),
43 + (1, 1),
44 + (42, 42),
45 + (100, 100),
46 + ],
47 +)
48 +def test_numeric_reasoning_effort(effort, budget):
49 + assert render(
50 + [{"role": "user", "content": "question"}], reasoning_effort=effort
51 + ) == (
52 + "<｜begin▁of▁sentence｜><｜System｜>"
53 + f"Reasoning Effort: {budget} (range 1-100, the higher the value, "
54 + "the more thorough the reasoning)\n\n"
55 + "<｜User｜>question<｜Assistant｜><think>"
56 + )
57 +
58 +
59 +@pytest.mark.parametrize("effort", ["medium", "minimal", -1, 0, 101, True, 1.5, []])
60 +def test_invalid_effort_is_a_request_error(effort):
61 + with pytest.raises(ValueError, match="reasoning_effort"):
62 + render([{"role": "user", "content": "question"}], reasoning_effort=effort)
63 +
64 +
65 +@pytest.mark.parametrize(
66 + "controls",
67 + [
68 + {"thinking": False},
69 + {"enable_thinking": False},
70 + {"thinking": True, "reasoning_effort": "none"},
71 + {"enable_thinking": True, "reasoning_effort": "none"},
72 + ],
73 +)
74 +def test_chat_mode_has_closed_thinking_prefix(controls):
75 + assert render([{"role": "user", "content": "question"}], **controls) == (
76 + "<｜begin▁of▁sentence｜><｜User｜>question<｜Assistant｜></think>"
77 + )
78 +
79 +
80 +def test_raw_text_parts_preserve_reference_separator():
81 + messages = [
82 + {
83 + "role": "user",
84 + "content": [
85 + {"type": "text", "text": "first"},
86 + {"type": "text", "text": "second"},
87 + ],
88 + }
89 + ]
90 + original = copy.deepcopy(messages)
91 + assert (
92 + render(
93 + messages,
94 + conversation=[{"role": "user", "content": "first\nsecond"}],
95 + thinking=False,
96 + )
97 + == "<｜begin▁of▁sentence｜><｜User｜>first\n\nsecond<｜Assistant｜></think>"
98 + )
99 + assert messages == original
100 +
101 +
102 +def test_mid_system_gets_its_own_marker_and_generation_header():
103 + assert render(
104 + [
105 + {"role": "user", "content": "question"},
106 + {"role": "system", "content": "answer briefly"},
107 + ],
108 + thinking=False,
109 + ) == (
110 + "<｜begin▁of▁sentence｜><｜User｜>question"
111 + "<｜System｜>answer briefly<｜Assistant｜></think>"
112 + )
113 +
114 +
115 +def test_top_level_effort_overrides_template_effort():
116 + request = ChatCompletionRequest(
117 + model="deepseek-ai/DeepSeek-V4.1-Flash",
118 + messages=[{"role": "user", "content": "question"}],
119 + reasoning_effort="low",
120 + chat_template_kwargs={"reasoning_effort": 100},
121 + )
122 + kwargs = request.build_chat_params(None, "auto").get_apply_chat_template_kwargs()
123 + assert "Reasoning Effort: 25 " in render(request.messages, **kwargs)
124 +
125 +
126 +def test_encode_uses_one_bos_and_forwards_truncation():
127 + tokenizer = get_deepseek_v41_tokenizer(FakeHfTokenizer())
128 + tokenizer.apply_chat_template(
129 + [{"role": "user", "content": "question"}],
130 + thinking=False,
131 + truncation=True,
132 + max_length=16,
133 + )
134 + text, add_special_tokens, kwargs = tokenizer.last_encode
135 + assert text.startswith("<｜begin▁of▁sentence｜>")
136 + assert add_special_tokens is False
137 + assert kwargs == {"truncation": True, "max_length": 16}
138 +
139 +
140 +@pytest.mark.parametrize("image_type", ["image_url", "input_image", "image_pil"])
141 +def test_images_preserve_content_order_and_reference_separator(image_type):
142 + from PIL import Image
143 +
144 + def image_part(color):
145 + if image_type == "image_pil":
146 + return {"type": image_type, "image_pil": Image.new("RGB", (2, 2), color)}
147 + url = f"https://example.com/{color}.png"
148 + return {
149 + "type": image_type,
150 + "image_url": {"url": url} if image_type == "image_url" else url,
151 + }
152 +
153 + parts = [
154 + {"type": "text", "text": "first"},
155 + image_part("red"),
156 + {"type": "text", "text": "second"},
157 + image_part("blue"),
158 + {"type": "text", "text": "compare"},
159 + ]
160 + assert render([{"role": "user", "content": parts}], thinking=False) == (
161 + "<｜begin▁of▁sentence｜><｜User｜>first\n\n<｜deepseek_image｜>\n\n"
162 + "second\n\n<｜deepseek_image｜>\n\ncompare<｜Assistant｜></think>"
163 + )
164 + assert parts[1]["type"] == parts[3]["type"] == image_type
165 +
166 +
167 +def test_unsupported_media_is_a_request_error():
168 + with pytest.raises(ValueError, match="text and image content only"):
169 + render([{"role": "user", "content": [{"type": "input_audio"}]}])
tests/tool_parsers/test_structural_tag_registry.py
+61
@@ -80,6 +80,67 @@ def test_supported_structural_tag_models_include_vllm_builtins():
80 assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
81
82
83 +@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
84 +def test_deepseek_v41_format_constraints_ignore_parameter_schema(
85 + choice,
86 + sample_tools_strict,
87 +):
88 + tool_choice = (
89 + ChatCompletionNamedToolChoiceParam(
90 + function=ChatCompletionNamedFunction(name=choice)
91 + )
92 + if choice == "get_weather"
93 + else choice
94 + )
95 + tag = get_model_structural_tag(
96 + "deepseek_v41", sample_tools_strict, tool_choice, reasoning=False
97 + )
98 + grammar = Grammar.from_structural_tag(tag)
99 + begin = '\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name="get_weather">\n'
100 + end = "</｜DSML｜ invoke>\n</｜DSML｜ calls>"
101 + parameter = (
102 + '<｜DSML｜ parameter name="undeclared" string="false">'
103 + '{"nested": [true, null, 1.5]}</｜DSML｜ parameter>\n'
104 + )
105 + # Required city is omitted; extra and repeated fields remain unconstrained.
106 + assert _is_grammar_accept_string(grammar, begin + end)
107 + assert _is_grammar_accept_string(grammar, begin + parameter * 2 + end)
108 + assert not _is_grammar_accept_string(
109 + grammar,
110 + begin + parameter.replace('{"nested": [true, null, 1.5]}', "invalid") + end,
111 + )
112 + assert not _is_grammar_accept_string(
113 + grammar, (begin + end).replace('name="get_weather"', 'name="unknown"')
114 + )
115 + assert not _is_grammar_accept_string(
116 + grammar, begin + parameter.replace("｜ parameter", "｜parameter") + end
117 + )
118 + assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
119 +
120 +
121 +def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
122 + tag = get_model_structural_tag(
123 + "deepseek_v41",
124 + sample_tools,
125 + ChatCompletionNamedToolChoiceParam(
126 + function=ChatCompletionNamedFunction(name="get_weather")
127 + ),
128 + reasoning=False,
129 + )
130 + grammar = Grammar.from_structural_tag(tag)
131 + call = '<｜DSML｜ invoke name="get_weather">\n</｜DSML｜ invoke>\n'
132 + assert _is_grammar_accept_string(
133 + grammar, "\n\n<｜DSML｜ calls>\n" + call + "</｜DSML｜ calls>"
134 + )
135 + assert not _is_grammar_accept_string(
136 + grammar, "\n\n<｜DSML｜ calls>\n" + call * 2 + "</｜DSML｜ calls>"
137 + )
138 + assert (
139 + get_model_structural_tag("deepseek_v41", sample_tools, "auto", reasoning=False)
140 + is None
141 + )
142 +
143 +
144 @pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
145 def test_get_model_structural_tag_supports_all_xgrammar_builtins(
146 model: str,
vllm/config/model.py
+3
@@ -152,6 +152,7 @@ class ModelConfig:
152 - "mistral" will always use the tokenizer from `mistral_common`.
153 - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
154 - "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
155 + - "deepseek_v41" will use the DeepSeek V4.1 prompt encoder.
156 - "kimi_k3" will always use the "hf" tokenizer but render chat prompts
157 with Kimi K3's Python XTML encoding instead of a Jinja template.
158 - "cohere" uses the standard HF tokenizer but renders the chat template
@@ -705,6 +706,8 @@ class ModelConfig:
706 "DeepseekV4ForConditionalGeneration",
707 ):
708 self.tokenizer_mode = "deepseek_v4"
709 + elif arch == "DeepseekV41ForCausalLM":
710 + self.tokenizer_mode = "deepseek_v41"
711 elif arch in ("InklingForCausalLM", "InklingForConditionalGeneration"):
712 self.tokenizer_mode = "inkling"
713
vllm/models/deepseek_v4_1/common/mm_preprocess.py new
+463
@@ -0,0 +1,463 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Multimodal preprocessing for the DeepSeek-V4.1 vision variant.
4 +
5 +The image transform and image-span construction are ported from the
6 +official repository's ``image_processor.py`` so that token counts bit-match
7 +the reference. Each ``<｜deepseek_image｜>`` placeholder in the prompt expands
8 +to ``[IMAGE_START] + ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h +
9 +[IMAGE_END]``; every span position carries ``image_token_id`` (129264) in
10 +``input_ids`` and the roles ride along in a per-image ``types`` tensor
11 +(the reference's out-of-band ``token_types``). IMAGE slots receive aligner
12 +rows in reading order; the delimiters take the learned ``image_start`` /
13 +``image_newline`` / ``image_end`` vectors.
14 +
15 +One vLLM-side deviation from the reference token stream: a leading
16 +compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start % COMPRESS_PAD_TO``
17 +positions, so the span always starts at the same compressor phase) is
18 +prepended when the block is spliced into the final prompt. Pad positions
19 +borrow the reserved in-vocab token ``<|place_holder_mm_span_0436|>`` so
20 +they stay distinguishable from real span positions; they embed as the
21 +plain image token (v4.1 has no ``image_pad`` vector) and are routed and
22 +engram-deadened like image tokens.
23 +"""
24 +
25 +import math
26 +from collections.abc import Mapping, Sequence
27 +from typing import Any, cast
28 +
29 +import numpy as np
30 +import torch
31 +from PIL import Image, ImageOps
32 +from transformers import BatchFeature
33 +from typing_extensions import assert_never
34 +
35 +from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
36 +from vllm.inputs import MultiModalDataDict
37 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
38 +from vllm.multimodal.parse import ImageSize, MultiModalDataItems
39 +from vllm.multimodal.processing import (
40 + BaseDummyInputsBuilder,
41 + BaseMultiModalProcessor,
42 + BaseProcessingInfo,
43 + PromptReplacement,
44 + PromptUpdate,
45 + PromptUpdateDetails,
46 +)
47 +from vllm.multimodal.processing.processor import (
48 + MultiModalPromptUpdates,
49 + MultiModalPromptUpdatesApplyResult,
50 + PlaceholderFeaturesInfo,
51 + UpdateMode,
52 + _plan_prompt_updates,
53 +)
54 +from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
55 +
56 +IMAGE_START, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(4)
57 +# Image spans are padded so they always start at the same compressor group
58 +# phase; v4.1 uses ratio-2 compressors (v4.0 used 4).
59 +COMPRESS_PAD_TO = 2
60 +
61 +IMAGE_PLACEHOLDER = "<｜deepseek_image｜>"
62 +
63 +# The checkpoint tokenizer's id for IMAGE_PLACEHOLDER (the config's
64 +# ``image_token_id``). Every image-span position carries this id, exactly
65 +# like the reference; the MoE router detects image tokens as the range
66 +# [IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID + 5), which also covers
67 +# IMAGE_PAD_ID below.
68 +IMAGE_SENTINEL_BASE_ID = 129264
69 +# Leading compressor-alignment pads borrow this reserved in-vocab token
70 +# (``<|place_holder_mm_span_0436|>``) so they can be told apart from real
71 +# span positions when the per-position role is decided from token ids.
72 +IMAGE_PAD_ID = 129265
73 +IMAGE_PAD_TOKEN_NAME = "<|place_holder_mm_span_0436|>"
74 +
75 +
76 +def image_sentinel_mask(token_ids: torch.Tensor) -> torch.Tensor:
77 + """Boolean mask for image-span positions (span tokens and align pads)."""
78 + return (token_ids == IMAGE_SENTINEL_BASE_ID) | (token_ids == IMAGE_PAD_ID)
79 +
80 +
81 +def validate_image_sentinel_ids(tokenizer) -> None:
82 + """Check the image/pad token ids against the tokenizer."""
83 + image_id = tokenizer.convert_tokens_to_ids(IMAGE_PLACEHOLDER)
84 + if image_id != IMAGE_SENTINEL_BASE_ID:
85 + raise ValueError(
86 + f"Image placeholder {IMAGE_PLACEHOLDER!r} has id {image_id}, "
87 + f"expected {IMAGE_SENTINEL_BASE_ID} (the config's "
88 + "image_token_id); the DeepSeek-V4.1 vision path keys image "
89 + "routing and engram masking off this id."
90 + )
91 + pad_id = tokenizer.convert_tokens_to_ids(IMAGE_PAD_TOKEN_NAME)
92 + if pad_id != IMAGE_PAD_ID:
93 + raise ValueError(
94 + f"Image pad token {IMAGE_PAD_TOKEN_NAME!r} has id {pad_id}, "
95 + f"expected {IMAGE_PAD_ID}; the DeepSeek-V4.1 vision path "
96 + "borrows this reserved id for compressor-alignment pads."
97 + )
98 +
99 +
100 +def llm_grid(best_height, best_width, patch_size, downsample_ratio):
101 + """Token grid the aligner produces from a patch grid of this pixel size."""
102 + return (
103 + math.ceil((best_height // patch_size) / downsample_ratio),
104 + math.ceil((best_width // patch_size) / downsample_ratio),
105 + )
106 +
107 +
108 +def num_image_tokens(n_llm_h: int, n_llm_w: int) -> int:
109 + return n_llm_h * (n_llm_w + 1) + 2
110 +
111 +
112 +def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token):
113 + """Largest aspect-preserving pixel size whose token grid still fits in
114 + max_n_token. Returns (best_height, best_width)."""
115 + r = height / width
116 + max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5
117 + max_h_float = max_w_float * r
118 + cell = patch_size * downsample_ratio
119 + if max_w_float < 1.0: # very tall: collapse to a single column
120 + return (max_n_token - 2) // 2 * cell, cell
121 + if max_h_float < 1.0: # very wide: collapse to a single row
122 + return cell, (max_n_token - 3) * cell
123 + beta = min(
124 + math.floor(max_w_float) * cell / width,
125 + math.floor(max_h_float) * cell / height,
126 + )
127 + return (
128 + math.floor(height * beta / patch_size) * patch_size,
129 + math.floor(width * beta / patch_size) * patch_size,
130 + )
131 +
132 +
133 +def safe_resize(
134 + height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token
135 +):
136 + """Shrink the pixel size until the image costs at most max_n_token LLM
137 + tokens (minus the reservation for the compressor-alignment pad)."""
138 + max_n_token -= COMPRESS_PAD_TO - 1
139 + n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio)
140 + if num_image_tokens(n_llm_h, n_llm_w) > max_n_token:
141 + best_height, best_width = solve_resize_ratio(
142 + height, width, patch_size, downsample_ratio, max_n_token
143 + )
144 + n_llm_h, n_llm_w = llm_grid(
145 + best_height, best_width, patch_size, downsample_ratio
146 + )
147 + assert num_image_tokens(n_llm_h, n_llm_w) <= max_n_token
148 + return n_llm_h, n_llm_w, best_height, best_width
149 +
150 +
151 +def load_image(
152 + image: Image.Image,
153 + *,
154 + patch_size: int,
155 + downsample_ratio: int,
156 + max_n_token: int,
157 + min_pixels: int,
158 + max_wh_ratio: float | None,
159 +):
160 + """Transform one PIL image into ViT patches.
161 +
162 + Same math as the reference ``load_image``, except the image is already
163 + decoded (vLLM supplies PIL images instead of a record dict).
164 + """
165 + p = patch_size
166 + image = image.convert("RGB")
167 + width, height = image.size
168 + if max_wh_ratio is not None and width > height * max_wh_ratio:
169 + width = height * max_wh_ratio
170 + if 0 < width * height < min_pixels:
171 + ratio = (min_pixels / (width * height)) ** 0.5
172 + width = int(width * ratio)
173 + height = int(height * ratio)
174 + best_width = math.ceil(width / p) * p
175 + best_height = math.ceil(height / p) * p
176 + n_llm_h, n_llm_w, best_height, best_width = safe_resize(
177 + height, width, best_height, best_width, p, downsample_ratio, max_n_token
178 + )
179 + n_vit_h, n_vit_w = best_height // p, best_width // p
180 + if max_wh_ratio is not None and image.width >= max_wh_ratio * image.height:
181 + image = image.resize((best_width, best_height))
182 + else:
183 + image = ImageOps.pad(image, (best_width, best_height), color=(127, 127, 127))
184 + x = torch.from_numpy(np.asarray(image, dtype=np.float32)).permute(2, 0, 1) / 255
185 + x = ((x - 0.5) / 0.5).to(torch.bfloat16)
186 + patches = (
187 + x.reshape(3, n_vit_h, p, n_vit_w, p)
188 + .permute(1, 3, 0, 2, 4)
189 + .reshape(n_vit_h * n_vit_w, 3, p, p)
190 + )
191 + return patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w
192 +
193 +
194 +def image_token_types(n_llm_h: int, n_llm_w: int) -> torch.Tensor:
195 + """Reading-order span layout: one IMAGE_NEW_LINE per row."""
196 + types = [IMAGE_START]
197 + types += ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h
198 + types.append(IMAGE_END)
199 + return torch.tensor(types, dtype=torch.int64)
200 +
201 +
202 +class DeepseekV4VLImageProcessor:
203 + """Per-image transform (the PIL-input equivalent of the reference
204 + ``load_image``)."""
205 +
206 + def __init__(self, config: DeepseekV41Config) -> None:
207 + super().__init__()
208 + self.patch_size = config.vision_patch_size
209 + self.downsample_ratio = config.vision_downsample_ratio
210 + self.max_n_token = config.vision_max_n_token
211 + self.min_pixels = config.vision_min_pixels
212 + self.max_wh_ratio = config.vision_max_wh_ratio
213 +
214 + def __call__(self, image: Image.Image):
215 + return load_image(
216 + image,
217 + patch_size=self.patch_size,
218 + downsample_ratio=self.downsample_ratio,
219 + max_n_token=self.max_n_token,
220 + min_pixels=self.min_pixels,
221 + max_wh_ratio=self.max_wh_ratio,
222 + )
223 +
224 +
225 +class DeepseekV4VLProcessor:
226 + """Minimal stand-in for the HF processor of DeepSeek-V4.1 vision models.
227 +
228 + The official repository ships image preprocessing as plain functions in
229 + ``image_processor.py`` (no ``auto_map`` processor), so this class wraps
230 + their ports directly and the model loads without ``--trust-remote-code``.
231 +
232 + ``__call__`` returns a ``BatchFeature`` with one entry per image
233 + (flattened across images):
234 +
235 + - ``patches``: ``(sum(n_vit_h * n_vit_w), 3, p, p)`` bf16 ViT patches.
236 + - ``vit_grid``: ``(num_images, 2)`` int64 ``[n_vit_h, n_vit_w]``.
237 + - ``llm_grid``: ``(num_images, 2)`` int64 ``[n_llm_h, n_llm_w]``.
238 + - ``types``: concatenated per-image pad-free span roles
239 + (IMAGE_START/IMAGE/IMAGE_NEW_LINE/IMAGE_END); every span position
240 + carries ``image_token_id`` in the prompt's token ids.
241 + """
242 +
243 + def __init__(self, config: DeepseekV41Config) -> None:
244 + super().__init__()
245 + self.config = config
246 + self.image_processor = DeepseekV4VLImageProcessor(config)
247 +
248 + def __call__(
249 + self,
250 + text: str | None = None,
251 + images: Sequence[Image.Image] | None = None,
252 + return_tensors: str | None = None,
253 + **kwargs: Any,
254 + ) -> BatchFeature:
255 + patches_list = []
256 + vit_grid = []
257 + llm_grid_list = []
258 + types_list = []
259 + for image in images or []:
260 + patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = self.image_processor(image)
261 + patches_list.append(patches)
262 + vit_grid.append((n_vit_h, n_vit_w))
263 + llm_grid_list.append((n_llm_h, n_llm_w))
264 + types_list.append(image_token_types(n_llm_h, n_llm_w))
265 +
266 + if not patches_list:
267 + return BatchFeature({})
268 +
269 + return BatchFeature(
270 + {
271 + "patches": torch.cat(patches_list),
272 + "vit_grid": torch.tensor(vit_grid, dtype=torch.int64),
273 + "llm_grid": torch.tensor(llm_grid_list, dtype=torch.int64),
274 + "types": torch.cat(types_list),
275 + }
276 + )
277 +
278 +
279 +class DeepseekV4VLProcessingInfo(BaseProcessingInfo):
280 + def get_hf_config(self) -> DeepseekV41Config:
281 + return self.ctx.get_hf_config(DeepseekV41Config)
282 +
283 + def get_hf_processor(self, **kwargs: object) -> DeepseekV4VLProcessor:
284 + if kwargs:
285 + raise ValueError(f"Unexpected processor kwargs: {sorted(kwargs)}")
286 + return DeepseekV4VLProcessor(self.get_hf_config())
287 +
288 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
289 + return {"image": None}
290 +
291 + def get_mm_max_tokens_per_item(
292 + self,
293 + seq_len: int,
294 + mm_counts: Mapping[str, int],
295 + ) -> Mapping[str, int]:
296 + # ``safe_resize`` reserves COMPRESS_PAD_TO - 1 tokens of
297 + # vision_max_n_token for the compressor-alignment pad, so the full
298 + # image span is bounded by vision_max_n_token; the margin is kept
299 + # in case that reservation changes.
300 + return {
301 + "image": self.get_hf_config().vision_max_n_token + COMPRESS_PAD_TO - 1,
302 + }
303 +
304 + def get_image_placeholder_token_id(self) -> int:
305 + token_id = self.get_tokenizer().convert_tokens_to_ids(IMAGE_PLACEHOLDER)
306 + if token_id is None:
307 + raise ValueError(f"Token not found in tokenizer: {IMAGE_PLACEHOLDER}")
308 + return token_id
309 +
310 + def get_image_size_with_most_features(self) -> ImageSize:
311 + hf_config = self.get_hf_config()
312 + patch_size = hf_config.vision_patch_size
313 + downsample_ratio = hf_config.vision_downsample_ratio
314 + # A square maximizes the ViT patch count (area) within the token
315 + # budget; solve the budget-derived size directly to keep the dummy
316 + # image small.
317 + budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1)
318 + side = budget * patch_size * downsample_ratio
319 + best_h, best_w = solve_resize_ratio(
320 + side, side, patch_size, downsample_ratio, budget
321 + )
322 + return ImageSize(width=best_w, height=best_h)
323 +
324 +
325 +class DeepseekV4VLDummyInputsBuilder(
326 + BaseDummyInputsBuilder[DeepseekV4VLProcessingInfo]
327 +):
328 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
329 + return IMAGE_PLACEHOLDER * mm_counts.get("image", 0)
330 +
331 + def get_dummy_mm_data(
332 + self,
333 + seq_len: int,
334 + mm_counts: Mapping[str, int],
335 + mm_options: Mapping[str, BaseDummyOptions],
336 + ) -> MultiModalDataDict:
337 + size = self.info.get_image_size_with_most_features()
338 + return {
339 + "image": self._get_dummy_images(
340 + width=size.width,
341 + height=size.height,
342 + num_images=mm_counts.get("image", 0),
343 + overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
344 + ),
345 + }
346 +
347 +
348 +class DeepseekV4VLMultiModalProcessor(
349 + BaseMultiModalProcessor[DeepseekV4VLProcessingInfo]
350 +):
351 + def _get_mm_fields_config(
352 + self,
353 + hf_inputs: BatchFeature,
354 + hf_processor_mm_kwargs: Mapping[str, object],
355 + ) -> Mapping[str, MultiModalFieldConfig]:
356 + vit_grid = hf_inputs.get("vit_grid")
357 + llm_grid = hf_inputs.get("llm_grid")
358 +
359 + if vit_grid is None or llm_grid is None:
360 + empty = torch.empty(0, dtype=torch.long)
361 + patch_sizes = types_sizes = empty
362 + else:
363 + patch_sizes = vit_grid.prod(-1)
364 + n_llm_h, n_llm_w = llm_grid[:, 0], llm_grid[:, 1]
365 + types_sizes = n_llm_h * (n_llm_w + 1) + 2
366 +
367 + return {
368 + "patches": MultiModalFieldConfig.flat_from_sizes("image", patch_sizes),
369 + "vit_grid": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
370 + "llm_grid": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
371 + "types": MultiModalFieldConfig.flat_from_sizes(
372 + "image", types_sizes, keep_on_cpu=True
373 + ),
374 + }
375 +
376 + def _get_prompt_updates(
377 + self,
378 + mm_items: MultiModalDataItems,
379 + hf_processor_mm_kwargs: Mapping[str, object],
380 + out_mm_kwargs: MultiModalKwargsItems,
381 + ) -> Sequence[PromptUpdate]:
382 + image_token_id = self.info.get_image_placeholder_token_id()
383 + validate_image_sentinel_ids(self.info.get_tokenizer())
384 +
385 + def get_image_replacement(item_idx: int) -> PromptUpdateDetails:
386 + types: torch.Tensor = out_mm_kwargs["image"][item_idx]["types"].data
387 + # Every span position carries image_token_id; the roles live in
388 + # ``types``. All of them are embed positions (delimiters get the
389 + # learned vectors from embed_multimodal, not the embed table).
390 + full = [image_token_id] * types.numel()
391 + return PromptUpdateDetails.select_token_id(full, image_token_id)
392 +
393 + return [
394 + PromptReplacement(
395 + modality="image",
396 + target=[image_token_id],
397 + replacement=get_image_replacement,
398 + ),
399 + ]
400 +
401 + def _apply_token_matches_with_placeholders(
402 + self,
403 + token_ids: list[int],
404 + mm_prompt_updates: MultiModalPromptUpdates,
405 + ) -> tuple[
406 + list[int],
407 + MultiModalPromptUpdatesApplyResult,
408 + Mapping[str, list[PlaceholderFeaturesInfo]],
409 + ]:
410 + # Same as the base implementation, except that each image block gets
411 + # its compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start %
412 + # COMPRESS_PAD_TO`` IMAGE_PAD_ID tokens) prepended while splicing:
413 + # the pad depends on the block's final position in the prompt, which
414 + # is unknown when the (cacheable) prompt updates are built. Pad
415 + # positions are not embed positions (their id differs from
416 + # image_token_id), so ``is_embed`` stays aligned with the pad-free
417 + # replacement content.
418 + matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates)
419 + placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
420 + modality: [] for modality in mm_prompt_updates
421 + }
422 +
423 + new_token_ids = list[int]()
424 + prev_end_idx = 0
425 + for matched_update in matched_updates:
426 + update = matched_update.update
427 + match = matched_update.match
428 +
429 + if update.mode == UpdateMode.INSERT:
430 + end_idx_to_insert = match.end_idx
431 + elif update.mode == UpdateMode.REPLACE:
432 + end_idx_to_insert = match.start_idx
433 + else:
434 + assert_never(update.mode)
435 +
436 + new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
437 + start_idx = len(new_token_ids)
438 +
439 + tokens = list(update.content.full)
440 + if tokens and update.modality == "image":
441 + compress_pad = COMPRESS_PAD_TO - 1 - start_idx % COMPRESS_PAD_TO
442 + tokens = [IMAGE_PAD_ID] * compress_pad + tokens
443 +
444 + if tokens:
445 + content_is_embed = update.content.is_embed
446 + is_embed = (
447 + content_is_embed(tokens) if content_is_embed is not None else None
448 + )
449 + placeholders[update.modality].append(
450 + PlaceholderFeaturesInfo(
451 + modality=update.modality,
452 + item_idx=update.item_idx,
453 + start_idx=start_idx,
454 + tokens=tokens,
455 + is_embed=is_embed,
456 + )
457 + )
458 + new_token_ids.extend(tokens)
459 +
460 + prev_end_idx = match.end_idx
461 +
462 + new_token_ids.extend(token_ids[prev_end_idx:])
463 + return new_token_ids, result, placeholders
vllm/parser/deepseek_v4.py
+15 -5
@@ -70,11 +70,17 @@ _PARTIAL_PARAM_RE = re.compile(
70 )
71
72
73 -def _dsml_arg_converter(raw_args: str, partial: bool) -> str:
73 +def _dsml_arg_converter(
74 + raw_args: str,
75 + partial: bool,
76 + *,
77 + param_re: re.Pattern = _PARAM_RE,
78 + partial_param_re: re.Pattern = _PARTIAL_PARAM_RE,
79 +) -> str:
80 params: dict[str, object] = {}
81
82 last_end = 0
77 - for m in _PARAM_RE.finditer(raw_args):
83 + for m in param_re.finditer(raw_args):
84 name, is_str, value = m.group(1), m.group(2), m.group(3)
85 if is_str == "true":
86 params[name] = value
@@ -86,7 +92,7 @@ def _dsml_arg_converter(raw_args: str, partial: bool) -> str:
92 last_end = m.end()
93
94 if partial:
89 - pm = _PARTIAL_PARAM_RE.search(raw_args, last_end)
95 + pm = partial_param_re.search(raw_args, last_end)
96 if pm:
97 name, is_str, value = pm.group(1), pm.group(2), pm.group(3)
98 if is_str == "true":
@@ -229,6 +235,8 @@ def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig:
235
236
237 class DeepSeekV4Parser(ParserEngine):
238 + parser_config = staticmethod(deepseek_v4_config)
239 +
240 def __init__(
241 self,
242 tokenizer: TokenizerLike,
@@ -245,13 +253,15 @@ class DeepSeekV4Parser(ParserEngine):
253 super().__init__(
254 tokenizer,
255 tools,
248 - parser_engine_config=deepseek_v4_config(thinking=thinking),
256 + parser_engine_config=self.parser_config(thinking=thinking),
257 **kwargs,
258 )
259 self._arg_converter = self._convert_args
260
261 def _convert_args(self, raw_args: str, partial: bool) -> str:
254 - result = _dsml_arg_converter(raw_args, partial)
262 + converter = self.parser_engine_config.arg_converter
263 + assert converter is not None
264 + result = converter(raw_args, partial)
265 if not self._tools:
266 return result
267 func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None)
vllm/parser/deepseek_v41.py new
+65
@@ -0,0 +1,65 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""DeepSeek V4.1 reasoning and spaced DSML tool calls."""
4 +
5 +import functools
6 +from dataclasses import replace
7 +
8 +import regex as re
9 +
10 +from vllm.parser.deepseek_v4 import (
11 + DeepSeekV4Parser,
12 + _dsml_arg_converter,
13 + deepseek_v4_config,
14 +)
15 +from vllm.parser.engine.parser_engine_config import ParserEngineConfig
16 +
17 +DSML_TOOL_START = "<｜DSML｜ calls>"
18 +DSML_TOOL_END = "</｜DSML｜ calls>"
19 +DSML_INVOKE_PREFIX = '<｜DSML｜ invoke name="'
20 +DSML_INVOKE_END = "</｜DSML｜ invoke>"
21 +DSML_PARAM_START = "<｜DSML｜ parameter"
22 +DSML_PARAM_CLOSE = "</｜DSML｜ parameter>"
23 +
24 +_PARAM_RE = re.compile(
25 + r'<｜DSML｜ parameter\s+name="([^"]+)"\s+string="(true|false)">'
26 + r"(.*?)"
27 + r"(?:</｜DSML｜ parameter>|(?=<｜DSML｜ parameter\s+name=))",
28 + re.DOTALL,
29 +)
30 +_PARTIAL_PARAM_RE = re.compile(
31 + r'<｜DSML｜ parameter\s+name="([^"]+)"\s+string="(true|false)">'
32 + r"(.*)$",
33 + re.DOTALL,
34 +)
35 +
36 +
37 +@functools.cache
38 +def deepseek_v41_config(thinking: bool = False) -> ParserEngineConfig:
39 + config = deepseek_v4_config(thinking=thinking)
40 + terminal_overrides = {
41 + "TOOL_START": DSML_TOOL_START,
42 + "TOOL_END": DSML_TOOL_END,
43 + "INVOKE_PREFIX": DSML_INVOKE_PREFIX,
44 + "INVOKE_END": DSML_INVOKE_END,
45 + "PARAM_START": DSML_PARAM_START,
46 + "PARAM_CLOSE": DSML_PARAM_CLOSE,
47 + }
48 + return replace(
49 + config,
50 + name="deepseek_v41",
51 + terminals={**config.terminals, **terminal_overrides},
52 + token_id_terminals={
53 + key: terminal_overrides.get(key, value)
54 + for key, value in config.token_id_terminals.items()
55 + },
56 + arg_converter=functools.partial(
57 + _dsml_arg_converter,
58 + param_re=_PARAM_RE,
59 + partial_param_re=_PARTIAL_PARAM_RE,
60 + ),
61 + )
62 +
63 +
64 +class DeepSeekV41Parser(DeepSeekV4Parser):
65 + parser_config = staticmethod(deepseek_v41_config)
vllm/parser/engine/registered_adapters.py
+6
@@ -9,6 +9,7 @@ names so that :class:`ReasoningParserManager` and
9
10 from vllm.parser.deepseek_v4 import DeepSeekV4Parser
11 from vllm.parser.deepseek_v32 import DeepSeekV32Parser
12 +from vllm.parser.deepseek_v41 import DeepSeekV41Parser
13 from vllm.parser.engine.adapters import make_adapters
14 from vllm.parser.gemma4 import Gemma4Parser
15 from vllm.parser.glm47_moe import Glm47MoeParser
@@ -30,6 +31,11 @@ from vllm.parser.seed_oss import SeedOssParser
31 DeepSeekV4ParserToolAdapter,
32 ) = make_adapters(DeepSeekV4Parser)
33
34 +(
35 + DeepSeekV41ParserReasoningAdapter,
36 + DeepSeekV41ParserToolAdapter,
37 +) = make_adapters(DeepSeekV41Parser)
38 +
39 (
40 MinimaxM2ParserReasoningAdapter,
41 MinimaxM2ParserToolAdapter,
vllm/reasoning/__init__.py
+4
@@ -32,6 +32,10 @@ _REASONING_PARSERS_TO_REGISTER = {
32 "deepseek_v4_engine_reasoning_parser",
33 "DeepSeekV4ParserReasoningAdapter",
34 ),
35 + "deepseek_v41": (
36 + "deepseek_v41_engine_reasoning_parser",
37 + "DeepSeekV41ParserReasoningAdapter",
38 + ),
39 "poolside_v1": (
40 "poolside_v1_reasoning_parser",
41 "PoolsideV1ReasoningParser",
vllm/reasoning/deepseek_v41_engine_reasoning_parser.py new
+6
@@ -0,0 +1,6 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.engine.registered_adapters import DeepSeekV41ParserReasoningAdapter
5 +
6 +__all__ = ["DeepSeekV41ParserReasoningAdapter"]
vllm/renderers/registry.py
+1
@@ -23,6 +23,7 @@ _VLLM_RENDERERS = {
23 "cohere": ("cohere", "CohereRenderer"),
24 "deepseek_v32": ("deepseek_v32", "DeepseekV32Renderer"),
25 "deepseek_v4": ("deepseek_v4", "DeepseekV4Renderer"),
26 + "deepseek_v41": ("deepseek_v4", "DeepseekV4Renderer"),
27 "hf": ("hf", "HfRenderer"),
28 "kimi_audio": ("hf", "HfRenderer"),
29 "kimi_k3": ("kimi_k3", "KimiK3Renderer"),
vllm/tokenizers/deepseek_v41.py new
+121
@@ -0,0 +1,121 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import copy
5 +from typing import Any
6 +
7 +from transformers import TokenizersBackend
8 +
9 +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
10 +
11 +from .deepseek_v41_encoding import (
12 + IMAGE_PLACEHOLDER,
13 + REASONING_EFFORT_MAPPINGS,
14 + encode_messages,
15 +)
16 +from .hf import HfTokenizer, get_cached_tokenizer
17 +from .protocol import TokenizerLike
18 +
19 +
20 +def _normalize_messages(
21 + messages: list[ChatCompletionMessageParam],
22 +) -> list[dict[str, Any]]:
23 + result = [dict(message) for message in copy.deepcopy(messages)]
24 + for message in result:
25 + role = message.get("role")
26 + if role not in ("system", "developer", "user", "assistant", "tool"):
27 + raise ValueError(f"Invalid role: {role}")
28 + if "reasoning" in message:
29 + message["reasoning_content"] = message["reasoning"]
30 + content = message.get("content")
31 + if isinstance(content, list):
32 + parts = []
33 + for block in content:
34 + part_type = block.get("type")
35 + if part_type == "text":
36 + parts.append(block.get("text", ""))
37 + elif part_type in ("image_url", "input_image", "image_pil"):
38 + parts.append(IMAGE_PLACEHOLDER)
39 + else:
40 + raise ValueError(
41 + "DeepSeek V4.1 supports text and image content only; "
42 + f"got {part_type!r}"
43 + )
44 + message["content"] = "\n\n".join(parts)
45 + return result
46 +
47 +
48 +def get_deepseek_v41_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
49 + """Wrap an HF tokenizer with the V4.1 prompt encoder."""
50 + wrapped = copy.copy(tokenizer)
51 + added_vocab = tokenizer.get_added_vocab()
52 +
53 + class _DeepseekV41Tokenizer(tokenizer.__class__): # type: ignore
54 + def apply_chat_template(
55 + self,
56 + messages: list[ChatCompletionMessageParam],
57 + tools: list[dict[str, Any]] | None = None,
58 + **kwargs,
59 + ) -> str | list[int]:
60 + # The generic renderer's conversation has already flattened text
61 + # parts with '\n'. V4.1 encodes the original messages with '\n\n'.
62 + conversation = _normalize_messages(messages)
63 + if tools:
64 + system = next((m for m in conversation if m["role"] == "system"), None)
65 + if system is None:
66 + system = {"role": "system", "content": ""}
67 + conversation.insert(0, system)
68 + system["tools"] = tools
69 +
70 + thinking = bool(kwargs.get("thinking") or kwargs.get("enable_thinking"))
71 + if "thinking" not in kwargs and "enable_thinking" not in kwargs:
72 + thinking = True
73 + effort = kwargs.get("reasoning_effort")
74 + if effort == "none":
75 + thinking = False
76 + effort = None
77 + if effort is None:
78 + effort = "high"
79 + if not (
80 + (type(effort) is int and 1 <= effort <= 100)
81 + or (isinstance(effort, str) and effort in REASONING_EFFORT_MAPPINGS)
82 + ):
83 + raise ValueError(
84 + "DeepSeek V4.1 reasoning_effort must be low, high, xhigh, max, "
85 + "or an integer within [1, 100] in chat_template_kwargs"
86 + )
87 +
88 + prompt = encode_messages(
89 + conversation,
90 + thinking_mode="thinking" if thinking else "chat",
91 + drop_thinking=kwargs.get("drop_thinking", True),
92 + reasoning_effort=effort,
93 + )
94 + if kwargs.get("tokenize", True):
95 + tokenizer_kwargs = {
96 + key: kwargs[key]
97 + for key in ("truncation", "max_length")
98 + if key in kwargs
99 + }
100 + return self.encode(prompt, add_special_tokens=False, **tokenizer_kwargs)
101 + return prompt
102 +
103 + def num_special_tokens_to_add(self) -> int:
104 + return len(self.encode(""))
105 +
106 + def get_added_vocab(self) -> dict[str, int]:
107 + return added_vocab.copy()
108 +
109 + def __reduce__(self):
110 + return get_deepseek_v41_tokenizer, (tokenizer,)
111 +
112 + _DeepseekV41Tokenizer.__name__ = f"DSV41{tokenizer.__class__.__name__}"
113 + wrapped.__class__ = _DeepseekV41Tokenizer
114 + return wrapped
115 +
116 +
117 +class DeepseekV41Tokenizer(TokenizerLike):
118 + @classmethod
119 + def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
120 + tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs)
121 + return get_cached_tokenizer(get_deepseek_v41_tokenizer(tokenizer))
vllm/tokenizers/deepseek_v41_encoding.py new
+577
@@ -0,0 +1,577 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# ruff: noqa
4 +
5 +"""Text encoding adapted from the DeepSeek V4.1 reference encoder.
6 +
7 +Message normalization and serving controls live in deepseek_v41.py.
8 +"""
9 +
10 +import copy
11 +import json
12 +from typing import Any, Dict, List, Optional, Union
13 +
14 +bos_token: str = "<｜begin▁of▁sentence｜>"
15 +eos_token: str = "<｜end▁of▁sentence｜>"
16 +thinking_start_token: str = "<think>"
17 +thinking_end_token: str = "</think>"
18 +dsml_token: str = "｜DSML｜"
19 +
20 +USER_SP_TOKEN = "<｜User｜>"
21 +ASSISTANT_SP_TOKEN = "<｜Assistant｜>"
22 +LATEST_REMINDER_SP_TOKEN = "<｜latest_reminder｜>"
23 +IMAGE_PLACEHOLDER = "<｜deepseek_image｜>"
24 +
25 +DS_TASK_SP_TOKENS = {
26 + "action": "<｜action｜>",
27 + "query": "<｜query｜>",
28 + "authority": "<｜authority｜>",
29 + "domain": "<｜domain｜>",
30 + "title": "<｜title｜>",
31 + "read_url": "<｜read_url｜>",
32 +}
33 +VALID_TASKS = set(DS_TASK_SP_TOKENS.keys())
34 +
35 +system_msg_template: str = "{content}"
36 +user_msg_template: str = "{content}"
37 +latest_reminder_msg_template: str = "{content}"
38 +assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token
39 +assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}"
40 +thinking_template: str = "{reasoning_content}"
41 +
42 +response_format_template: str = (
43 + "## Response Format:\n\n"
44 + "You MUST strictly adhere to the following schema to reply:\n{schema}"
45 +)
46 +tool_output_template: str = "<tool_result>{content}</tool_result>"
47 +
48 +
49 +def to_json(value: Any) -> str:
50 + """Serialize a value to JSON string."""
51 + try:
52 + return json.dumps(value, ensure_ascii=False)
53 + except:
54 + return json.dumps(value, ensure_ascii=True)
55 +
56 +
57 +def tools_from_openai_format(tools):
58 + """Extract function definitions from OpenAI-format tool list."""
59 + return [tool["function"] for tool in tools]
60 +
61 +
62 +def tool_calls_from_openai_format(tool_calls):
63 + """Convert OpenAI-format tool calls to internal format."""
64 + return [
65 + {
66 + "name": tool_call["function"]["name"],
67 + "arguments": tool_call["function"]["arguments"],
68 + }
69 + for tool_call in tool_calls
70 + ]
71 +
72 +
73 +def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
74 + """
75 + Merge tool messages into the preceding user message using content_blocks format.
76 +
77 + DeepSeek-V4.1 does not have a standalone "tool" role; instead, tool results
78 + are encoded as <tool_result> blocks within user messages.
79 + """
80 + merged: List[Dict[str, Any]] = []
81 +
82 + for msg in messages:
83 + msg = copy.deepcopy(msg)
84 + role = msg.get("role")
85 +
86 + if role == "tool":
87 + # Convert tool message to a user message with tool_result block
88 + tool_block = {
89 + "type": "tool_result",
90 + "tool_use_id": msg.get("tool_call_id", ""),
91 + "content": msg.get("content", ""),
92 + }
93 + # Merge into previous message if it's already a user (merged tool)
94 + if (
95 + merged
96 + and merged[-1].get("role") == "user"
97 + and "content_blocks" in merged[-1]
98 + ):
99 + merged[-1]["content_blocks"].append(tool_block)
100 + else:
101 + merged.append(
102 + {
103 + "role": "user",
104 + "content_blocks": [tool_block],
105 + }
106 + )
107 + elif role == "user":
108 + content_blocks = msg.get("content_blocks")
109 + if content_blocks is None:
110 + content_blocks = [{"type": "text", "text": msg.get("content", "")}]
111 + if (
112 + merged
113 + and merged[-1].get("role") == "user"
114 + and "content_blocks" in merged[-1]
115 + and merged[-1].get("task") is None
116 + ):
117 + merged[-1]["content_blocks"].extend(content_blocks)
118 + else:
119 + # Preserve structured content and all message-level metadata.
120 + new_msg = msg
121 + new_msg["content_blocks"] = content_blocks
122 + merged.append(new_msg)
123 + else:
124 + merged.append(msg)
125 +
126 + return merged
127 +
128 +
129 +def sort_tool_results_by_call_order(
130 + messages: List[Dict[str, Any]],
131 +) -> List[Dict[str, Any]]:
132 + """
133 + Sort tool_result blocks within user messages by the order of tool_calls
134 + in the preceding assistant message.
135 + """
136 + last_tool_call_order: Dict[str, int] = {}
137 +
138 + for msg in messages:
139 + role = msg.get("role")
140 + if role == "assistant" and msg.get("tool_calls"):
141 + last_tool_call_order = {}
142 + for idx, tc in enumerate(msg["tool_calls"]):
143 + tc_id = tc.get("id") or tc.get("function", {}).get("id", "")
144 + if tc_id:
145 + last_tool_call_order[tc_id] = idx
146 +
147 + elif role == "user" and msg.get("content_blocks"):
148 + tool_blocks = [
149 + b for b in msg["content_blocks"] if b.get("type") == "tool_result"
150 + ]
151 + if len(tool_blocks) > 1 and last_tool_call_order:
152 + sorted_blocks = sorted(
153 + tool_blocks,
154 + key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0),
155 + )
156 + sorted_idx = 0
157 + new_blocks = []
158 + for block in msg["content_blocks"]:
159 + if block.get("type") == "tool_result":
160 + new_blocks.append(sorted_blocks[sorted_idx])
161 + sorted_idx += 1
162 + else:
163 + new_blocks.append(block)
164 + msg["content_blocks"] = new_blocks
165 +
166 + return messages
167 +
168 +
169 +SYSTEM_SP_TOKEN = "<｜System｜>"
170 +
171 +tool_calls_block_name: str = " calls"
172 +tool_call_tag_name: str = " invoke"
173 +tool_parameter_tag_name: str = " parameter"
174 +tool_call_template: str = '<{dsml_token}{tool_call_tag_name} name="{name}">\n{arguments}\n</{dsml_token}{tool_call_tag_name}>'
175 +tool_calls_template = (
176 + "<{dsml_token}{tc_block_name}>\n{tool_calls}\n</{dsml_token}{tc_block_name}>"
177 +)
178 +
179 +REASONING_EFFORT_TEMPLATE = (
180 + "Reasoning Effort: {budget} "
181 + "(range 1-100, the higher the value, the more thorough the reasoning)\n\n"
182 +)
183 +REASONING_EFFORT_MAPPINGS: Dict[str, int] = {
184 + "low": 25,
185 + "high": 50,
186 + "xhigh": 75,
187 + "max": 100,
188 +}
189 +
190 +DEFAULT_REASONING_EFFORT = "high"
191 +
192 +
193 +def render_reasoning_effort(
194 + index: int,
195 + thinking_mode: str,
196 + effort: Union[str, int, None],
197 +) -> str:
198 + """Render the V4.1 numeric reasoning effort prefix (thinking mode, index 0 only)."""
199 + if effort is None:
200 + effort = DEFAULT_REASONING_EFFORT
201 + assert (
202 + type(effort) is int and 1 <= effort <= 100
203 + ) or effort in REASONING_EFFORT_MAPPINGS, (
204 + "Invalid reasoning effort for deepseek_v41: "
205 + f"{effort}, should be int within [1,100] or {list(REASONING_EFFORT_MAPPINGS)}"
206 + )
207 + if type(effort) is str:
208 + effort = REASONING_EFFORT_MAPPINGS[effort]
209 + if index == 0 and thinking_mode == "thinking":
210 + return REASONING_EFFORT_TEMPLATE.format(budget=effort)
211 + return ""
212 +
213 +
214 +TOOLS_TEMPLATE = """## Tools
215 +
216 +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}{tc_block_name}>" block like the following:
217 +
218 +<{dsml_token}{tc_block_name}>
219 +<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME">
220 +<{dsml_token}{tool_parameter_tag_name} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml_token}{tool_parameter_tag_name}>
221 +...
222 +</{dsml_token}{tool_call_tag_name}>
223 +<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME2">
224 +...
225 +</{dsml_token}{tool_call_tag_name}>
226 +</{dsml_token}{tc_block_name}>
227 +
228 +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
229 +
230 +If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response.
231 +
232 +Otherwise, output directly after {thinking_end_token} with tool calls or final response.
233 +
234 +### Available Tool Schemas
235 +
236 +{tool_schemas}
237 +
238 +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
239 +"""
240 +
241 +
242 +def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str:
243 + """Render tool schemas into the V4.1 system prompt format."""
244 + tools_json = [to_json(t) for t in tools]
245 +
246 + return TOOLS_TEMPLATE.format(
247 + tool_schemas="\n".join(tools_json),
248 + dsml_token=dsml_token,
249 + tc_block_name=tool_calls_block_name,
250 + tool_call_tag_name=tool_call_tag_name,
251 + tool_parameter_tag_name=tool_parameter_tag_name,
252 + thinking_start_token=thinking_start_token,
253 + thinking_end_token=thinking_end_token,
254 + )
255 +
256 +
257 +def encode_arguments_to_dsml(tool_call: Dict[str, Any]) -> str:
258 + """Encode tool call arguments into V4.1 DSML parameter format."""
259 + p_dsml_template = (
260 + '<{dsml_token}{tool_parameter_tag_name} name="{key}" string="{is_str}">'
261 + "{value}</{dsml_token}{tool_parameter_tag_name}>"
262 + )
263 + P_dsml_strs = []
264 +
265 + arguments = tool_call["arguments"]
266 + if not isinstance(arguments, dict):
267 + # Tolerate JSON strings, including double-encoded ones.
268 + for _ in range(2):
269 + if isinstance(arguments, str):
270 + try:
271 + arguments = json.loads(arguments)
272 + except Exception:
273 + break
274 + else:
275 + break
276 + if not isinstance(arguments, dict):
277 + arguments = {"arguments": tool_call["arguments"]}
278 +
279 + for k, v in arguments.items():
280 + P_dsml_strs.append(
281 + p_dsml_template.format(
282 + dsml_token=dsml_token,
283 + tool_parameter_tag_name=tool_parameter_tag_name,
284 + key=k,
285 + is_str="true" if isinstance(v, str) else "false",
286 + value=v if isinstance(v, str) else to_json(v),
287 + )
288 + )
289 +
290 + return "\n".join(P_dsml_strs)
291 +
292 +
293 +def find_last_user_index(messages: List[Dict[str, Any]]) -> int:
294 + """
295 + Find the index of the last user/developer message.
296 +
297 + V4.1 supports mid-conversation system messages, which count as user
298 + messages for the purposes of the assistant generation header.
299 + """
300 + last_user_index = -1
301 + for idx in range(len(messages) - 1, -1, -1):
302 + role = messages[idx].get("role")
303 + if role in ["user", "developer"] or (role == "system" and idx > 0):
304 + last_user_index = idx
305 + break
306 + return last_user_index
307 +
308 +
309 +def render_message(
310 + index: int,
311 + messages: List[Dict[str, Any]],
312 + thinking_mode: str,
313 + drop_thinking: bool = True,
314 + reasoning_effort: Union[str, int, None] = None,
315 +) -> str:
316 + """
317 + Render a single message at the given index into its V4.1 encoded string form.
318 + """
319 + assert 0 <= index < len(messages)
320 + assert thinking_mode in ["chat", "thinking"], (
321 + f"Invalid thinking_mode `{thinking_mode}`"
322 + )
323 +
324 + msg = messages[index]
325 + last_user_idx = find_last_user_index(messages)
326 +
327 + role = msg.get("role")
328 + content = msg.get("content")
329 + tools = msg.get("tools")
330 + response_format = msg.get("response_format")
331 + tool_calls = msg.get("tool_calls")
332 + reasoning_content = msg.get("reasoning_content")
333 + wo_eos = msg.get("wo_eos", False)
334 +
335 + if tools:
336 + tools = tools_from_openai_format(tools)
337 + if tool_calls:
338 + tool_calls = tool_calls_from_openai_format(tool_calls)
339 +
340 + # Reasoning effort prefix (thinking mode, index 0 only)
341 + reasoning_effort_prompt = render_reasoning_effort(
342 + index, thinking_mode, reasoning_effort
343 + )
344 + # System token leads the conversation when there is a reasoning effort prompt
345 + # or the first message is a system message.
346 + prompt = (
347 + SYSTEM_SP_TOKEN
348 + if index == 0 and (reasoning_effort_prompt or role == "system")
349 + else ""
350 + )
351 + prompt += reasoning_effort_prompt
352 +
353 + if role == "system":
354 + if index > 0:
355 + # Mid-conversation system message
356 + prompt += SYSTEM_SP_TOKEN
357 + prompt += system_msg_template.format(content=content or "")
358 + if tools:
359 + prompt += "\n\n" + render_tools(tools)
360 + if response_format:
361 + prompt += "\n\n" + response_format_template.format(
362 + schema=to_json(response_format)
363 + )
364 +
365 + elif role == "developer":
366 + assert content, f"Invalid message for role `{role}`: {msg}"
367 +
368 + content_developer = USER_SP_TOKEN
369 + content_developer += content
370 +
371 + if tools:
372 + content_developer += "\n\n" + render_tools(tools)
373 + if response_format:
374 + content_developer += "\n\n" + response_format_template.format(
375 + schema=to_json(response_format)
376 + )
377 +
378 + prompt += user_msg_template.format(content=content_developer)
379 +
380 + elif role == "user":
381 + prompt += USER_SP_TOKEN
382 +
383 + # Handle content blocks (tool results mixed with text)
384 + content_blocks = msg.get("content_blocks")
385 + if content_blocks:
386 + parts = []
387 + for block in content_blocks:
388 + block_type = block.get("type")
389 + if block_type == "text":
390 + parts.append(block.get("text", ""))
391 + elif block_type == "tool_result":
392 + tool_content = block.get("content", "")
393 + if isinstance(tool_content, list):
394 + text_parts = []
395 + for b in tool_content:
396 + if b.get("type") == "text":
397 + text_parts.append(b.get("text", ""))
398 + else:
399 + text_parts.append(f"[Unsupported {b.get('type')}]")
400 + tool_content = "\n\n".join(text_parts)
401 + parts.append(tool_output_template.format(content=tool_content))
402 + else:
403 + parts.append(f"[Unsupported {block_type}]")
404 + prompt += "\n\n".join(parts)
405 + else:
406 + prompt += content or ""
407 +
408 + elif role == "latest_reminder":
409 + prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(
410 + content=content
411 + )
412 +
413 + elif role == "tool":
414 + raise NotImplementedError(
415 + "deepseek_v41 merges tool messages into user; please preprocess with merge_tool_messages()"
416 + )
417 +
418 + elif role == "assistant":
419 + thinking_part = ""
420 + tc_content = ""
421 +
422 + if tool_calls:
423 + tc_list = [
424 + tool_call_template.format(
425 + dsml_token=dsml_token,
426 + tool_call_tag_name=tool_call_tag_name,
427 + name=tc.get("name"),
428 + arguments=encode_arguments_to_dsml(tc),
429 + )
430 + for tc in tool_calls
431 + ]
432 + tc_content += "\n\n" + tool_calls_template.format(
433 + dsml_token=dsml_token,
434 + tool_calls="\n".join(tc_list),
435 + tc_block_name=tool_calls_block_name,
436 + )
437 +
438 + summary_content = content or ""
439 + rc = reasoning_content or ""
440 +
441 + # Check if previous message has a task - if so, this is a task output (no thinking)
442 + prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None
443 +
444 + if thinking_mode == "thinking" and not prev_has_task:
445 + if not drop_thinking or index > last_user_idx:
446 + thinking_part = (
447 + thinking_template.format(reasoning_content=rc) + thinking_end_token
448 + )
449 + else:
450 + thinking_part = ""
451 +
452 + if wo_eos:
453 + prompt += assistant_msg_wo_eos_template.format(
454 + reasoning=thinking_part,
455 + content=summary_content,
456 + tool_calls=tc_content,
457 + )
458 + else:
459 + prompt += assistant_msg_template.format(
460 + reasoning=thinking_part,
461 + content=summary_content,
462 + tool_calls=tc_content,
463 + )
464 + else:
465 + raise NotImplementedError(f"Unknown role: {role}")
466 +
467 + # Append transition tokens based on what follows
468 + if index + 1 < len(messages) and messages[index + 1].get("role") not in [
469 + "assistant",
470 + "latest_reminder",
471 + ]:
472 + return prompt
473 +
474 + task = messages[index].get("task")
475 + if task is not None:
476 + # Task special token for internal classification tasks
477 + assert task in VALID_TASKS, (
478 + f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}"
479 + )
480 + task_sp_token = DS_TASK_SP_TOKENS[task]
481 +
482 + if task != "action":
483 + # Non-action tasks: append task sp token directly after the message
484 + prompt += task_sp_token
485 + else:
486 + # Action task: append Assistant + thinking token + action sp token
487 + prompt += ASSISTANT_SP_TOKEN
488 + prompt += (
489 + thinking_end_token
490 + if thinking_mode != "thinking"
491 + else thinking_start_token
492 + )
493 + prompt += task_sp_token
494 +
495 + elif messages[index].get("role") in ["user", "developer"] or (
496 + messages[index].get("role") == "system" and index > 0
497 + ):
498 + # Normal generation: append Assistant + thinking token
499 + # (mid-conversation system messages also trigger the assistant header)
500 + prompt += ASSISTANT_SP_TOKEN
501 + if not drop_thinking and thinking_mode == "thinking":
502 + prompt += thinking_start_token
503 + elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx:
504 + prompt += thinking_start_token
505 + else:
506 + prompt += thinking_end_token
507 +
508 + return prompt
509 +
510 +
511 +def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
512 + """
513 + Drop reasoning_content and non-essential messages before the last user message.
514 + Same as V4, but uses the V4.1 last-user definition (mid systems count).
515 + """
516 + last_user_idx = find_last_user_index(messages)
517 + result = []
518 + keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"}
519 +
520 + for idx, msg in enumerate(messages):
521 + role = msg.get("role")
522 + if role in keep_roles or idx >= last_user_idx:
523 + result.append(msg)
524 + elif role == "assistant":
525 + msg = copy.copy(msg)
526 + msg.pop("reasoning_content", None)
527 + result.append(msg)
528 + # developer and other roles before last_user_idx are dropped
529 +
530 + return result
531 +
532 +
533 +def encode_messages(
534 + messages: List[Dict[str, Any]],
535 + thinking_mode: str,
536 + context: Optional[List[Dict[str, Any]]] = None,
537 + drop_thinking: bool = True,
538 + add_default_bos_token: bool = True,
539 + reasoning_effort: Union[str, int, None] = None,
540 +) -> str:
541 + """Encode preprocessed (text-only) messages into the V4.1 prompt format."""
542 + context = context if context else []
543 +
544 + # Preprocess: merge tool messages and sort tool results
545 + messages = merge_tool_messages(messages)
546 + messages = sort_tool_results_by_call_order(context + messages)[len(context) :]
547 + if context:
548 + context = merge_tool_messages(context)
549 + context = sort_tool_results_by_call_order(context)
550 +
551 + full_messages = context + messages
552 +
553 + prompt = bos_token if add_default_bos_token and len(context) == 0 else ""
554 +
555 + # Resolve drop_thinking: if any message has tools defined, don't drop thinking
556 + effective_drop_thinking = drop_thinking
557 + if any(m.get("tools") for m in full_messages):
558 + effective_drop_thinking = False
559 +
560 + if thinking_mode == "thinking" and effective_drop_thinking:
561 + full_messages = _drop_thinking_messages(full_messages)
562 + num_to_render = len(full_messages) - len(_drop_thinking_messages(context))
563 + context_len = len(full_messages) - num_to_render
564 + else:
565 + num_to_render = len(messages)
566 + context_len = len(context)
567 +
568 + for idx in range(num_to_render):
569 + prompt += render_message(
570 + idx + context_len,
571 + full_messages,
572 + thinking_mode=thinking_mode,
573 + drop_thinking=effective_drop_thinking,
574 + reasoning_effort=reasoning_effort,
575 + )
576 +
577 + return prompt
vllm/tokenizers/registry.py
+1
@@ -45,6 +45,7 @@ _VLLM_TOKENIZERS = {
45 "cohere": ("hf", "CachedHfTokenizer"),
46 "deepseek_v32": ("deepseek_v32", "DeepseekV32Tokenizer"),
47 "deepseek_v4": ("deepseek_v4", "DeepseekV4Tokenizer"),
48 + "deepseek_v41": ("deepseek_v41", "DeepseekV41Tokenizer"),
49 "hf": ("hf", "CachedHfTokenizer"),
50 "kimi_audio": ("kimi_audio", "KimiAudioTokenizer"),
51 "kimi_k3": ("hf", "CachedHfTokenizer"),
vllm/tool_parsers/__init__.py
+4
@@ -42,6 +42,10 @@ _TOOL_PARSERS_TO_REGISTER = {
42 "deepseekv4_engine_tool_parser",
43 "DeepSeekV4EngineToolParser",
44 ),
45 + "deepseek_v41": (
46 + "deepseekv41_engine_tool_parser",
47 + "DeepSeekV41EngineToolParser",
48 + ),
49 "cohere_command3": (
50 "cohere_command_tool_parser",
51 "CohereCommand3ToolParser",
vllm/tool_parsers/deepseekv41_engine_tool_parser.py new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.engine.registered_adapters import DeepSeekV41ParserToolAdapter
5 +
6 +
7 +class DeepSeekV41EngineToolParser(DeepSeekV41ParserToolAdapter): # type: ignore[valid-type, misc]
8 + structural_tag_model = "deepseek_v41"
vllm/tool_parsers/structural_tag_registry.py
+96 -1
@@ -71,7 +71,9 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
71 "deepseek_v4",
72 }
73 )
74 -VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"})
74 +VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
75 + {"deepseek_v41", "hermes", "hy_v4", "kimi_k3"}
76 +)
77 SUPPORTED_STRUCTURAL_TAG_MODELS = (
78 XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
79 )
@@ -225,6 +227,99 @@ def get_function_parameters(function) -> dict[str, Any] | bool:
227 return function.parameters if function.parameters is not None else True
228
229
230 +_V41_CALLS_START = "<｜DSML｜ calls>"
231 +_V41_CALLS_END = "</｜DSML｜ calls>"
232 +_V41_INVOKE_END = "</｜DSML｜ invoke>"
233 +_V41_PARAMETER_END = "</｜DSML｜ parameter>"
234 +
235 +
236 +@register_vllm_structural_tag("deepseek_v41")
237 +def get_deepseek_v41_structural_tag(
238 + tools: list[FunctionToolParam],
239 + builtin_tools: list[BuiltinToolParam],
240 + tool_choice: SimplifiedToolChoice,
241 + reasoning: bool,
242 + token_suffix: str = "",
243 +) -> StructuralTag:
244 + # Serving enables this visible-text grammar after the reasoning boundary.
245 + del builtin_tools, reasoning, token_suffix
246 +
247 + # TODO: lower parameter schemas into DSML constraints. This builder constrains
248 + # tool names and DSML/value syntax only. Parameter names, presence, uniqueness
249 + # and value schemas remain unconstrained, including for strict=true. The request
250 + # layer still uses strict to decide whether auto tool choice activates a grammar.
251 + parameter = TagFormat(
252 + begin='<｜DSML｜ parameter name="',
253 + content=SequenceFormat(
254 + elements=[
255 + RegexFormat(pattern=r'[^"]+'),
256 + ConstStringFormat(value='" string="'),
257 + OrFormat(
258 + elements=[
259 + SequenceFormat(
260 + elements=[
261 + ConstStringFormat(value='true">'),
262 + AnyTextFormat(
263 + excludes=[
264 + _V41_PARAMETER_END,
265 + _V41_INVOKE_END,
266 + _V41_CALLS_END,
267 + ]
268 + ),
269 + ]
270 + ),
271 + SequenceFormat(
272 + elements=[
273 + ConstStringFormat(value='false">'),
274 + JSONSchemaFormat(json_schema=True),
275 + ]
276 + ),
277 + ]
278 + ),
279 + ]
280 + ),
281 + end=f"{_V41_PARAMETER_END}\n",
282 + )
283 + calls = TagsWithSeparatorFormat(
284 + tags=[
285 + TagFormat(
286 + begin=f'<｜DSML｜ invoke name="{tool.function.name}">\n',
287 + content=StarFormat(content=parameter),
288 + end=f"{_V41_INVOKE_END}\n",
289 + )
290 + for tool in tools
291 + ],
292 + separator="",
293 + at_least_one=True,
294 + stop_after_first=tool_choice == "forced",
295 + )
296 + if tool_choice == "auto":
297 + return StructuralTag(
298 + format=TriggeredTagsFormat(
299 + triggers=[_V41_CALLS_START],
300 + tags=[
301 + TagFormat(
302 + begin=f"{_V41_CALLS_START}\n",
303 + content=calls,
304 + end=_V41_CALLS_END,
305 + )
306 + ],
307 + excludes=["<think>", "</think>"],
308 + )
309 + if tools
310 + else AnyTextFormat(excludes=["<think>", "</think>"])
311 + )
312 + return StructuralTag(
313 + format=SequenceFormat(
314 + elements=[
315 + ConstStringFormat(value=f"\n\n{_V41_CALLS_START}\n"),
316 + calls,
317 + ConstStringFormat(value=_V41_CALLS_END),
318 + ]
319 + )
320 + )
321 +
322 +
323 def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
324 arguments_field_prefix = '", "arguments": '
325 formats = [
vllm/transformers_utils/config.py
+1
@@ -152,6 +152,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
152 laguna="LagunaConfig",
153 lfm2_moe="Lfm2MoeConfig",
154 **{"unlimited-ocr": "UnlimitedOCRConfig"},
155 + **{"deepseek_v41": "DeepseekV41Config"},
156 inkling_mm_model="InklingMMConfig",
157 inkling_model="InklingModelConfig",
158 )
vllm/transformers_utils/configs/__init__.py
+1
@@ -33,6 +33,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
33 "DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma",
34 "DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
35 "DeepseekV4Config": "vllm.transformers_utils.configs.deepseek_v4",
36 + "DeepseekV41Config": "vllm.transformers_utils.configs.deepseek_v41",
37 "Dots3NoteConfig": "vllm.transformers_utils.configs.dots3_note",
38 "K3DSparkConfig": "vllm.transformers_utils.configs.k3_dspark",
39 "DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr",
vllm/transformers_utils/configs/deepseek_v41.py new
+76
@@ -0,0 +1,76 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from typing import Any
4 +
5 +from transformers import PretrainedConfig
6 +
7 +
8 +class DeepseekV41Config(PretrainedConfig):
9 + """DeepSeek V4.1 config.
10 +
11 + The HF config nests the text model under ``text_config`` and the vision
12 + tower under ``vision_config``. The vLLM model code (ported from
13 + ``deepseek_v4``) reads flat attributes, so both sub-configs are flattened
14 + onto the top level here: text fields are exposed as-is, vision fields
15 + with the ``vision_*`` naming used by ``deepseek_v4_1.common.vision``.
16 + """
17 +
18 + model_type = "deepseek_v41"
19 +
20 + def __init__(
21 + self,
22 + text_config: dict[str, Any] | None = None,
23 + vision_config: dict[str, Any] | None = None,
24 + **kwargs,
25 + ):
26 + text_config = dict(text_config or {})
27 + vision_config = dict(vision_config or {})
28 +
29 + # ``rope_scaling`` is a property in Transformers v5 (backed by
30 + # ``rope_parameters``); capture it here and restore after
31 + # super().__init__, which re-standardizes rope params.
32 + rope_scaling = text_config.pop("rope_scaling", None)
33 +
34 + for key, value in text_config.items():
35 + if key == "model_type":
36 + continue
37 + # Don't clobber PretrainedConfig properties (e.g. is_encoder_decoder).
38 + if isinstance(getattr(type(self), key, None), property):
39 + continue
40 + setattr(self, key, value)
41 +
42 + super().__init__(**kwargs)
43 + if rope_scaling is not None:
44 + self.rope_parameters = rope_scaling
45 +
46 + # The v4.1 quantization_config carries the expert dtype; the model
47 + # code reads it from the top-level hf_config.
48 + quant_cfg = getattr(self, "quantization_config", None) or {}
49 + if not hasattr(self, "expert_dtype") and "expert_dtype" in quant_cfg:
50 + self.expert_dtype = quant_cfg["expert_dtype"]
51 +
52 + vision_n_layers = vision_config.get("num_hidden_layers", 0)
53 + self.vision_n_layers = vision_n_layers
54 + self.vision_dim = vision_config.get("hidden_size", 1024)
55 + self.vision_n_heads = vision_config.get("num_attention_heads", 16)
56 + self.vision_inter_dim = vision_config.get("intermediate_size", 2816)
57 + self.vision_patch_size = vision_config.get("patch_size", 14)
58 + self.vision_rope_theta = vision_config.get("rope_theta", 10000.0)
59 + self.vision_downsample_ratio = vision_config.get("downsample_ratio", 3)
60 + self.vision_max_n_token = vision_config.get("max_image_tokens", 1024)
61 + self.vision_min_pixels = vision_config.get("min_pixels", 295936)
62 + self.vision_max_wh_ratio = vision_config.get("max_wh_ratio")
63 + # The vision variant needs the mm-prefix plumbing (atomic image-span
64 + # prefill + sparse-SWA window widening); the base arch-config
65 + # convertor reads this config-level attribute.
66 + self.is_mm_prefix_lm = vision_n_layers > 0
67 + # The sparse-SWA index kernels widen the window within image spans
68 + # in-kernel, so mm-prefix ranges longer than sliding_window must be
69 + # kept (they are only consumed by those kernels).
70 + self.mm_prefix_clamp_sliding_window = vision_n_layers > 0
71 + # The visibility span covers the image block [IMAGE_START,
72 + # IMAGE_END]; the mm placeholder additionally carries a leading
73 + # compressor-alignment pad of ``COMPRESS_PAD_TO - 1 - offset %
74 + # COMPRESS_PAD_TO`` tokens (see common/mm_preprocess.py), with
75 + # COMPRESS_PAD_TO = 2 for v4.1's ratio-2 compressors.
76 + self.mm_prefix_span_leading_pad_modulus = 2 if vision_n_layers > 0 else 0
vllm/transformers_utils/model_arch_config_convertor.py
+1
@@ -317,6 +317,7 @@ class ModelArchConfigConvertorBase:
317 "deepseek_v3",
318 "deepseek_v32",
319 "deepseek_v4",
320 + "deepseek_v41",
321 "dots3_note",
322 "deepseek_mtp",
323 "k3_dspark",