@ondeinference / onde / commits / a985f57

Add DeepSeek Coder 6.7B GGUF model with chat template

- Add DeepSeek Coder 6.7B (Q4_K_M) model constants and metadata - Add GgufModelConfig::deepseek_coder_6_7b() with embedded Jinja chat template - Write chat template to temp file if provided in config for mistral.rs compatibility - Update SUPPORTED_MODELS and SUPPORTED_MODEL_INFO with DeepSeek Coder entry - Extend GgufModelConfig with optional chat_template field

keypair34 committed Apr 26, 2026 at 06:59 UTC a985f57631b47fa55b63a45a44377e9e991579d5

4 files changed +84

src/inference/deepseek_coder_chat_template.txt new

@@ -0,0 +1 @@

1 +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{%- set ns = namespace(found=false) -%}{%- for message in messages -%}{%- if message['role'] == 'system' -%}{%- set ns.found = true -%}{%- endif -%}{%- endfor -%}{{bos_token}}{%- if not ns.found -%}{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n'}}{%- endif %}{%- for message in messages %}{% if message['role'] == 'system' %}{{ message['content'] }}{% else %}{% if message['role'] == 'user' %}{{'### Instruction:\n' + message['content'] + '\n'}}{% else %}{{'### Response:\n' + message['content'] + '\n<|EOT|>\n'}}{% endif %}{% endif %}{%- endfor %}{% if add_generation_prompt %}{{'### Response:\n'}}{% endif %}

\ No newline at end of file

src/inference/engine.rs

+57

@@ -47,6 +47,7 @@

47 //! tok_model_id: None,

48 //! display_name: "Qwen 2.5 1.5B".into(),

49 //! approx_memory: "~941 MB".into(),

50 +//! chat_template: None,

51 //! };

52 //!

53 //! engine.load_gguf_model(config, None).await?;

@@ -383,6 +384,27 @@ impl ChatEngine {

384 builder = builder.with_tok_model_id(tok_id);

385 }

386

387 + // Some older GGUF files (e.g. TheBloke) do not embed a chat template.

388 + // When the config provides one, write it to a temporary .jinja file

389 + // and pass the path — mistral.rs only accepts file paths ending in

390 + // .json or .jinja, not literal template strings.

391 + let _chat_template_tempfile = if let Some(ref template) = config.chat_template {

392 + let tmp_dir = std::env::temp_dir().join("onde-chat-templates");

393 + std::fs::create_dir_all(&tmp_dir).ok();

394 + let tmp_path = tmp_dir.join("chat_template.jinja");

395 + std::fs::write(&tmp_path, template).map_err(|e| InferenceError::ModelBuild {

396 + reason: format!(

397 + "Failed to write chat template to {}: {}",

398 + tmp_path.display(),

399 + e

400 + ),

401 + })?;

402 + builder = builder.with_chat_template(tmp_path.to_string_lossy().to_string());

403 + Some(tmp_path)

404 + } else {

405 + None

406 + };

407 +

408 let model = builder

409 .build()

410 .await

@@ -555,6 +577,7 @@ impl ChatEngine {

577 tok_model_id,

578 display_name: resp.name.unwrap_or_else(|| hf_repo_id.to_string()),

579 approx_memory,

580 + chat_template: None,

581 };

582

583 self.load_gguf_model(config, system_prompt, sampling).await

@@ -1839,6 +1862,14 @@ impl Default for ChatEngine {

1862 // Prebuilt model configs (convenience constructors)

1863 // ═════════════════════════════════════════════════════════════════════════════

1864

1865 +/// DeepSeek Coder v1 Jinja chat template.

1866 +///

1867 +/// TheBloke's GGUF files do not embed a chat template — without an explicit

1868 +/// template, mistral.rs rejects chat requests with "model does not have a

1869 +/// chat template". This template reproduces the official DeepSeek Coder v1

1870 +/// `### Instruction: / ### Response:` format.

1871 +const DEEPSEEK_CODER_CHAT_TEMPLATE: &str = include_str!("deepseek_coder_chat_template.txt");

1872 +

1873 /// Convenience constructors for common GGUF model configurations.

1874 ///

1875 /// These mirror the constants in [`super::models`] but return a ready-to-use

@@ -1858,6 +1889,7 @@ impl GgufModelConfig {

1889 },

1890 display_name: "Qwen 2.5 1.5B".into(),

1891 approx_memory: "~941 MB (GGUF Q4_K_M)".into(),

1892 + chat_template: None,

1893 }

1894 }

1895

@@ -1876,6 +1908,7 @@ impl GgufModelConfig {

1908 },

1909 display_name: "Qwen 2.5 3B".into(),

1910 approx_memory: "~1.93 GB (GGUF Q4_K_M)".into(),

1911 + chat_template: None,

1912 }

1913 }

1914

@@ -1899,6 +1932,7 @@ impl GgufModelConfig {

1932 },

1933 display_name: "Qwen 2.5 Coder 1.5B".into(),

1934 approx_memory: "~941 MB (GGUF Q4_K_M)".into(),

1935 + chat_template: None,

1936 }

1937 }

1938

@@ -1918,6 +1952,7 @@ impl GgufModelConfig {

1952 },

1953 display_name: "Qwen 2.5 Coder 3B".into(),

1954 approx_memory: "~1.93 GB (GGUF Q4_K_M)".into(),

1955 + chat_template: None,

1956 }

1957 }

1958

@@ -1933,6 +1968,7 @@ impl GgufModelConfig {

1968 tok_model_id: None,

1969 display_name: "Qwen 3 4B (Q4_K_M)".into(),

1970 approx_memory: "~2.7 GB".into(),

1971 + chat_template: None,

1972 }

1973 }

1974

@@ -1944,6 +1980,7 @@ impl GgufModelConfig {

1980 tok_model_id: None,

1981 display_name: "Qwen 3 1.7B (Q4_K_M)".into(),

1982 approx_memory: "~1.3 GB".into(),

1983 + chat_template: None,

1984 }

1985 }

1986

@@ -1958,6 +1995,26 @@ impl GgufModelConfig {

1995 tok_model_id: None,

1996 display_name: "Qwen 3 8B (Q4_K_M)".into(),

1997 approx_memory: "~5 GB".into(),

1998 + chat_template: None,

1999 + }

2000 + }

2001 +

2002 + /// DeepSeek Coder v1 6.7B Instruct (GGUF Q4_K_M) — ~3.8 GB.

2003 + ///

2004 + /// Strong code generation model using the `llama` GGUF architecture.

2005 + /// Requires 8+ GB RAM; recommended for macOS desktops and Linux.

2006 + pub fn deepseek_coder_6_7b() -> Self {

2007 + Self {

2008 + model_id: super::models::THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF.into(),

2009 + files: vec![super::models::DEEPSEEK_CODER_6_7B_GGUF_FILE.into()],

2010 + tok_model_id: if cfg!(target_os = "android") {

2011 + Some(super::models::DEEPSEEK_CODER_6_7B_TOK_MODEL_ID.into())

2012 + } else {

2013 + None

2014 + },

2015 + display_name: "DeepSeek Coder 6.7B (Q4_K_M)".into(),

2016 + approx_memory: "~3.8 GB".into(),

2017 + chat_template: Some(DEEPSEEK_CODER_CHAT_TEMPLATE.into()),

2018 }

2019 }

2020

src/inference/models.rs

+22

@@ -89,6 +89,18 @@ pub const BARTOWSKI_QWEN3_1_7B_GGUF: &str = "bartowski/Qwen_Qwen3-1.7B-GGUF";

89 /// The specific GGUF filename for the Qwen3 1.7B repo.

90 pub const QWEN3_1_7B_GGUF_FILE: &str = "Qwen_Qwen3-1.7B-Q4_K_M.gguf";

92 +/// DeepSeek Coder v1 6.7B Instruct (GGUF Q4_K_M) — dedicated code generation model (~3.8 GB).

93 +///

94 +/// Uses the `llama` GGUF architecture, so it loads through the existing

95 +/// `quantized_llama.rs` path in mistral.rs without any code changes.

96 +/// Strong coding performance. Requires ~8 GB RAM.

97 +pub const THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF: &str =

98 + "TheBloke/deepseek-coder-6.7B-instruct-GGUF";

99 +/// The specific GGUF filename.

100 +pub const DEEPSEEK_CODER_6_7B_GGUF_FILE: &str = "deepseek-coder-6.7b-instruct.Q4_K_M.gguf";

101 +/// Base model repo used for the HF tokenizer on Android.

102 +pub const DEEPSEEK_CODER_6_7B_TOK_MODEL_ID: &str = "deepseek-ai/deepseek-coder-6.7b-instruct";

103 +

104 /// All model IDs that the Onde inference engine supports.

105 /// Used by `list_local_hf_models` to filter the HuggingFace cache

106 /// to only show models that can actually be used for generation.

@@ -98,6 +110,7 @@ pub const SUPPORTED_MODELS: &[&str] = &[

110 BARTOWSKI_QWEN3_4B_GGUF,

111 BARTOWSKI_QWEN3_8B_GGUF,

112 BARTOWSKI_QWEN3_1_7B_GGUF,

113 + THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF,

114 ];

115

116 /// Rich metadata for a supported model, used by the frontend to display

@@ -171,6 +184,14 @@ pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[

184 Smallest Qwen 3 variant with tool calling support.",

185 expected_size_bytes: 1_282_439_584,

186 },

187 + SupportedModelInfo {

188 + id: THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF,

189 + name: "DeepSeek Coder 6.7B (GGUF)",

190 + org: "DeepSeek AI",

191 + description: "Strong code generation model using the llama architecture (~3.8 GB). \

192 + Requires 8+ GB RAM. Not recommended for mobile devices.",

193 + expected_size_bytes: 4_083_015_904,

194 + },

195 ];

196

197 /// Return the explicit tokenizer model ID required on Android for `hf_repo_id`.

@@ -185,6 +206,7 @@ pub fn tok_model_id_for_repo(hf_repo_id: &str) -> Option<&'static str> {

206 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF => Some(QWEN25_3B_TOK_MODEL_ID),

207 BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF => Some(QWEN25_CODER_1_5B_TOK_MODEL_ID),

208 BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF => Some(QWEN25_CODER_3B_TOK_MODEL_ID),

209 + THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF => Some(DEEPSEEK_CODER_6_7B_TOK_MODEL_ID),

210 _ => None,

211 }

212 }

src/inference/types.rs

@@ -203,6 +203,10 @@ pub struct GgufModelConfig {

203 pub display_name: String,

204 /// Approximate memory footprint description, e.g. `"~941 MB (GGUF Q4_K_M)"`.

205 pub approx_memory: String,

206 + /// Optional Jinja chat template string. When set, this overrides the

207 + /// template embedded in the GGUF file (if any). Required for older GGUF

208 + /// files that don't embed a chat template (e.g. TheBloke models).

209 + pub chat_template: Option<String>,

210 }

211

212 // ── ISQ model configuration ──────────────────────────────────────────────────