A high-throughput and memory-efficient inference and serving engine for LLMs
[Rust Frontend] Add serde defaults for omit_defaults fields in `EngineCoreSamplingParams` (#45848)
Signed-off-by: Will Eaton <weaton@redhat.com>
Will Eaton committed
Jun 17, 2026 at 02:40 UTC
17bc1445562435b608041d434e9738440954159c
3 files changed
+121
-13
rust/src/engine-core-client/src/protocol/mod.rs
+70
-11
@@ -44,6 +44,14 @@ fn default_repetition_penalty() -> f32 {
44
1.0
45
}
46
47
+fn default_temperature() -> f32 {
48
+ 1.0
49
+}
50
+
51
+fn default_max_tokens() -> u32 {
52
+ 16
53
+}
54
+
55
mod classified_outputs;
56
pub mod dtype;
57
pub mod handshake;
@@ -246,24 +254,28 @@ pub struct StructuredOutputsParams {
254
///
255
/// Original Python definition:
256
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
257
+// Python's SamplingParams is `omit_defaults=True`, so msgpack drops
258
+// default-valued keys; default the whole struct. Per-field fns cover the
259
+// non-zero defaults.
260
#[serde_with::skip_serializing_none]
250
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
261
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)]
262
+#[serde(default)]
263
pub struct EngineCoreSamplingParams {
264
/// Controls randomness. Lower values are more deterministic; zero means
265
/// greedy sampling.
266
+ #[serde(default = "default_temperature")]
267
pub temperature: f32,
268
/// Cumulative probability threshold for nucleus sampling.
269
#[serde(default = "default_top_p")]
270
pub top_p: f32,
271
/// Maximum number of top tokens to consider. `0` means all tokens.
259
- #[serde(default)]
272
pub top_k: u32,
273
/// Random seed used by the sampler when present.
274
pub seed: Option<i64>,
275
/// Maximum number of tokens to generate per output sequence.
276
+ #[serde(default = "default_max_tokens")]
277
pub max_tokens: u32,
278
/// Minimum number of tokens to generate before EOS or stop-token handling.
266
- #[serde(default)]
279
pub min_tokens: u32,
280
/// Number of log probabilities to return per generated token.
281
///
@@ -274,7 +286,6 @@ pub struct EngineCoreSamplingParams {
286
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
287
pub prompt_logprobs: Option<i32>,
288
/// Minimum probability threshold for token sampling.
277
- #[serde(default)]
289
pub min_p: f32,
290
/// Frequency penalty applied by the sampler.
291
pub frequency_penalty: f32,
@@ -301,16 +312,13 @@ pub struct EngineCoreSamplingParams {
312
pub all_stop_token_ids: BTreeSet<u32>,
313
/// Logit biases to apply during sampling.
314
/// Keys are token IDs
304
- #[serde(default)]
315
pub logit_bias: Option<HashMap<u32, f32>>,
316
/// Restrict output to these token IDs only.
307
- #[serde(default)]
317
pub allowed_token_ids: Option<Vec<u32>>,
318
/// Tokenized bad words to avoid during generation.
310
- #[serde(default, rename = "_bad_words_token_ids")]
319
+ #[serde(rename = "_bad_words_token_ids")]
320
pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
321
/// Parameters for configuring structured outputs (guided decoding).
313
- #[serde(default)]
322
pub structured_outputs: Option<StructuredOutputsParams>,
323
/// Specific token IDs for which log probabilities should be returned at
324
/// each position.
@@ -318,15 +326,12 @@ pub struct EngineCoreSamplingParams {
326
/// When set, the engine returns logprobs for exactly these tokens in
327
/// addition to the sampled/scored token. Mutually exclusive with the
328
/// `logprobs` count field in practice.
321
- #[serde(default)]
329
pub logprob_token_ids: Option<Vec<u32>>,
330
/// If `Some(true)`, the request will not attempt to read from the prefix
331
/// cache; newly computed blocks may still populate the cache. `None`
332
/// defers to engine-core defaults.
326
- #[serde(default)]
333
pub skip_reading_prefix_cache: Option<bool>,
334
/// Additional request parameters for custom extensions (from `vllm_xargs`).
329
- #[serde(default)]
335
pub extra_args: Option<HashMap<String, serde_json::Value>>,
336
}
337
@@ -640,4 +645,58 @@ mod tests {
645
let value = serde_json::to_value(params).unwrap();
646
assert_eq!(value["_backend"], "guidance");
647
}
648
+
649
+ /// A real `sampling_params` is a sparse `omit_defaults` map; absent fields
650
+ /// must fall back to defaults. `python_compat` can't catch this since Rust
651
+ /// encodes full maps (see `engine_core_request_serializes_as_full_array`).
652
+ #[test]
653
+ fn decodes_sampling_params_with_omitted_defaults() {
654
+ let sampling_params = Value::Map(vec![
655
+ (
656
+ Value::from("stop_token_ids"),
657
+ Value::Array(vec![Value::from(151643u32)]),
658
+ ),
659
+ (Value::from("skip_reading_prefix_cache"), Value::from(false)),
660
+ ]);
661
+ let request = Value::Array(vec![
662
+ Value::from("req-omit-defaults"),
663
+ Value::Array(vec![
664
+ Value::from(1u32),
665
+ Value::from(2u32),
666
+ Value::from(3u32),
667
+ ]),
668
+ Value::Nil,
669
+ sampling_params,
670
+ Value::Nil,
671
+ Value::from(1.0f64),
672
+ ]);
673
+
674
+ let mut bytes = Vec::new();
675
+ rmpv::encode::write_value(&mut bytes, &request).unwrap();
676
+
677
+ let decoded: EngineCoreRequest = decode_msgpack(&bytes)
678
+ .expect("a real omit_defaults request must decode (regression: missing field)");
679
+
680
+ assert_eq!(decoded.request_id, "req-omit-defaults");
681
+ let sampling = decoded.sampling_params.expect("sampling params present");
682
+
683
+ assert_eq!(sampling.stop_token_ids, vec![151643]);
684
+ assert_eq!(sampling.skip_reading_prefix_cache, Some(false));
685
+
686
+ // Omitted fields -> Python defaults.
687
+ assert_eq!(sampling.temperature, 1.0);
688
+ assert_eq!(sampling.top_p, 1.0);
689
+ assert_eq!(sampling.top_k, 0);
690
+ assert_eq!(sampling.seed, None);
691
+ assert_eq!(sampling.max_tokens, 16);
692
+ assert_eq!(sampling.min_tokens, 0);
693
+ assert_eq!(sampling.min_p, 0.0);
694
+ assert_eq!(sampling.frequency_penalty, 0.0);
695
+ assert_eq!(sampling.presence_penalty, 0.0);
696
+ assert_eq!(sampling.repetition_penalty, 1.0);
697
+ assert_eq!(sampling.logprobs, None);
698
+ assert_eq!(sampling.prompt_logprobs, None);
699
+ assert_eq!(sampling.eos_token_id, None);
700
+ assert!(sampling.all_stop_token_ids.is_empty());
701
+ }
702
}
rust/src/engine-core-client/src/tests/client.rs
+37
@@ -2438,6 +2438,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2438
let stdout = String::from_utf8(output.stdout).unwrap();
2439
let mut lines = stdout.lines();
2440
let request_hex = lines.next().expect("missing request fixture line");
2441
+ let defaults_request_hex = lines.next().expect("missing defaults request fixture line");
2442
let multimodal_request_hex = lines.next().expect("missing multimodal request fixture line");
2443
let outputs_hex = lines.next().expect("missing outputs fixture line");
2444
let inline_logprobs_frames = lines.next().expect("missing inline logprobs fixture line");
@@ -2455,6 +2456,42 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2456
let expected_request = sample_request();
2457
assert_eq!(decoded_request, expected_request);
2458
2459
+ // All-default sampling params -> empty map; must decode to Python defaults.
2460
+ let defaults_request_bytes = hex::decode(defaults_request_hex).unwrap();
2461
+ let decoded_defaults: EngineCoreRequest =
2462
+ rmp_serde::from_slice(&defaults_request_bytes).unwrap();
2463
+ assert_eq!(decoded_defaults.request_id, "req-defaults");
2464
+ let sampling = decoded_defaults
2465
+ .sampling_params
2466
+ .expect("defaults request carries sampling params");
2467
+ assert_eq!(
2468
+ sampling,
2469
+ EngineCoreSamplingParams {
2470
+ temperature: 1.0,
2471
+ top_p: 1.0,
2472
+ top_k: 0,
2473
+ seed: None,
2474
+ max_tokens: 16,
2475
+ min_tokens: 0,
2476
+ logprobs: None,
2477
+ prompt_logprobs: None,
2478
+ min_p: 0.0,
2479
+ frequency_penalty: 0.0,
2480
+ presence_penalty: 0.0,
2481
+ repetition_penalty: 1.0,
2482
+ stop_token_ids: Vec::new(),
2483
+ eos_token_id: None,
2484
+ all_stop_token_ids: BTreeSet::new(),
2485
+ logit_bias: None,
2486
+ allowed_token_ids: None,
2487
+ bad_words_token_ids: None,
2488
+ structured_outputs: None,
2489
+ logprob_token_ids: None,
2490
+ skip_reading_prefix_cache: None,
2491
+ extra_args: None,
2492
+ },
2493
+ );
2494
+
2495
let decoded_multimodal_request: EngineCoreRequest =
2496
rmp_serde::from_slice(&multimodal_request_bytes).unwrap();
2497
assert_eq!(decoded_multimodal_request, sample_multimodal_request());
rust/src/engine-core-client/src/tests/python_compat.py
+14
-2
@@ -31,12 +31,13 @@ class FinishReason(IntEnum):
31
REPETITION = 4
32
33
34
-class EngineCoreSamplingParams(msgspec.Struct, dict=True):
34
+# Mirror of real SamplingParams; omit_defaults makes fixtures match real maps.
35
+class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True):
36
temperature: float = 1.0
37
top_p: float = 1.0
38
top_k: int = 0
39
seed: int | None = None
39
- max_tokens: int = 65536
40
+ max_tokens: int = 16
41
min_tokens: int = 0
42
min_p: float = 0.0
43
frequency_penalty: float = 0.0
@@ -135,6 +136,16 @@ request = EngineCoreRequest(
136
client_index=0,
137
)
138
139
+# All defaults -> empty map. Regression guard for the sparse-map decode.
140
+defaults_request = EngineCoreRequest(
141
+ request_id="req-defaults",
142
+ prompt_token_ids=[5, 6, 7],
143
+ mm_features=None,
144
+ sampling_params=EngineCoreSamplingParams(),
145
+ pooling_params=None,
146
+ arrival_time=1.0,
147
+)
148
+
149
multimodal_tensor = np.array([[1.0, 2.0], [3.5, 4.25]], dtype=np.float32)
150
multimodal_features = [
151
{
@@ -361,6 +372,7 @@ ready_response = EngineCoreReadyResponse(
372
)
373
374
print(msgspec.msgpack.encode(request).hex())
375
+print(msgspec.msgpack.encode(defaults_request).hex())
376
print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
377
print(msgspec.msgpack.encode(outputs).hex())
378
print(" ".join(frame.hex() for frame in encode_output_frames(inline_logprobs)))