[Rust Frontend] Stamp `arrival_time` at the frontend entry (#47787)
Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com>
Tahsin Tunan committed
Jul 7, 2026 at 09:27 UTC
34e6dfced82e3231e5e6f1a4eeafbc6d8fefad94
10 files changed
+63
-12
rust/src/chat/src/lib.rs
+4
@@ -172,6 +172,9 @@ impl ChatLlm {
172
pub async fn chat(&self, mut request: ChatRequest) -> Result<ChatEventStream> {
173
request.validate()?;
174
175
+ // Stamp before rendering so render and tokenize count toward TTFT/e2e.
176
+ let arrival_time = vllm_llm::current_unix_timestamp_secs();
177
+
178
let output_processor = self.backend.new_chat_output_processor(
179
&mut request,
180
NewChatOutputProcessorOptions {
@@ -210,6 +213,7 @@ impl ChatLlm {
213
data_parallel_rank: request.data_parallel_rank,
214
reasoning_parser_kwargs,
215
lora_request: request.lora_request,
216
+ arrival_time: Some(arrival_time),
217
};
218
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
219
rust/src/llm/src/lib.rs
+1
@@ -14,6 +14,7 @@ pub use output::{
14
GenerateOutputStreamExt, GeneratePromptInfo, TokenUsage,
15
};
16
pub use request::GenerateRequest;
17
+pub use request_metrics::current_unix_timestamp_secs;
18
pub use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob};
19
20
use crate::inflight::InflightRequests;
rust/src/llm/src/request.rs
+4
-10
@@ -1,5 +1,4 @@
1
use std::collections::BTreeMap;
2
-use std::time::{SystemTime, UNIX_EPOCH};
2
3
use uuid::Uuid;
4
use vllm_engine_core_client::protocol::lora::LoraRequest;
@@ -8,6 +7,7 @@ use vllm_engine_core_client::protocol::request::{EngineCoreRequest, ReasoningPar
7
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
8
9
use crate::error::{Error, Result};
10
+use crate::request_metrics::current_unix_timestamp_secs;
11
12
/// Tokenized decoder-only generate request accepted by [`crate::Llm`].
13
///
@@ -30,8 +30,9 @@ pub struct GenerateRequest {
30
pub mm_features: Option<MmFeatures>,
31
/// Unix timestamp, in seconds, when this request arrived at the frontend.
32
///
33
- /// When omitted, the Rust frontend fills it immediately before sending the
34
- /// request to engine-core, matching Python's default arrival-time behavior.
33
+ /// Stamped at the frontend entry, before render and tokenization, to match
34
+ /// Python's renderer-entry arrival_time. When omitted, it is filled as a
35
+ /// fallback before the request is sent to engine-core.
36
pub arrival_time: Option<f64>,
37
/// Optional salt used to partition prefix-cache entries for this request.
38
pub cache_salt: Option<String>,
@@ -122,13 +123,6 @@ impl PreparedGenerateRequest {
123
}
124
}
125
125
-fn current_unix_timestamp_secs() -> f64 {
126
- SystemTime::now()
127
- .duration_since(UNIX_EPOCH)
128
- .expect("system clock is before unix epoch")
129
- .as_secs_f64()
130
-}
131
-
126
#[cfg(test)]
127
mod tests {
128
use std::collections::BTreeMap;
rust/src/llm/src/request_metrics.rs
+1
-1
@@ -330,7 +330,7 @@ fn diff_or_zero(end: f64, start: f64) -> f64 {
330
///
331
/// Original Python request timestamp source:
332
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L206-L216>
333
-pub(crate) fn current_unix_timestamp_secs() -> f64 {
333
+pub fn current_unix_timestamp_secs() -> f64 {
334
SystemTime::now()
335
.duration_since(UNIX_EPOCH)
336
.expect("system clock is before unix epoch")
rust/src/server/src/grpc/convert.rs
+1
@@ -94,6 +94,7 @@ pub fn to_text_request(
94
data_parallel_rank: None,
95
reasoning_parser_kwargs: None,
96
lora_request: None,
97
+ arrival_time: None,
98
})
99
}
100
rust/src/server/src/routes/inference/generate/convert.rs
+1
@@ -70,6 +70,7 @@ pub(super) fn prepare_generate_request(
70
data_parallel_rank: ctx.data_parallel_rank,
71
reasoning_parser_kwargs: None,
72
lora_request: lora_resolution.lora_request.clone(),
73
+ arrival_time: None,
74
};
75
76
Ok(PreparedRequest {
rust/src/server/src/routes/openai/completions/convert.rs
+1
@@ -144,6 +144,7 @@ pub(super) fn prepare_completion_request(
144
data_parallel_rank: ctx.data_parallel_rank,
145
reasoning_parser_kwargs: None,
146
lora_request: lora_resolution.lora_request.clone(),
147
+ arrival_time: None,
148
};
149
150
Ok(PreparedRequest {
rust/src/text/src/lib.rs
+4
@@ -132,6 +132,10 @@ impl TextLlm {
132
) -> Result<(TextRequest, GenerateOutputStream)> {
133
request.validate()?;
134
135
+ if request.arrival_time.is_none() {
136
+ request.arrival_time = Some(vllm_llm::current_unix_timestamp_secs());
137
+ }
138
+
139
let tokenizer = self.backend.tokenizer();
140
let prompt_token_ids = match take(&mut request.prompt) {
141
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
rust/src/text/src/lower.rs
+39
-1
@@ -56,7 +56,7 @@ pub fn lower_text_request(
56
data_parallel_rank: request.data_parallel_rank,
57
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
58
lora_request: request.lora_request.clone(),
59
- arrival_time: None,
59
+ arrival_time: request.arrival_time,
60
trace_headers: None,
61
};
62
@@ -1143,6 +1143,44 @@ mod tests {
1143
assert_eq!(prepared.generate_request.request_id, "text-1");
1144
}
1145
1146
+ #[test]
1147
+ fn lower_text_request_passes_arrival_time_through() {
1148
+ let request = TextRequest {
1149
+ arrival_time: Some(42.5),
1150
+ ..sample_request()
1151
+ };
1152
+
1153
+ let prepared = lower_text_request(
1154
+ request,
1155
+ vec![1, 2, 3],
1156
+ sample_sampling_hints(),
1157
+ sample_sampling_limits(),
1158
+ &stub_tokenizer(),
1159
+ )
1160
+ .unwrap();
1161
+
1162
+ assert_eq!(prepared.generate_request.arrival_time, Some(42.5));
1163
+ }
1164
+
1165
+ #[test]
1166
+ fn lower_text_request_leaves_arrival_time_unset_when_absent() {
1167
+ let request = TextRequest {
1168
+ arrival_time: None,
1169
+ ..sample_request()
1170
+ };
1171
+
1172
+ let prepared = lower_text_request(
1173
+ request,
1174
+ vec![1, 2, 3],
1175
+ sample_sampling_hints(),
1176
+ sample_sampling_limits(),
1177
+ &stub_tokenizer(),
1178
+ )
1179
+ .unwrap();
1180
+
1181
+ assert_eq!(prepared.generate_request.arrival_time, None);
1182
+ }
1183
+
1184
#[test]
1185
fn resolve_max_tokens_user_smaller_than_model_limit() {
1186
let result = resolve_max_tokens(Some(50), None, 200, 100);
rust/src/text/src/request.rs
+7
@@ -187,6 +187,12 @@ pub struct TextRequest {
187
/// LoRA adapter selected for this request.
188
#[serde(default)]
189
pub lora_request: Option<LoraRequest>,
190
+ /// Wall-clock unix timestamp (seconds) when this request arrived at the
191
+ /// frontend, stamped before render/tokenize to match Python's
192
+ /// renderer-entry arrival_time. When unset, it is stamped before
193
+ /// tokenization.
194
+ #[serde(default)]
195
+ pub arrival_time: Option<f64>,
196
}
197
198
impl TextRequest {
@@ -205,6 +211,7 @@ impl TextRequest {
211
data_parallel_rank: None,
212
reasoning_parser_kwargs: None,
213
lora_request: None,
214
+ arrival_time: None,
215
}
216
}
217