A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[Bugfix] Fix None value handling in trace span creation for cancelled requests (#20272)
Bram committed
Jul 30, 2025 at 14:44 UTC
601f856d5679a474b6488fb7dd75ebbd7125d1ca
1 file changed
+20
-7
vllm/engine/llm_engine.py
+20
-7
@@ -1862,8 +1862,14 @@ class LLMEngine:
1862
context=trace_context,
1863
start_time=arrival_time_nano_seconds) as seq_span:
1864
metrics = seq_group.metrics
1865
- ttft = metrics.first_token_time - metrics.arrival_time
1866
- e2e_time = metrics.finished_time - metrics.arrival_time
1865
+
1866
+ # Handle potential None values for cancelled/aborted requests
1867
+ ttft = (metrics.first_token_time - metrics.arrival_time
1868
+ if metrics.first_token_time is not None else None)
1869
+
1870
+ e2e_time = (metrics.finished_time - metrics.arrival_time
1871
+ if metrics.finished_time is not None else None)
1872
+
1873
seq_span.set_attribute(SpanAttributes.GEN_AI_RESPONSE_MODEL,
1874
self.model_config.model)
1875
seq_span.set_attribute(SpanAttributes.GEN_AI_REQUEST_ID,
@@ -1886,11 +1892,18 @@ class LLMEngine:
1892
seq.get_output_len()
1893
for seq in seq_group.get_finished_seqs()
1894
]))
1889
- seq_span.set_attribute(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE,
1890
- metrics.time_in_queue)
1891
- seq_span.set_attribute(
1892
- SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
1893
- seq_span.set_attribute(SpanAttributes.GEN_AI_LATENCY_E2E, e2e_time)
1895
+
1896
+ # Only set timing attributes if the values are available
1897
+ if metrics.time_in_queue is not None:
1898
+ seq_span.set_attribute(
1899
+ SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE,
1900
+ metrics.time_in_queue)
1901
+ if ttft is not None:
1902
+ seq_span.set_attribute(
1903
+ SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
1904
+ if e2e_time is not None:
1905
+ seq_span.set_attribute(SpanAttributes.GEN_AI_LATENCY_E2E,
1906
+ e2e_time)
1907
if metrics.scheduler_time is not None:
1908
seq_span.set_attribute(
1909
SpanAttributes.GEN_AI_LATENCY_TIME_IN_SCHEDULER,