A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V1] feat:add engine v1 tracing (#20372)
Signed-off-by: Mu Huai <tianbowen.tbw@antgroup.com> Signed-off-by: Ye Zhang <zhysishu@gmail.com> Signed-off-by: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Signed-off-by: simon-mo <simon.mo@hey.com> Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com> Co-authored-by: Mu Huai <tianbowen.tbw@antgroup.com> Co-authored-by: Ye Zhang <zhysishu@gmail.com> Co-authored-by: Benjamin Bartels <benjamin@bartels.dev> Co-authored-by: simon-mo <simon.mo@hey.com> Co-authored-by: 瑜琮 <ly186375@antfin.com> Co-authored-by: Aaron Pham <contact@aarnphm.xyz> Co-authored-by: 22quinn <33176974+22quinn@users.noreply.github.com>
RichardoMu committed
Sep 12, 2025 at 08:10 UTC
40b6c9122b5997b88f5463da15ec465707d45626
12 files changed
+253
-20
.buildkite/test-pipeline.yaml
+1
-1
@@ -231,7 +231,7 @@ steps:
231
source_file_dependencies:
232
- vllm/
233
- tests/metrics
234
- - tests/tracing
234
+ - tests/v1/tracing
235
commands:
236
- pytest -v -s metrics
237
- "pip install \
tests/v1/tracing/test_tracing.py
new
+137
@@ -0,0 +1,137 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# ruff: noqa
4
+# type: ignore
5
+from __future__ import annotations
6
+
7
+import threading
8
+from collections.abc import Iterable
9
+from concurrent import futures
10
+from typing import Callable, Generator, Literal
11
+
12
+import grpc
13
+import pytest
14
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
15
+ ExportTraceServiceResponse)
16
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import (
17
+ TraceServiceServicer, add_TraceServiceServicer_to_server)
18
+from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
19
+from opentelemetry.sdk.environment_variables import (
20
+ OTEL_EXPORTER_OTLP_TRACES_INSECURE)
21
+
22
+from vllm import LLM, SamplingParams
23
+from vllm.tracing import SpanAttributes
24
+
25
+FAKE_TRACE_SERVER_ADDRESS = "localhost:4317"
26
+
27
+FieldName = Literal['bool_value', 'string_value', 'int_value', 'double_value',
28
+ 'array_value']
29
+
30
+
31
+def decode_value(value: AnyValue):
32
+ field_decoders: dict[FieldName, Callable] = {
33
+ "bool_value": (lambda v: v.bool_value),
34
+ "string_value": (lambda v: v.string_value),
35
+ "int_value": (lambda v: v.int_value),
36
+ "double_value": (lambda v: v.double_value),
37
+ "array_value":
38
+ (lambda v: [decode_value(item) for item in v.array_value.values]),
39
+ }
40
+ for field, decoder in field_decoders.items():
41
+ if value.HasField(field):
42
+ return decoder(value)
43
+ raise ValueError(f"Couldn't decode value: {value}")
44
+
45
+
46
+def decode_attributes(attributes: Iterable[KeyValue]):
47
+ return {kv.key: decode_value(kv.value) for kv in attributes}
48
+
49
+
50
+class FakeTraceService(TraceServiceServicer):
51
+
52
+ def __init__(self):
53
+ self.request = None
54
+ self.evt = threading.Event()
55
+
56
+ def Export(self, request, context):
57
+ self.request = request
58
+ self.evt.set()
59
+ return ExportTraceServiceResponse()
60
+
61
+
62
+@pytest.fixture
63
+def trace_service() -> Generator[FakeTraceService, None, None]:
64
+ """Fixture to set up a fake gRPC trace service"""
65
+ server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
66
+ service = FakeTraceService()
67
+ add_TraceServiceServicer_to_server(service, server)
68
+ server.add_insecure_port(FAKE_TRACE_SERVER_ADDRESS)
69
+ server.start()
70
+
71
+ yield service
72
+
73
+ server.stop(None)
74
+
75
+
76
+def test_traces(
77
+ monkeypatch: pytest.MonkeyPatch,
78
+ trace_service: FakeTraceService,
79
+):
80
+ with monkeypatch.context() as m:
81
+ m.setenv(OTEL_EXPORTER_OTLP_TRACES_INSECURE, "true")
82
+ m.setenv("VLLM_USE_V1", "1")
83
+ sampling_params = SamplingParams(
84
+ temperature=0.01,
85
+ top_p=0.1,
86
+ max_tokens=256,
87
+ )
88
+ model = "facebook/opt-125m"
89
+ llm = LLM(model=model,
90
+ otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,
91
+ gpu_memory_utilization=0.3,
92
+ disable_log_stats=False)
93
+ prompts = ["This is a short prompt"]
94
+ outputs = llm.generate(prompts, sampling_params=sampling_params)
95
+ print(f"test_traces outputs is : {outputs}")
96
+
97
+ timeout = 10
98
+ if not trace_service.evt.wait(timeout):
99
+ raise TimeoutError(
100
+ f"The fake trace service didn't receive a trace within "
101
+ f"the {timeout} seconds timeout")
102
+
103
+ request = trace_service.request
104
+ assert len(request.resource_spans) == 1, (
105
+ f"Expected 1 resource span, "
106
+ f"but got {len(request.resource_spans)}")
107
+ assert len(request.resource_spans[0].scope_spans) == 1, (
108
+ f"Expected 1 scope span, "
109
+ f"but got {len(request.resource_spans[0].scope_spans)}")
110
+ assert len(request.resource_spans[0].scope_spans[0].spans) == 1, (
111
+ f"Expected 1 span, "
112
+ f"but got {len(request.resource_spans[0].scope_spans[0].spans)}")
113
+
114
+ attributes = decode_attributes(
115
+ request.resource_spans[0].scope_spans[0].spans[0].attributes)
116
+ # assert attributes.get(SpanAttributes.GEN_AI_RESPONSE_MODEL) == model
117
+ assert attributes.get(
118
+ SpanAttributes.GEN_AI_REQUEST_ID) == outputs[0].request_id
119
+ assert attributes.get(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE
120
+ ) == sampling_params.temperature
121
+ assert attributes.get(
122
+ SpanAttributes.GEN_AI_REQUEST_TOP_P) == sampling_params.top_p
123
+ assert attributes.get(SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS
124
+ ) == sampling_params.max_tokens
125
+ assert attributes.get(
126
+ SpanAttributes.GEN_AI_REQUEST_N) == sampling_params.n
127
+ assert attributes.get(
128
+ SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS) == len(
129
+ outputs[0].prompt_token_ids)
130
+ completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
131
+ assert attributes.get(
132
+ SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS) == completion_tokens
133
+
134
+ assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE) > 0
135
+ assert attributes.get(
136
+ SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN) > 0
137
+ assert attributes.get(SpanAttributes.GEN_AI_LATENCY_E2E) > 0
vllm/engine/arg_utils.py
-6
@@ -1491,12 +1491,6 @@ class EngineArgs:
1491
recommend_to_remove=False)
1492
return False
1493
1494
- # No OTLP observability so far.
1495
- if (self.otlp_traces_endpoint or self.collect_detailed_traces):
1496
- _raise_or_fallback(feature_name="--otlp-traces-endpoint",
1497
- recommend_to_remove=False)
1498
- return False
1499
-
1494
# V1 supports N-gram, Medusa, and Eagle speculative decoding.
1495
if (self.speculative_config is not None
1496
and self.speculative_config.get("method") == "draft_model"):
vllm/tracing.py
+5
@@ -119,6 +119,11 @@ class SpanAttributes:
119
# forward, block/sync across workers, cpu-gpu sync time and sampling time.
120
GEN_AI_LATENCY_TIME_IN_MODEL_EXECUTE = (
121
"gen_ai.latency.time_in_model_execute")
122
+ GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL = \
123
+ "gen_ai.latency.time_in_model_prefill"
124
+ GEN_AI_LATENCY_TIME_IN_MODEL_DECODE = "gen_ai.latency.time_in_model_decode"
125
+ GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE = \
126
+ "gen_ai.latency.time_in_model_inference"
127
128
129
def contains_trace_headers(headers: Mapping[str, str]) -> bool:
vllm/v1/core/sched/scheduler.py
+1
-1
@@ -969,9 +969,9 @@ class Scheduler(SchedulerInterface):
969
stop_reason=request.stop_reason,
970
events=request.take_events(),
971
kv_transfer_params=kv_transfer_params,
972
+ trace_headers=request.trace_headers,
973
num_cached_tokens=request.num_cached_tokens,
974
))
974
-
975
else:
976
# Invariant: EngineCore returns no partial prefill outputs.
977
assert not prompt_logprobs_tensors
vllm/v1/engine/__init__.py
+5
-1
@@ -3,6 +3,7 @@
3
4
import enum
5
import time
6
+from collections.abc import Mapping
7
from typing import Any, Optional, Union
8
9
import msgspec
@@ -66,6 +67,8 @@ class EngineCoreRequest(
67
current_wave: int = 0
68
priority: int = 0
69
70
+ trace_headers: Optional[Mapping[str, str]] = None
71
+
72
73
class EngineCoreEventType(enum.IntEnum):
74
"""The type of engine core request event."""
@@ -111,6 +114,7 @@ class EngineCoreOutput(
114
events: Optional[list[EngineCoreEvent]] = None
115
kv_transfer_params: Optional[dict[str, Any]] = None
116
117
+ trace_headers: Optional[Mapping[str, str]] = None
118
# The number of tokens with prefix cache hits.
119
num_cached_tokens: int = 0
120
@@ -144,7 +148,7 @@ class EngineCoreOutputs(
148
omit_defaults=True, # type: ignore[call-arg]
149
gc=False): # type: ignore[call-arg]
150
147
- #NOTE(Nick): We could consider ways to make this more compact,
151
+ # NOTE(Nick): We could consider ways to make this more compact,
152
# e.g. columnwise layout
153
154
engine_index: int = 0
vllm/v1/engine/async_llm.py
+8
-1
@@ -26,6 +26,7 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput
26
from vllm.pooling_params import PoolingParams
27
from vllm.sampling_params import SamplingParams
28
from vllm.tasks import SupportedTask
29
+from vllm.tracing import init_tracer
30
from vllm.transformers_utils.config import (
31
maybe_register_config_serialize_by_value)
32
from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -97,6 +98,7 @@ class AsyncLLM(EngineClient):
98
99
self.model_config = vllm_config.model_config
100
self.vllm_config = vllm_config
101
+ self.observability_config = vllm_config.observability_config
102
self.log_requests = log_requests
103
104
self.log_stats = log_stats or (stat_loggers is not None)
@@ -124,6 +126,11 @@ class AsyncLLM(EngineClient):
126
# OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
127
self.output_processor = OutputProcessor(self.tokenizer,
128
log_stats=self.log_stats)
129
+ if self.observability_config.otlp_traces_endpoint is not None:
130
+ tracer = init_tracer(
131
+ "vllm.llm_engine",
132
+ self.observability_config.otlp_traces_endpoint)
133
+ self.output_processor.tracer = tracer
134
135
# EngineCore (starts the engine in background process).
136
self.engine_core = EngineCoreClient.make_async_mp_client(
@@ -603,7 +610,7 @@ class AsyncLLM(EngineClient):
610
return self.tokenizer.get_lora_tokenizer(lora_request)
611
612
async def is_tracing_enabled(self) -> bool:
606
- return False
613
+ return self.observability_config.otlp_traces_endpoint is not None
614
615
async def do_log_stats(
616
self,
vllm/v1/engine/llm_engine.py
+7
@@ -19,6 +19,7 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput
19
from vllm.pooling_params import PoolingParams
20
from vllm.sampling_params import SamplingParams
21
from vllm.tasks import SupportedTask
22
+from vllm.tracing import init_tracer
23
from vllm.transformers_utils.tokenizer_group import (
24
TokenizerGroup, init_tokenizer_from_configs)
25
from vllm.usage.usage_lib import UsageContext
@@ -65,6 +66,7 @@ class LLMEngine:
66
"Set VLLM_USE_V1=0 and file and issue on Github.")
67
68
self.vllm_config = vllm_config
69
+ self.observability_config = vllm_config.observability_config
70
self.model_config = vllm_config.model_config
71
self.cache_config = vllm_config.cache_config
72
@@ -99,6 +101,11 @@ class LLMEngine:
101
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
102
self.output_processor = OutputProcessor(self.tokenizer,
103
log_stats=self.log_stats)
104
+ if self.observability_config.otlp_traces_endpoint is not None:
105
+ tracer = init_tracer(
106
+ "vllm.llm_engine",
107
+ self.observability_config.otlp_traces_endpoint)
108
+ self.output_processor.tracer = tracer
109
110
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
111
self.engine_core = EngineCoreClient.make_client(
vllm/v1/engine/output_processor.py
+79
-7
@@ -11,6 +11,8 @@ import torch
11
from vllm.outputs import (CompletionOutput, PoolingOutput,
12
PoolingRequestOutput, RequestOutput)
13
from vllm.sampling_params import RequestOutputKind
14
+from vllm.tracing import (SpanAttributes, SpanKind, Tracer,
15
+ extract_trace_context)
16
from vllm.transformers_utils.tokenizer import AnyTokenizer
17
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
18
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
@@ -71,7 +73,6 @@ class RequestOutputCollector:
73
74
@dataclass
75
class OutputProcessorOutput:
74
-
76
request_outputs: list[Union[RequestOutput, PoolingRequestOutput]]
77
reqs_to_abort: list[str]
78
@@ -93,6 +94,9 @@ class RequestState:
94
arrival_time: float,
95
queue: Optional[RequestOutputCollector],
96
log_stats: bool,
97
+ top_p: Optional[float] = None,
98
+ n: Optional[int] = None,
99
+ temperature: Optional[float] = None,
100
):
101
self.request_id = request_id
102
self.parent_req = parent_req
@@ -105,6 +109,9 @@ class RequestState:
109
self.logprobs_processor = logprobs_processor
110
self.detokenizer = detokenizer
111
self.max_tokens_param = max_tokens_param
112
+ self.top_p = top_p
113
+ self.n = n
114
+ self.temperature = temperature
115
self.is_prefilling = True
116
self.queue = queue
117
self.num_cached_tokens = 0
@@ -137,10 +144,16 @@ class RequestState:
144
request=request,
145
)
146
max_tokens_param = sampling_params.max_tokens
147
+ top_p = sampling_params.top_p
148
+ n = sampling_params.n
149
+ temperature = sampling_params.temperature
150
else:
151
logprobs_processor = None
152
detokenizer = None
153
max_tokens_param = None
154
+ top_p = None
155
+ n = None
156
+ temperature = None
157
assert request.pooling_params is not None
158
output_kind = request.pooling_params.output_kind
159
@@ -156,6 +169,9 @@ class RequestState:
169
logprobs_processor=logprobs_processor,
170
detokenizer=detokenizer,
171
max_tokens_param=max_tokens_param,
172
+ top_p=top_p,
173
+ n=n,
174
+ temperature=temperature,
175
arrival_time=request.arrival_time,
176
queue=queue,
177
log_stats=log_stats,
@@ -274,16 +290,13 @@ class RequestState:
290
class OutputProcessor:
291
"""Process EngineCoreOutputs into RequestOutputs."""
292
277
- def __init__(
278
- self,
279
- tokenizer: TokenizerGroup,
280
- log_stats: bool,
281
- ):
293
+ def __init__(self, tokenizer: TokenizerGroup, log_stats: bool):
294
self.log_stats = log_stats
295
self.tokenizer = tokenizer
296
self.request_states: dict[str, RequestState] = {}
297
self.parent_requests: dict[str, ParentRequest] = {}
298
self.lora_states = LoRARequestStates()
299
+ self.tracer: Optional[Tracer] = None
300
301
def get_num_unfinished_requests(self):
302
return len(self.request_states)
@@ -441,7 +454,9 @@ class OutputProcessor:
454
# Track per-request stats
455
self._update_stats_from_finished(req_state, finish_reason,
456
iteration_stats)
444
-
457
+ if self.tracer:
458
+ self.do_tracing(engine_core_output, req_state,
459
+ iteration_stats)
460
self.lora_states.update_iteration_stats(iteration_stats)
461
462
return OutputProcessorOutput(
@@ -449,6 +464,63 @@ class OutputProcessor:
464
reqs_to_abort=reqs_to_abort,
465
)
466
467
+ def do_tracing(self, engine_core_output: EngineCoreOutput,
468
+ req_state: RequestState,
469
+ iteration_stats: Optional[IterationStats]) -> None:
470
+ assert req_state.stats is not None
471
+ assert iteration_stats is not None
472
+ assert self.tracer is not None
473
+
474
+ arrival_time_nano_seconds = int(req_state.stats.arrival_time * 1e9)
475
+ trace_context = extract_trace_context(engine_core_output.trace_headers)
476
+ with (self.tracer.start_as_current_span(
477
+ "llm_request",
478
+ kind=SpanKind.SERVER,
479
+ context=trace_context,
480
+ start_time=arrival_time_nano_seconds) as span):
481
+ metrics = req_state.stats
482
+ e2e_time = iteration_stats.iteration_timestamp - \
483
+ metrics.arrival_time
484
+ queued_time = metrics.scheduled_ts - metrics.queued_ts
485
+ prefill_time = metrics.first_token_ts - metrics.scheduled_ts
486
+ decode_time = metrics.last_token_ts - metrics.first_token_ts
487
+ inference_time = metrics.last_token_ts - metrics.scheduled_ts
488
+ span.set_attribute(
489
+ SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN,
490
+ metrics.first_token_latency)
491
+ span.set_attribute(SpanAttributes.GEN_AI_LATENCY_E2E, e2e_time)
492
+ span.set_attribute(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE,
493
+ queued_time)
494
+ span.set_attribute(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS,
495
+ len(req_state.prompt_token_ids))
496
+ span.set_attribute(SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS,
497
+ metrics.num_generation_tokens)
498
+ span.set_attribute(
499
+ SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL,
500
+ prefill_time)
501
+ span.set_attribute(
502
+ SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE,
503
+ decode_time)
504
+ span.set_attribute(
505
+ SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE,
506
+ inference_time)
507
+
508
+ # meta
509
+ span.set_attribute(SpanAttributes.GEN_AI_REQUEST_ID,
510
+ req_state.request_id)
511
+ if req_state.top_p:
512
+ span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TOP_P,
513
+ req_state.top_p)
514
+ if req_state.max_tokens_param:
515
+ span.set_attribute(SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS,
516
+ req_state.max_tokens_param)
517
+ if req_state.temperature:
518
+ span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE,
519
+ req_state.temperature)
520
+ if req_state.n:
521
+ span.set_attribute(SpanAttributes.GEN_AI_REQUEST_N,
522
+ req_state.n)
523
+
524
def _update_stats_from_output(self, req_state: RequestState,
525
engine_core_output: EngineCoreOutput,
526
engine_core_timestamp: Optional[float],
vllm/v1/engine/processor.py
+1
-2
@@ -327,8 +327,6 @@ class Processor:
327
# TODO(woosuk): Support pooling models.
328
self._validate_lora(lora_request)
329
self._validate_params(params, lora_request)
330
- if trace_headers is not None:
331
- raise ValueError("V1 does not support tracing yet.")
330
331
data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
332
if data_parallel_rank is not None and not (0 <= data_parallel_rank <
@@ -435,6 +433,7 @@ class Processor:
433
cache_salt=decoder_inputs.get("cache_salt"),
434
priority=priority,
435
data_parallel_rank=data_parallel_rank,
436
+ trace_headers=trace_headers,
437
)
438
439
def _validate_model_inputs(self,
vllm/v1/metrics/stats.py
+4
@@ -68,6 +68,9 @@ class RequestStateStats:
68
first_token_ts: float = 0.0
69
last_token_ts: float = 0.0
70
71
+ # first token latency
72
+ first_token_latency: float = 0.0
73
+
74
75
@dataclass
76
class FinishedRequestStats:
@@ -116,6 +119,7 @@ class IterationStats:
119
120
first_token_latency = self._time_since(req_stats.arrival_time)
121
self.time_to_first_tokens_iter.append(first_token_latency)
122
+ req_stats.first_token_latency = first_token_latency
123
124
req_stats.num_generation_tokens += num_new_generation_tokens
125
vllm/v1/request.py
+5
-1
@@ -3,6 +3,7 @@
3
4
import enum
5
import time
6
+from collections.abc import Mapping
7
from functools import partial
8
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
9
@@ -35,6 +36,7 @@ class Request:
36
structured_output_request: Optional["StructuredOutputRequest"] = None,
37
cache_salt: Optional[str] = None,
38
priority: int = 0,
39
+ trace_headers: Optional[Mapping[str, str]] = None,
40
block_hasher: Optional[Callable[["Request"],
41
list["BlockHash"]]] = None,
42
) -> None:
@@ -100,7 +102,8 @@ class Request:
102
# they should also be updated simultaneously.
103
self.output_token_ids = ConstantList(self._output_token_ids)
104
self.all_token_ids = ConstantList(self._all_token_ids)
103
-
105
+ # trace_headers
106
+ self.trace_headers = trace_headers
107
# State
108
# The number of tokens with prefix cache hits.
109
self.num_cached_tokens = -1
@@ -136,6 +139,7 @@ class Request:
139
if request.sampling_params else None,
140
cache_salt=request.cache_salt,
141
priority=request.priority,
142
+ trace_headers=request.trace_headers,
143
block_hasher=block_hasher,
144
)
145