@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 325ab6b0a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 19:00 UTC

[Feature] OTEL tracing during loading (#31162)

emricksini-h committed Feb 6, 2026 at 01:59 UTC 325ab6b0a896f4b483b26d99afdfc6d75ea61074
29 files changed +873 -280
requirements/common.txt
+1 -1
@@ -52,4 +52,4 @@ anthropic >= 0.71.0
52 model-hosting-container-standards >= 0.1.13, < 1.0.0
53 mcp
54 grpcio
55 -grpcio-reflection
55 +grpcio-reflection
\ No newline at end of file
setup.py
+7
@@ -1049,6 +1049,13 @@ setup(
1049 "petit-kernel": ["petit-kernel"],
1050 # Optional deps for Helion kernel development
1051 "helion": ["helion"],
1052 + # Optional deps for OpenTelemetry tracing
1053 + "otel": [
1054 + "opentelemetry-sdk>=1.26.0",
1055 + "opentelemetry-api>=1.26.0",
1056 + "opentelemetry-exporter-otlp>=1.26.0",
1057 + "opentelemetry-semantic-conventions-ai>=0.4.1",
1058 + ],
1059 },
1060 cmdclass=cmdclass,
1061 package_data=package_data,
tests/tracing/__init__.py
tests/tracing/conftest.py new
+127
@@ -0,0 +1,127 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import threading
4 +from collections.abc import Callable, Generator, Iterable
5 +from concurrent import futures
6 +from typing import Any, Literal
7 +
8 +import grpc
9 +import pytest
10 +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
11 + ExportTraceServiceRequest,
12 + ExportTraceServiceResponse,
13 +)
14 +from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import (
15 + TraceServiceServicer,
16 + add_TraceServiceServicer_to_server,
17 +)
18 +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
19 +
20 +FAKE_TRACE_SERVER_ADDRESS = "localhost:4317"
21 +
22 +FieldName = Literal[
23 + "bool_value", "string_value", "int_value", "double_value", "array_value"
24 +]
25 +
26 +
27 +def decode_value(value: AnyValue):
28 + """Decode an OpenTelemetry AnyValue protobuf message to a Python value."""
29 + field_decoders: dict[FieldName, Callable] = {
30 + "bool_value": (lambda v: v.bool_value),
31 + "string_value": (lambda v: v.string_value),
32 + "int_value": (lambda v: v.int_value),
33 + "double_value": (lambda v: v.double_value),
34 + "array_value": (
35 + lambda v: [decode_value(item) for item in v.array_value.values]
36 + ),
37 + }
38 + for field, decoder in field_decoders.items():
39 + if value.HasField(field):
40 + return decoder(value)
41 + raise ValueError(f"Couldn't decode value: {value}")
42 +
43 +
44 +def decode_attributes(attributes: Iterable[KeyValue]) -> dict[str, Any]:
45 + """Decode OpenTelemetry KeyValue attributes to a Python dictionary."""
46 + return {kv.key: decode_value(kv.value) for kv in attributes}
47 +
48 +
49 +class FakeTraceService(TraceServiceServicer):
50 + """A fake gRPC trace service for testing OpenTelemetry trace exports."""
51 +
52 + def __init__(self):
53 + self.requests: list[ExportTraceServiceRequest] = []
54 + self.evt = threading.Event()
55 + self._lock = threading.Lock()
56 +
57 + def Export(self, request, context):
58 + with self._lock:
59 + self.requests.append(request)
60 + self.evt.set()
61 + return ExportTraceServiceResponse()
62 +
63 + @property
64 + def request(self) -> ExportTraceServiceRequest | None:
65 + """Returns the first request received (for backward compatibility)."""
66 + with self._lock:
67 + return self.requests[0] if self.requests else None
68 +
69 + def get_all_spans(self) -> list[dict]:
70 + """Returns all spans from all received requests as decoded dicts."""
71 + spans = []
72 + with self._lock:
73 + for request in self.requests:
74 + for resource_span in request.resource_spans:
75 + for scope_span in resource_span.scope_spans:
76 + for span in scope_span.spans:
77 + spans.append(
78 + {
79 + "name": span.name,
80 + "attributes": decode_attributes(span.attributes),
81 + "trace_id": span.trace_id.hex(),
82 + "span_id": span.span_id.hex(),
83 + "parent_span_id": span.parent_span_id.hex()
84 + if span.parent_span_id
85 + else None,
86 + "start_time_unix_nano": span.start_time_unix_nano,
87 + "end_time_unix_nano": span.end_time_unix_nano,
88 + }
89 + )
90 + return spans
91 +
92 + def wait_for_spans(self, count: int = 1, timeout: float = 10) -> bool:
93 + """Wait until at least `count` spans have been received."""
94 + import time
95 +
96 + deadline = time.time() + timeout
97 + while time.time() < deadline:
98 + if len(self.get_all_spans()) >= count:
99 + return True
100 + time.sleep(0.1)
101 + return False
102 +
103 + def clear(self):
104 + """Clear all received requests."""
105 + with self._lock:
106 + self.requests.clear()
107 + self.evt.clear()
108 +
109 +
110 +@pytest.fixture
111 +def trace_service() -> Generator[FakeTraceService, None, None]:
112 + """Fixture to set up a fake gRPC trace service."""
113 + server = grpc.server(futures.ThreadPoolExecutor(max_workers=2))
114 + service = FakeTraceService()
115 + add_TraceServiceServicer_to_server(service, server)
116 + server.add_insecure_port(FAKE_TRACE_SERVER_ADDRESS)
117 + server.start()
118 +
119 + yield service
120 +
121 + server.stop(grace=None)
122 +
123 +
124 +@pytest.fixture
125 +def trace_server_address() -> str:
126 + """Returns the address of the fake trace server."""
127 + return FAKE_TRACE_SERVER_ADDRESS
tests/tracing/test_loading_tracing.py new
+87
@@ -0,0 +1,87 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import asyncio
4 +
5 +import pytest
6 +from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_TRACES_INSECURE
7 +
8 +from tests.tracing.conftest import FAKE_TRACE_SERVER_ADDRESS, FakeTraceService
9 +from vllm.tracing import init_tracer, instrument, is_otel_available
10 +
11 +# Skip everything if OTel is missing
12 +pytestmark = pytest.mark.skipif(not is_otel_available(), reason="OTel required")
13 +
14 +
15 +class TestCoreInstrumentation:
16 + """Focuses on the @instrument decorator's ability to capture execution data."""
17 +
18 + @pytest.fixture(autouse=True)
19 + def setup_tracing(self, monkeypatch):
20 + monkeypatch.setenv(OTEL_EXPORTER_OTLP_TRACES_INSECURE, "true")
21 + init_tracer("test.core", FAKE_TRACE_SERVER_ADDRESS)
22 +
23 + def test_decorator_captures_sync_and_async(self, trace_service: FakeTraceService):
24 + """Verify basic span creation for both sync and async functions."""
25 +
26 + @instrument(span_name="sync_task")
27 + def sync_task():
28 + return True
29 +
30 + @instrument(span_name="async_task")
31 + async def async_task():
32 + return True
33 +
34 + sync_task()
35 + asyncio.run(async_task())
36 +
37 + assert trace_service.wait_for_spans(count=2)
38 + span_names = [s["name"] for s in trace_service.get_all_spans()]
39 + assert "sync_task" in span_names
40 + assert "async_task" in span_names
41 +
42 + def test_nested_spans_hierarchy(self, trace_service: FakeTraceService):
43 + """Verify that nested calls create a parent-child relationship."""
44 +
45 + @instrument(span_name="child")
46 + def child():
47 + pass
48 +
49 + @instrument(span_name="parent")
50 + def parent():
51 + child()
52 +
53 + parent()
54 +
55 + assert trace_service.wait_for_spans(count=2)
56 + spans = trace_service.get_all_spans()
57 + parent_span = next(s for s in spans if s["name"] == "parent")
58 + child_span = next(s for s in spans if s["name"] == "child")
59 +
60 + assert child_span["parent_span_id"] == parent_span["span_id"]
61 +
62 +
63 +class TestInterProcessPropagation:
64 + """Test the propagation of trace context between processes."""
65 +
66 + def test_pickup_external_context(self, monkeypatch, trace_service):
67 + """Test that vLLM attaches to an existing trace ID if in environment."""
68 + monkeypatch.setenv(OTEL_EXPORTER_OTLP_TRACES_INSECURE, "true")
69 +
70 + # Manually simulate an external parent trace ID
71 + fake_trace_id = "4bf92f3577b34da6a3ce929d0e0e4736"
72 + fake_parent_id = "00f067aa0ba902b7"
73 + monkeypatch.setenv("traceparent", f"00-{fake_trace_id}-{fake_parent_id}-01")
74 +
75 + init_tracer("test.external", FAKE_TRACE_SERVER_ADDRESS)
76 +
77 + @instrument(span_name="follower")
78 + def follower_func():
79 + pass
80 +
81 + follower_func()
82 +
83 + assert trace_service.wait_for_spans(count=1)
84 + span = trace_service.get_all_spans()[0]
85 +
86 + assert span["trace_id"] == fake_trace_id
87 + assert span["parent_span_id"] == fake_parent_id
tests/v1/tracing/test_tracing.py
+24 -85
@@ -2,76 +2,19 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 # ruff: noqa
4 # type: ignore
5 -import threading
6 -from collections.abc import Iterable
7 -from concurrent import futures
8 -from typing import Callable, Generator, Literal
9 -
10 -import grpc
5 import pytest
12 -from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
13 - ExportTraceServiceResponse,
14 -)
15 -from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import (
16 - TraceServiceServicer,
17 - add_TraceServiceServicer_to_server,
18 -)
19 -from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
6 +import time
7 from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_TRACES_INSECURE
8
9 from vllm import LLM, SamplingParams
10 from vllm.tracing import SpanAttributes
11
25 -FAKE_TRACE_SERVER_ADDRESS = "localhost:4317"
26 -
27 -FieldName = Literal[
28 - "bool_value", "string_value", "int_value", "double_value", "array_value"
29 -]
30 -
31 -
32 -def decode_value(value: AnyValue):
33 - field_decoders: dict[FieldName, Callable] = {
34 - "bool_value": (lambda v: v.bool_value),
35 - "string_value": (lambda v: v.string_value),
36 - "int_value": (lambda v: v.int_value),
37 - "double_value": (lambda v: v.double_value),
38 - "array_value": (
39 - lambda v: [decode_value(item) for item in v.array_value.values]
40 - ),
41 - }
42 - for field, decoder in field_decoders.items():
43 - if value.HasField(field):
44 - return decoder(value)
45 - raise ValueError(f"Couldn't decode value: {value}")
46 -
47 -
48 -def decode_attributes(attributes: Iterable[KeyValue]):
49 - return {kv.key: decode_value(kv.value) for kv in attributes}
50 -
51 -
52 -class FakeTraceService(TraceServiceServicer):
53 - def __init__(self):
54 - self.request = None
55 - self.evt = threading.Event()
56 -
57 - def Export(self, request, context):
58 - self.request = request
59 - self.evt.set()
60 - return ExportTraceServiceResponse()
61 -
62 -
63 -@pytest.fixture
64 -def trace_service() -> Generator[FakeTraceService, None, None]:
65 - """Fixture to set up a fake gRPC trace service"""
66 - server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
67 - service = FakeTraceService()
68 - add_TraceServiceServicer_to_server(service, server)
69 - server.add_insecure_port(FAKE_TRACE_SERVER_ADDRESS)
70 - server.start()
71 -
72 - yield service
73 -
74 - server.stop(None)
12 +# Import shared fixtures from the tracing conftest
13 +from tests.tracing.conftest import ( # noqa: F401
14 + FAKE_TRACE_SERVER_ADDRESS,
15 + FakeTraceService,
16 + trace_service,
17 +)
18
19
20 def test_traces(
@@ -97,29 +40,25 @@ def test_traces(
40 outputs = llm.generate(prompts, sampling_params=sampling_params)
41 print(f"test_traces outputs is : {outputs}")
42
100 - timeout = 10
101 - if not trace_service.evt.wait(timeout):
102 - raise TimeoutError(
103 - f"The fake trace service didn't receive a trace within "
104 - f"the {timeout} seconds timeout"
105 - )
106 -
107 - request = trace_service.request
108 - assert len(request.resource_spans) == 1, (
109 - f"Expected 1 resource span, but got {len(request.resource_spans)}"
110 - )
111 - assert len(request.resource_spans[0].scope_spans) == 1, (
112 - f"Expected 1 scope span, "
113 - f"but got {len(request.resource_spans[0].scope_spans)}"
114 - )
115 - assert len(request.resource_spans[0].scope_spans[0].spans) == 1, (
116 - f"Expected 1 span, "
117 - f"but got {len(request.resource_spans[0].scope_spans[0].spans)}"
43 + # Wait for the "llm_request" span to be exported.
44 + # The BatchSpanProcessor batches spans and exports them periodically,
45 + # so we need to wait specifically for the llm_request span to appear.
46 + timeout = 15
47 + deadline = time.time() + timeout
48 + llm_request_spans = []
49 + while time.time() < deadline:
50 + all_spans = trace_service.get_all_spans()
51 + llm_request_spans = [s for s in all_spans if s["name"] == "llm_request"]
52 + if llm_request_spans:
53 + break
54 + time.sleep(0.5)
55 +
56 + assert len(llm_request_spans) == 1, (
57 + f"Expected exactly 1 'llm_request' span, but got {len(llm_request_spans)}. "
58 + f"All span names: {[s['name'] for s in all_spans]}"
59 )
60
120 - attributes = decode_attributes(
121 - request.resource_spans[0].scope_spans[0].spans[0].attributes
122 - )
61 + attributes = llm_request_spans[0]["attributes"]
62 # assert attributes.get(SpanAttributes.GEN_AI_RESPONSE_MODEL) == model
63 assert attributes.get(SpanAttributes.GEN_AI_REQUEST_ID) == outputs[0].request_id
64 assert (
vllm/compilation/backends.py
+8
@@ -33,6 +33,7 @@ from vllm.config.utils import Range, hash_factors
33 from vllm.logger import init_logger
34 from vllm.logging_utils import lazy
35 from vllm.platforms import current_platform
36 +from vllm.tracing import instrument, instrument_manual
37 from vllm.utils.import_utils import resolve_obj_by_qualname
38
39 from .compiler_interface import (
@@ -234,6 +235,7 @@ class CompilerManager:
235 )
236 return compiled_graph
237
238 + @instrument(span_name="Compile graph")
239 def compile(
240 self,
241 graph: fx.GraphModule,
@@ -497,6 +499,7 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter): # type: ignore[misc]
499 # When True, it annoyingly dumps the torch.fx.Graph on errors.
500 self.extra_traceback = False
501
502 + @instrument(span_name="Inductor compilation")
503 def run(self, *args: Any) -> Any:
504 # maybe instead just assert inputs are fake?
505 fake_args = [
@@ -922,6 +925,11 @@ class VllmBackend:
925 )
926 self.compilation_config.compilation_time += dynamo_time
927
928 + # Record Dynamo time in tracing if available
929 + start_time = int(torch_compile_start_time * 1e9)
930 + attributes = {"dynamo.time_seconds": dynamo_time}
931 + instrument_manual("Dynamo bytecode transform", start_time, None, attributes)
932 +
933 # we control the compilation process, each instance can only be
934 # called once
935 assert not self._called, "VllmBackend can only be called once"
vllm/config/observability.py
+2 -2
@@ -122,9 +122,9 @@ class ObservabilityConfig:
122 @classmethod
123 def _validate_otlp_traces_endpoint(cls, value: str | None) -> str | None:
124 if value is not None:
125 - from vllm.tracing import is_otel_available, otel_import_error_traceback
125 + from vllm.tracing import is_tracing_available, otel_import_error_traceback
126
127 - if not is_otel_available():
127 + if not is_tracing_available():
128 raise ValueError(
129 "OpenTelemetry is not available. Unable to configure "
130 "'otlp_traces_endpoint'. Ensure OpenTelemetry packages are "
vllm/entrypoints/openai/api_server.py
+2
@@ -50,6 +50,7 @@ from vllm.logger import init_logger
50 from vllm.reasoning import ReasoningParserManager
51 from vllm.tasks import POOLING_TASKS, SupportedTask
52 from vllm.tool_parsers import ToolParserManager
53 +from vllm.tracing import instrument
54 from vllm.usage.usage_lib import UsageContext
55 from vllm.utils.argparse_utils import FlexibleArgumentParser
56 from vllm.utils.network_utils import is_valid_ipv6_address
@@ -377,6 +378,7 @@ def validate_api_server_args(args):
378 )
379
380
381 +@instrument(span_name="API server setup")
382 def setup_server(args):
383 """Validate API server args, set up signal handler, create socket
384 ready to serve."""
vllm/model_executor/model_loader/base_loader.py
+2
@@ -14,6 +14,7 @@ from vllm.model_executor.model_loader.utils import (
14 process_weights_after_loading,
15 )
16 from vllm.platforms import current_platform
17 +from vllm.tracing import instrument
18 from vllm.utils.mem_utils import format_gib
19 from vllm.utils.torch_utils import set_default_torch_dtype
20
@@ -37,6 +38,7 @@ class BaseModelLoader(ABC):
38 inplace weights loading for an already-initialized model"""
39 raise NotImplementedError
40
41 + @instrument(span_name="Load model")
42 def load_model(
43 self, vllm_config: VllmConfig, model_config: ModelConfig, prefix: str = ""
44 ) -> nn.Module:
vllm/model_executor/model_loader/default_loader.py
+2
@@ -30,6 +30,7 @@ from vllm.model_executor.model_loader.weight_utils import (
30 pt_weights_iterator,
31 safetensors_weights_iterator,
32 )
33 +from vllm.tracing import instrument
34 from vllm.transformers_utils.repo_utils import list_filtered_repo_files
35
36 logger = init_logger(__name__)
@@ -274,6 +275,7 @@ class DefaultModelLoader(BaseModelLoader):
275 allow_patterns_overrides=None,
276 )
277
278 + @instrument(span_name="Load weights")
279 def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
280 if model_config.quantization == "torchao":
281 quant_config = get_quant_config(model_config, self.load_config)
vllm/model_executor/model_loader/utils.py
+2
@@ -23,11 +23,13 @@ from vllm.model_executor.model_loader.reload import (
23 set_torchao_reload_attrs,
24 )
25 from vllm.model_executor.models.interfaces import SupportsQuant
26 +from vllm.tracing import instrument
27 from vllm.utils.platform_utils import is_pin_memory_available
28
29 logger = init_logger(__name__)
30
31
32 +@instrument(span_name="Initialize model")
33 def initialize_model(
34 vllm_config: VllmConfig,
35 *,
vllm/model_executor/model_loader/weight_utils.py
+2
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.quantization import (
36 get_quantization_config,
37 )
38 from vllm.platforms import current_platform
39 +from vllm.tracing import instrument
40 from vllm.utils.import_utils import PlaceholderModule
41
42 try:
@@ -443,6 +444,7 @@ def download_gguf(
444 return local_files[0]
445
446
447 +@instrument(span_name="Download weights - HF")
448 def download_weights_from_hf(
449 model_name_or_path: str,
450 cache_dir: str | None,
vllm/model_executor/warmup/deep_gemm_warmup.py
+2
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.triton_deep_gemm_moe import (
19 )
20 from vllm.model_executor.layers.linear import LinearBase
21 from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
22 +from vllm.tracing import instrument
23 from vllm.utils.deep_gemm import (
24 fp8_gemm_nt,
25 get_mk_alignment_for_contiguous_layout,
@@ -358,6 +359,7 @@ def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
359 return total
360
361
362 +@instrument(span_name="DeepGemm warmup")
363 def deep_gemm_warmup(model: torch.nn.Module, max_tokens: int):
364 total = _count_warmup_iterations(model, max_tokens)
365 if total == 0:
vllm/tracing.py deleted
-135
@@ -1,135 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import os
5 -from collections.abc import Mapping
6 -
7 -from vllm.logger import init_logger
8 -from vllm.utils.func_utils import run_once
9 -
10 -TRACE_HEADERS = ["traceparent", "tracestate"]
11 -
12 -logger = init_logger(__name__)
13 -
14 -_is_otel_imported = False
15 -otel_import_error_traceback: str | None = None
16 -try:
17 - from opentelemetry.context.context import Context
18 - from opentelemetry.sdk.environment_variables import (
19 - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL,
20 - )
21 - from opentelemetry.sdk.trace import TracerProvider
22 - from opentelemetry.sdk.trace.export import BatchSpanProcessor
23 - from opentelemetry.trace import SpanKind, Tracer, set_tracer_provider
24 - from opentelemetry.trace.propagation.tracecontext import (
25 - TraceContextTextMapPropagator,
26 - )
27 -
28 - _is_otel_imported = True
29 -except ImportError:
30 - # Capture and format traceback to provide detailed context for the import
31 - # error. Only the string representation of the error is retained to avoid
32 - # memory leaks.
33 - # See https://github.com/vllm-project/vllm/pull/7266#discussion_r1707395458
34 - import traceback
35 -
36 - otel_import_error_traceback = traceback.format_exc()
37 -
38 - class Context: # type: ignore
39 - pass
40 -
41 - class BaseSpanAttributes: # type: ignore
42 - pass
43 -
44 - class SpanKind: # type: ignore
45 - pass
46 -
47 - class Tracer: # type: ignore
48 - pass
49 -
50 -
51 -def is_otel_available() -> bool:
52 - return _is_otel_imported
53 -
54 -
55 -def init_tracer(
56 - instrumenting_module_name: str, otlp_traces_endpoint: str
57 -) -> Tracer | None:
58 - if not is_otel_available():
59 - raise ValueError(
60 - "OpenTelemetry is not available. Unable to initialize "
61 - "a tracer. Ensure OpenTelemetry packages are installed. "
62 - f"Original error:\n{otel_import_error_traceback}"
63 - )
64 - trace_provider = TracerProvider()
65 -
66 - span_exporter = get_span_exporter(otlp_traces_endpoint)
67 - trace_provider.add_span_processor(BatchSpanProcessor(span_exporter))
68 - set_tracer_provider(trace_provider)
69 -
70 - tracer = trace_provider.get_tracer(instrumenting_module_name)
71 - return tracer
72 -
73 -
74 -def get_span_exporter(endpoint):
75 - protocol = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "grpc")
76 - if protocol == "grpc":
77 - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
78 - OTLPSpanExporter,
79 - )
80 - elif protocol == "http/protobuf":
81 - from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
82 - OTLPSpanExporter, # type: ignore
83 - )
84 - else:
85 - raise ValueError(f"Unsupported OTLP protocol '{protocol}' is configured")
86 -
87 - return OTLPSpanExporter(endpoint=endpoint)
88 -
89 -
90 -def extract_trace_context(headers: Mapping[str, str] | None) -> Context | None:
91 - if is_otel_available():
92 - headers = headers or {}
93 - return TraceContextTextMapPropagator().extract(headers)
94 - else:
95 - return None
96 -
97 -
98 -def extract_trace_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
99 - return {h: headers[h] for h in TRACE_HEADERS if h in headers}
100 -
101 -
102 -class SpanAttributes:
103 - # Attribute names copied from here to avoid version conflicts:
104 - # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md
105 - GEN_AI_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens"
106 - GEN_AI_USAGE_PROMPT_TOKENS = "gen_ai.usage.prompt_tokens"
107 - GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
108 - GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p"
109 - GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"
110 - GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"
111 - # Attribute names added until they are added to the semantic conventions:
112 - GEN_AI_REQUEST_ID = "gen_ai.request.id"
113 - GEN_AI_REQUEST_N = "gen_ai.request.n"
114 - GEN_AI_USAGE_NUM_SEQUENCES = "gen_ai.usage.num_sequences"
115 - GEN_AI_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
116 - GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token"
117 - GEN_AI_LATENCY_E2E = "gen_ai.latency.e2e"
118 - GEN_AI_LATENCY_TIME_IN_SCHEDULER = "gen_ai.latency.time_in_scheduler"
119 - # Time taken in the forward pass for this across all workers
120 - GEN_AI_LATENCY_TIME_IN_MODEL_FORWARD = "gen_ai.latency.time_in_model_forward"
121 - # Time taken in the model execute function. This will include model
122 - # forward, block/sync across workers, cpu-gpu sync time and sampling time.
123 - GEN_AI_LATENCY_TIME_IN_MODEL_EXECUTE = "gen_ai.latency.time_in_model_execute"
124 - GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL = "gen_ai.latency.time_in_model_prefill"
125 - GEN_AI_LATENCY_TIME_IN_MODEL_DECODE = "gen_ai.latency.time_in_model_decode"
126 - GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE = "gen_ai.latency.time_in_model_inference"
127 -
128 -
129 -def contains_trace_headers(headers: Mapping[str, str]) -> bool:
130 - return any(h in headers for h in TRACE_HEADERS)
131 -
132 -
133 -@run_once
134 -def log_tracing_disabled_warning() -> None:
135 - logger.warning("Received a request with trace context but tracing is disabled")
vllm/tracing/__init__.py new
+157
@@ -0,0 +1,157 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import functools
5 +from collections.abc import Callable
6 +from typing import Any, TypeAlias
7 +
8 +# Import the implementation details
9 +from .otel import (
10 + SpanKind,
11 + extract_trace_context,
12 + init_otel_tracer,
13 + init_otel_worker_tracer,
14 + instrument_otel,
15 + is_otel_available,
16 + manual_instrument_otel,
17 + otel_import_error_traceback,
18 +)
19 +from .utils import (
20 + SpanAttributes,
21 + contains_trace_headers,
22 + extract_trace_headers,
23 + log_tracing_disabled_warning,
24 +)
25 +
26 +__all__ = [
27 + "instrument",
28 + "instrument_manual",
29 + "init_tracer",
30 + "maybe_init_worker_tracer",
31 + "is_tracing_available",
32 + "SpanAttributes",
33 + "SpanKind",
34 + "extract_trace_context",
35 + "extract_trace_headers",
36 + "log_tracing_disabled_warning",
37 + "contains_trace_headers",
38 + "otel_import_error_traceback",
39 +]
40 +
41 +BackendAvailableFunc: TypeAlias = Callable[[], bool]
42 +InstrumentFunc: TypeAlias = Callable[..., Any]
43 +InstrumentManualFunc: TypeAlias = Callable[..., Any]
44 +InitTracerFunc: TypeAlias = Callable[..., Any]
45 +InitWorkerTracerFunc: TypeAlias = Callable[..., Any]
46 +_REGISTERED_TRACING_BACKENDS: dict[
47 + str,
48 + tuple[
49 + BackendAvailableFunc,
50 + InitTracerFunc,
51 + InitWorkerTracerFunc,
52 + InstrumentFunc,
53 + InstrumentManualFunc,
54 + ],
55 +] = {
56 + "otel": (
57 + is_otel_available,
58 + init_otel_tracer,
59 + init_otel_worker_tracer,
60 + instrument_otel,
61 + manual_instrument_otel,
62 + ),
63 +}
64 +
65 +
66 +def init_tracer(
67 + instrumenting_module_name: str,
68 + otlp_traces_endpoint: str,
69 + extra_attributes: dict[str, str] | None = None,
70 +):
71 + is_available, init_tracer_fn, _, _, _ = _REGISTERED_TRACING_BACKENDS["otel"]
72 + if is_available():
73 + return init_tracer_fn(
74 + instrumenting_module_name, otlp_traces_endpoint, extra_attributes
75 + )
76 +
77 +
78 +def maybe_init_worker_tracer(
79 + instrumenting_module_name: str,
80 + process_kind: str,
81 + process_name: str,
82 +):
83 + is_available, _, init_worker_tracer_fn, _, _ = _REGISTERED_TRACING_BACKENDS["otel"]
84 + if is_available():
85 + return init_worker_tracer_fn(
86 + instrumenting_module_name, process_kind, process_name
87 + )
88 +
89 +
90 +def instrument(
91 + obj: Callable | None = None,
92 + *,
93 + span_name: str = "",
94 + attributes: dict[str, str] | None = None,
95 + record_exception: bool = True,
96 +):
97 + """
98 + Generic decorator to instrument functions.
99 + """
100 + if obj is None:
101 + return functools.partial(
102 + instrument,
103 + span_name=span_name,
104 + attributes=attributes,
105 + record_exception=record_exception,
106 + )
107 +
108 + # Dispatch to OTel (and potentially others later)
109 + is_available, _, _, otel_instrument, _ = _REGISTERED_TRACING_BACKENDS["otel"]
110 + if is_available():
111 + return otel_instrument(
112 + func=obj,
113 + span_name=span_name,
114 + attributes=attributes,
115 + record_exception=record_exception,
116 + )
117 + else:
118 + return obj
119 +
120 +
121 +def instrument_manual(
122 + span_name: str,
123 + start_time: int,
124 + end_time: int | None = None,
125 + attributes: dict[str, Any] | None = None,
126 + context: Any = None,
127 + kind: Any = None,
128 +):
129 + """Manually create a span with explicit timestamps.
130 +
131 + Args:
132 + span_name: Name of the span to create.
133 + start_time: Start time in nanoseconds since epoch.
134 + end_time: Optional end time in nanoseconds. If None, ends immediately.
135 + attributes: Optional dict of span attributes.
136 + context: Optional trace context (e.g., from extract_trace_context).
137 + kind: Optional SpanKind (e.g., SpanKind.SERVER).
138 + """
139 + is_available, _, _, _, manual_instrument_fn = _REGISTERED_TRACING_BACKENDS["otel"]
140 + if is_available():
141 + return manual_instrument_fn(
142 + span_name, start_time, end_time, attributes, context, kind
143 + )
144 + else:
145 + return None
146 +
147 +
148 +def is_tracing_available() -> bool:
149 + """
150 + Returns True if any tracing backend (OTel, Profiler, etc.) is available.
151 + Use this to guard expensive tracing logic in the main code.
152 + """
153 + check_available = [
154 + is_available
155 + for is_available, _, _, _, _ in _REGISTERED_TRACING_BACKENDS.values()
156 + ]
157 + return any(check_available)
vllm/tracing/otel.py new
+265
@@ -0,0 +1,265 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import atexit
5 +import functools
6 +import inspect
7 +import os
8 +import traceback
9 +from collections.abc import Mapping
10 +from contextlib import contextmanager
11 +from typing import Any
12 +
13 +from vllm.logger import init_logger
14 +from vllm.tracing.utils import TRACE_HEADERS, LoadingSpanAttributes
15 +
16 +logger = init_logger(__name__)
17 +
18 +try:
19 + from opentelemetry import trace
20 + from opentelemetry.context.context import Context
21 + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
22 + OTLPSpanExporter as OTLPGrpcExporter,
23 + )
24 + from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
25 + OTLPSpanExporter as OTLPHttpExporter,
26 + )
27 + from opentelemetry.propagate import inject
28 + from opentelemetry.sdk.environment_variables import (
29 + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL,
30 + )
31 + from opentelemetry.sdk.resources import Resource
32 + from opentelemetry.sdk.trace import TracerProvider
33 + from opentelemetry.sdk.trace.export import BatchSpanProcessor
34 + from opentelemetry.trace import (
35 + SpanKind, # noqa: F401
36 + Tracer,
37 + set_tracer_provider,
38 + )
39 + from opentelemetry.trace.propagation.tracecontext import (
40 + TraceContextTextMapPropagator,
41 + )
42 +
43 + _IS_OTEL_AVAILABLE = True
44 + otel_import_error_traceback = None
45 +except ImportError:
46 + _IS_OTEL_AVAILABLE = False
47 + otel_import_error_traceback = traceback.format_exc()
48 + trace = None # type: ignore
49 + Context = Any # type: ignore
50 + Tracer = Any # type: ignore
51 + inject = None # type: ignore
52 + Resource = None # type: ignore
53 + SpanKind = Any # type: ignore
54 +
55 +
56 +def is_otel_available() -> bool:
57 + return _IS_OTEL_AVAILABLE
58 +
59 +
60 +def init_otel_tracer(
61 + instrumenting_module_name: str,
62 + otlp_traces_endpoint: str,
63 + extra_attributes: dict[str, str] | None = None,
64 +) -> Tracer:
65 + """Initializes the OpenTelemetry tracer provider."""
66 + if not _IS_OTEL_AVAILABLE:
67 + raise ValueError(
68 + "OpenTelemetry is not available. Unable to initialize "
69 + "a tracer. Ensure OpenTelemetry packages are installed. "
70 + f"Original error:\n{otel_import_error_traceback}"
71 + )
72 +
73 + # Store the endpoint in environment so child processes can inherit it
74 + os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = otlp_traces_endpoint
75 +
76 + resource_attrs = {}
77 + resource_attrs["vllm.instrumenting_module_name"] = instrumenting_module_name
78 + resource_attrs["vllm.process_id"] = str(os.getpid())
79 + if extra_attributes:
80 + resource_attrs.update(extra_attributes)
81 + resource = Resource.create(resource_attrs)
82 +
83 + trace_provider = TracerProvider(resource=resource)
84 + span_exporter = get_span_exporter(otlp_traces_endpoint)
85 + trace_provider.add_span_processor(BatchSpanProcessor(span_exporter))
86 + set_tracer_provider(trace_provider)
87 +
88 + atexit.register(trace_provider.shutdown)
89 +
90 + tracer = trace_provider.get_tracer(instrumenting_module_name)
91 + return tracer
92 +
93 +
94 +def get_span_exporter(endpoint):
95 + protocol = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "grpc")
96 + if protocol == "grpc":
97 + exporter = OTLPGrpcExporter(endpoint=endpoint, insecure=True)
98 + elif protocol == "http/protobuf":
99 + exporter = OTLPHttpExporter(endpoint=endpoint)
100 + else:
101 + raise ValueError(f"Unsupported OTLP protocol '{protocol}' is configured")
102 + return exporter
103 +
104 +
105 +def init_otel_worker_tracer(
106 + instrumenting_module_name: str,
107 + process_kind: str,
108 + process_name: str,
109 +) -> Tracer:
110 + """
111 + Backend-specific initialization for OpenTelemetry in a worker process.
112 + """
113 + # Initialize the tracer if an OTLP endpoint is configured.
114 + # The endpoint is propagated via environment variable from the main process.
115 + otlp_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
116 + if not otlp_endpoint:
117 + return None
118 +
119 + extra_attrs = {
120 + "vllm.process_kind": process_kind,
121 + "vllm.process_name": process_name,
122 + }
123 +
124 + return init_otel_tracer(instrumenting_module_name, otlp_endpoint, extra_attrs)
125 +
126 +
127 +def extract_trace_context(headers: Mapping[str, str] | None) -> Context | None:
128 + """Extracts context from HTTP headers."""
129 + if _IS_OTEL_AVAILABLE and headers:
130 + return TraceContextTextMapPropagator().extract(headers)
131 + return None
132 +
133 +
134 +def instrument_otel(func, span_name, attributes, record_exception):
135 + """Internal wrapper logic for sync and async functions."""
136 +
137 + # Pre-calculate static code attributes once (these don't change)
138 + code_attrs = {
139 + LoadingSpanAttributes.CODE_FUNCTION: func.__qualname__,
140 + LoadingSpanAttributes.CODE_NAMESPACE: func.__module__,
141 + LoadingSpanAttributes.CODE_FILEPATH: func.__code__.co_filename,
142 + LoadingSpanAttributes.CODE_LINENO: str(func.__code__.co_firstlineno),
143 + }
144 + if attributes:
145 + code_attrs.update(attributes)
146 +
147 + final_span_name = span_name or func.__qualname__
148 + module_name = func.__module__
149 +
150 + @functools.wraps(func)
151 + async def async_wrapper(*args, **kwargs):
152 + tracer = trace.get_tracer(module_name)
153 + ctx = _get_smart_context()
154 + with (
155 + tracer.start_as_current_span(
156 + final_span_name,
157 + context=ctx,
158 + attributes=code_attrs,
159 + record_exception=record_exception,
160 + ),
161 + propagate_trace_to_env(),
162 + ):
163 + return await func(*args, **kwargs)
164 +
165 + @functools.wraps(func)
166 + def sync_wrapper(*args, **kwargs):
167 + tracer = trace.get_tracer(module_name)
168 + ctx = _get_smart_context()
169 + with (
170 + tracer.start_as_current_span(
171 + final_span_name,
172 + context=ctx,
173 + attributes=code_attrs,
174 + record_exception=record_exception,
175 + ),
176 + propagate_trace_to_env(),
177 + ):
178 + return func(*args, **kwargs)
179 +
180 + return async_wrapper if inspect.iscoroutinefunction(func) else sync_wrapper
181 +
182 +
183 +def manual_instrument_otel(
184 + span_name: str,
185 + start_time: int,
186 + end_time: int | None = None,
187 + attributes: dict[str, Any] | None = None,
188 + context: Context | None = None,
189 + kind: Any = None, # SpanKind, but typed as Any for when OTEL unavailable
190 +):
191 + """Manually create and end a span with explicit timestamps."""
192 + if not _IS_OTEL_AVAILABLE:
193 + return
194 +
195 + tracer = trace.get_tracer(__name__)
196 + # Use provided context, or fall back to smart context detection
197 + ctx = context if context is not None else _get_smart_context()
198 +
199 + span_kwargs: dict[str, Any] = {
200 + "name": span_name,
201 + "context": ctx,
202 + "start_time": start_time,
203 + }
204 + if kind is not None:
205 + span_kwargs["kind"] = kind
206 +
207 + span = tracer.start_span(**span_kwargs)
208 + if attributes:
209 + span.set_attributes(attributes)
210 + if end_time is not None:
211 + span.end(end_time=end_time)
212 + else:
213 + span.end()
214 +
215 +
216 +def _get_smart_context() -> Context | None:
217 + """
218 + Determines the parent context.
219 + 1. If a Span is already active in this process, use it.
220 + 2. If not, extract from os.environ, handling the case-sensitivity mismatch.
221 + """
222 + current_span = trace.get_current_span()
223 + if current_span.get_span_context().is_valid:
224 + return None
225 +
226 + carrier = {}
227 +
228 + if tp := os.environ.get("traceparent", os.environ.get("TRACEPARENT")): # noqa: SIM112
229 + carrier["traceparent"] = tp
230 +
231 + if ts := os.environ.get("tracestate", os.environ.get("TRACESTATE")): # noqa: SIM112
232 + carrier["tracestate"] = ts
233 +
234 + if not carrier:
235 + carrier = dict(os.environ)
236 +
237 + return TraceContextTextMapPropagator().extract(carrier)
238 +
239 +
240 +@contextmanager
241 +def propagate_trace_to_env():
242 + """
243 + Temporarily injects the current OTel context into os.environ.
244 + This ensures that any subprocesses (like vLLM workers) spawned
245 + within this context inherit the correct traceparent.
246 + """
247 + if not _IS_OTEL_AVAILABLE:
248 + yield
249 + return
250 +
251 + # Capture original state of relevant keys
252 + original_state = {k: os.environ.get(k) for k in TRACE_HEADERS}
253 +
254 + try:
255 + # inject() writes 'traceparent' and 'tracestate' to os.environ
256 + inject(os.environ)
257 + yield
258 +
259 + finally:
260 + # Restore original environment
261 + for key, original_value in original_state.items():
262 + if original_value is None:
263 + os.environ.pop(key, None)
264 + else:
265 + os.environ[key] = original_value
vllm/tracing/utils.py new
+72
@@ -0,0 +1,72 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Mapping
5 +
6 +from vllm.logger import init_logger
7 +from vllm.utils.func_utils import run_once
8 +
9 +logger = init_logger(__name__)
10 +
11 +# Standard W3C headers used for context propagation
12 +TRACE_HEADERS = ["traceparent", "tracestate"]
13 +
14 +
15 +class SpanAttributes:
16 + """
17 + Standard attributes for spans.
18 +
19 + These are largely based on OpenTelemetry Semantic Conventions but are defined
20 + here as constants so they can be used by any backend or logger.
21 + """
22 +
23 + # Attribute names copied from OTel semantic conventions to avoid version conflicts
24 + GEN_AI_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens"
25 + GEN_AI_USAGE_PROMPT_TOKENS = "gen_ai.usage.prompt_tokens"
26 + GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
27 + GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p"
28 + GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"
29 + GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"
30 +
31 + # Custom attributes added until they are standardized
32 + GEN_AI_REQUEST_ID = "gen_ai.request.id"
33 + GEN_AI_REQUEST_N = "gen_ai.request.n"
34 + GEN_AI_USAGE_NUM_SEQUENCES = "gen_ai.usage.num_sequences"
35 + GEN_AI_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
36 + GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token"
37 + GEN_AI_LATENCY_E2E = "gen_ai.latency.e2e"
38 + GEN_AI_LATENCY_TIME_IN_SCHEDULER = "gen_ai.latency.time_in_scheduler"
39 +
40 + # Latency breakdowns
41 + GEN_AI_LATENCY_TIME_IN_MODEL_FORWARD = "gen_ai.latency.time_in_model_forward"
42 + GEN_AI_LATENCY_TIME_IN_MODEL_EXECUTE = "gen_ai.latency.time_in_model_execute"
43 + GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL = "gen_ai.latency.time_in_model_prefill"
44 + GEN_AI_LATENCY_TIME_IN_MODEL_DECODE = "gen_ai.latency.time_in_model_decode"
45 + GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE = "gen_ai.latency.time_in_model_inference"
46 +
47 +
48 +class LoadingSpanAttributes:
49 + """Custom attributes for code-level tracing (file, line number)."""
50 +
51 + CODE_NAMESPACE = "code.namespace"
52 + CODE_FUNCTION = "code.function"
53 + CODE_FILEPATH = "code.filepath"
54 + CODE_LINENO = "code.lineno"
55 +
56 +
57 +def contains_trace_headers(headers: Mapping[str, str]) -> bool:
58 + """Check if the provided headers dictionary contains trace context."""
59 + return any(h in headers for h in TRACE_HEADERS)
60 +
61 +
62 +def extract_trace_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
63 + """
64 + Extract only trace-related headers from a larger header dictionary.
65 + Useful for logging or passing context to a non-OTel client.
66 + """
67 + return {h: headers[h] for h in TRACE_HEADERS if h in headers}
68 +
69 +
70 +@run_once
71 +def log_tracing_disabled_warning() -> None:
72 + logger.warning("Received a request with trace context but tracing is disabled")
vllm/v1/engine/async_llm.py
+6 -4
@@ -110,6 +110,10 @@ class AsyncLLM(EngineClient):
110 self.model_config = vllm_config.model_config
111 self.vllm_config = vllm_config
112 self.observability_config = vllm_config.observability_config
113 + tracing_endpoint = self.observability_config.otlp_traces_endpoint
114 + if tracing_endpoint is not None:
115 + init_tracer("vllm.llm_engine", tracing_endpoint)
116 +
117 self.log_requests = log_requests
118
119 custom_stat_loggers = list(stat_loggers or [])
@@ -136,10 +140,8 @@ class AsyncLLM(EngineClient):
140 log_stats=self.log_stats,
141 stream_interval=self.vllm_config.scheduler_config.stream_interval,
142 )
139 - endpoint = self.observability_config.otlp_traces_endpoint
140 - if endpoint is not None:
141 - tracer = init_tracer("vllm.llm_engine", endpoint)
142 - self.output_processor.tracer = tracer
143 + if tracing_endpoint is not None:
144 + self.output_processor.tracing_enabled = True
145
146 # EngineCore (starts the engine in background process).
147 self.engine_core = EngineCoreClient.make_async_mp_client(
vllm/v1/engine/core.py
+21
@@ -24,6 +24,7 @@ from vllm.logging_utils.dump_input import dump_engine_exception
24 from vllm.lora.request import LoRARequest
25 from vllm.multimodal import MULTIMODAL_REGISTRY
26 from vllm.tasks import POOLING_TASKS, SupportedTask
27 +from vllm.tracing import instrument, maybe_init_worker_tracer
28 from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
29 from vllm.utils.gc_utils import (
30 freeze_gc_heap,
@@ -217,6 +218,7 @@ class EngineCore:
218 # environment variable overrides after this point)
219 enable_envs_cache()
220
221 + @instrument(span_name="Prepare model")
222 def _initialize_kv_caches(
223 self, vllm_config: VllmConfig
224 ) -> tuple[int, int, KVCacheConfig]:
@@ -658,6 +660,7 @@ class EngineCoreProc(EngineCore):
660
661 ENGINE_CORE_DEAD = b"ENGINE_CORE_DEAD"
662
663 + @instrument(span_name="EngineCoreProc init")
664 def __init__(
665 self,
666 vllm_config: VllmConfig,
@@ -926,8 +929,18 @@ class EngineCoreProc(EngineCore):
929 data_parallel = parallel_config.data_parallel_size > 1 or dp_rank > 0
930 if data_parallel:
931 parallel_config.data_parallel_rank_local = local_dp_rank
932 + maybe_init_worker_tracer(
933 + instrumenting_module_name="vllm.engine_core",
934 + process_kind="engine_core",
935 + process_name=f"EngineCore_DP{dp_rank}",
936 + )
937 set_process_title("EngineCore", f"DP{dp_rank}")
938 else:
939 + maybe_init_worker_tracer(
940 + instrumenting_module_name="vllm.engine_core",
941 + process_kind="engine_core",
942 + process_name="EngineCore",
943 + )
944 set_process_title("EngineCore")
945 decorate_logs()
946
@@ -956,6 +969,7 @@ class EngineCoreProc(EngineCore):
969 parallel_config.data_parallel_rank = 0
970 engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)
971
972 + assert engine_core is not None
973 engine_core.run_busy_loop()
974
975 except SystemExit:
@@ -1485,6 +1499,13 @@ class EngineCoreActorMixin:
1499 dp_rank: int = 0,
1500 local_dp_rank: int = 0,
1501 ):
1502 + # Initialize tracer for distributed tracing if configured.
1503 + maybe_init_worker_tracer(
1504 + instrumenting_module_name="vllm.engine_core",
1505 + process_kind="engine_core",
1506 + process_name=f"DPEngineCoreActor_DP{dp_rank}",
1507 + )
1508 +
1509 self.addresses = addresses
1510 vllm_config.parallel_config.data_parallel_index = dp_rank
1511 vllm_config.parallel_config.data_parallel_rank_local = local_dp_rank
vllm/v1/engine/core_client.py
+4
@@ -24,6 +24,7 @@ from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S
24 from vllm.logger import init_logger
25 from vllm.lora.request import LoRARequest
26 from vllm.tasks import SupportedTask
27 +from vllm.tracing import instrument
28 from vllm.utils.async_utils import in_loop
29 from vllm.utils.network_utils import (
30 close_sockets,
@@ -96,6 +97,7 @@ class EngineCoreClient(ABC):
97 return InprocClient(vllm_config, executor_class, log_stats)
98
99 @staticmethod
100 + @instrument(span_name="Overall Loading")
101 def make_async_mp_client(
102 vllm_config: VllmConfig,
103 executor_class: type[Executor],
@@ -650,6 +652,7 @@ def _process_utility_output(
652 class SyncMPClient(MPClient):
653 """Synchronous client for multi-proc EngineCore."""
654
655 + @instrument(span_name="SyncMPClient init")
656 def __init__(
657 self, vllm_config: VllmConfig, executor_class: type[Executor], log_stats: bool
658 ):
@@ -819,6 +822,7 @@ class SyncMPClient(MPClient):
822 class AsyncMPClient(MPClient):
823 """Asyncio-compatible client for multi-proc EngineCore."""
824
825 + @instrument(span_name="AsyncMPClient init")
826 def __init__(
827 self,
828 vllm_config: VllmConfig,
vllm/v1/engine/llm_engine.py
+2 -2
@@ -100,8 +100,8 @@ class LLMEngine:
100 )
101 endpoint = self.observability_config.otlp_traces_endpoint
102 if endpoint is not None:
103 - tracer = init_tracer("vllm.llm_engine", endpoint)
104 - self.output_processor.tracer = tracer
103 + init_tracer("vllm.llm_engine", endpoint)
104 + self.output_processor.tracing_enabled = True
105
106 # EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
107 self.engine_core = EngineCoreClient.make_client(
vllm/v1/engine/output_processor.py
+53 -51
@@ -20,7 +20,12 @@ from vllm.outputs import (
20 )
21 from vllm.sampling_params import RequestOutputKind
22 from vllm.tokenizers import TokenizerLike
23 -from vllm.tracing import SpanAttributes, SpanKind, Tracer, extract_trace_context
23 +from vllm.tracing import (
24 + SpanAttributes,
25 + SpanKind,
26 + extract_trace_context,
27 + instrument_manual,
28 +)
29 from vllm.utils import length_from_prompt_token_ids_or_embeds
30 from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
31 from vllm.v1.engine.detokenizer import IncrementalDetokenizer
@@ -422,7 +427,7 @@ class OutputProcessor:
427 self.parent_requests: dict[str, ParentRequest] = {}
428 self.external_req_ids: defaultdict[str, list[str]] = defaultdict(list)
429 self.lora_states = LoRARequestStates(log_stats)
425 - self.tracer: Tracer | None = None
430 + self.tracing_enabled: bool = False
431 self._requests_drained = asyncio.Event()
432 self._requests_drained.set()
433
@@ -678,7 +683,7 @@ class OutputProcessor:
683 self._update_stats_from_finished(
684 req_state, finish_reason, iteration_stats
685 )
681 - if self.tracer:
686 + if self.tracing_enabled:
687 self.do_tracing(engine_core_output, req_state, iteration_stats)
688
689 return OutputProcessorOutput(
@@ -714,62 +719,59 @@ class OutputProcessor:
719 ) -> None:
720 assert req_state.stats is not None
721 assert iteration_stats is not None
717 - assert self.tracer is not None
722
719 - arrival_time_nano_seconds = int(req_state.stats.arrival_time * 1e9)
723 + metrics = req_state.stats
724 + arrival_time_ns = int(metrics.arrival_time * 1e9)
725 trace_context = extract_trace_context(engine_core_output.trace_headers)
726 prompt_length = length_from_prompt_token_ids_or_embeds(
727 req_state.prompt_token_ids, req_state.prompt_embeds
728 )
724 - with self.tracer.start_as_current_span(
725 - "llm_request",
726 - kind=SpanKind.SERVER,
727 - context=trace_context,
728 - start_time=arrival_time_nano_seconds,
729 - ) as span:
730 - metrics = req_state.stats
731 - e2e_time = iteration_stats.iteration_timestamp - metrics.arrival_time
732 - queued_time = metrics.scheduled_ts - metrics.queued_ts
733 - prefill_time = metrics.first_token_ts - metrics.scheduled_ts
734 - decode_time = metrics.last_token_ts - metrics.first_token_ts
735 - inference_time = metrics.last_token_ts - metrics.scheduled_ts
736 - span.set_attribute(
737 - SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN,
738 - metrics.first_token_latency,
739 - )
740 - span.set_attribute(SpanAttributes.GEN_AI_LATENCY_E2E, e2e_time)
741 - span.set_attribute(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE, queued_time)
742 - span.set_attribute(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS, prompt_length)
743 - span.set_attribute(
744 - SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS,
745 - metrics.num_generation_tokens,
746 - )
747 - span.set_attribute(
748 - SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL, prefill_time
749 - )
750 - span.set_attribute(
751 - SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE, decode_time
729 +
730 + # Calculate timing metrics
731 + e2e_time = iteration_stats.iteration_timestamp - metrics.arrival_time
732 + queued_time = metrics.scheduled_ts - metrics.queued_ts
733 + prefill_time = metrics.first_token_ts - metrics.scheduled_ts
734 + decode_time = metrics.last_token_ts - metrics.first_token_ts
735 + inference_time = metrics.last_token_ts - metrics.scheduled_ts
736 +
737 + # Build attributes dict
738 + attributes: dict[str, Any] = {
739 + SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN: (
740 + metrics.first_token_latency
741 + ),
742 + SpanAttributes.GEN_AI_LATENCY_E2E: e2e_time,
743 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE: queued_time,
744 + SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS: prompt_length,
745 + SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS: (
746 + metrics.num_generation_tokens
747 + ),
748 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL: prefill_time,
749 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE: decode_time,
750 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE: inference_time,
751 + SpanAttributes.GEN_AI_REQUEST_ID: req_state.external_req_id,
752 + }
753 +
754 + # Add optional request parameters
755 + if req_state.top_p:
756 + attributes[SpanAttributes.GEN_AI_REQUEST_TOP_P] = req_state.top_p
757 + if req_state.max_tokens_param:
758 + attributes[SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS] = (
759 + req_state.max_tokens_param
760 )
753 - span.set_attribute(
754 - SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE, inference_time
761 + if req_state.temperature:
762 + attributes[SpanAttributes.GEN_AI_REQUEST_TEMPERATURE] = (
763 + req_state.temperature
764 )
765 + if req_state.n:
766 + attributes[SpanAttributes.GEN_AI_REQUEST_N] = req_state.n
767
757 - # meta
758 - span.set_attribute(
759 - SpanAttributes.GEN_AI_REQUEST_ID, req_state.external_req_id
760 - )
761 - if req_state.top_p:
762 - span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TOP_P, req_state.top_p)
763 - if req_state.max_tokens_param:
764 - span.set_attribute(
765 - SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS, req_state.max_tokens_param
766 - )
767 - if req_state.temperature:
768 - span.set_attribute(
769 - SpanAttributes.GEN_AI_REQUEST_TEMPERATURE, req_state.temperature
770 - )
771 - if req_state.n:
772 - span.set_attribute(SpanAttributes.GEN_AI_REQUEST_N, req_state.n)
768 + instrument_manual(
769 + span_name="llm_request",
770 + start_time=arrival_time_ns,
771 + attributes=attributes,
772 + context=trace_context,
773 + kind=SpanKind.SERVER,
774 + )
775
776 def _update_stats_from_output(
777 self,
vllm/v1/executor/abstract.py
+2
@@ -15,6 +15,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
15 from vllm.logger import init_logger
16 from vllm.lora.request import LoRARequest
17 from vllm.tasks import SupportedTask
18 +from vllm.tracing import instrument
19 from vllm.utils.import_utils import resolve_obj_by_qualname
20 from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
21 from vllm.v1.engine import ReconfigureDistributedRequest
@@ -84,6 +85,7 @@ class Executor(ABC):
85 )
86 return executor_class
87
88 + @instrument(span_name="Executor init")
89 def __init__(
90 self,
91 vllm_config: VllmConfig,
vllm/v1/executor/multiproc_executor.py
+11
@@ -41,6 +41,7 @@ from vllm.distributed.parallel_state import (
41 )
42 from vllm.envs import enable_envs_cache
43 from vllm.logger import init_logger
44 +from vllm.tracing import instrument, maybe_init_worker_tracer
45 from vllm.utils.network_utils import (
46 get_distributed_init_method,
47 get_loopback_ip,
@@ -527,6 +528,7 @@ class WorkerProc:
528 )
529 )
530
531 + @instrument(span_name="Worker init")
532 def __init__(
533 self,
534 vllm_config: VllmConfig,
@@ -740,6 +742,15 @@ class WorkerProc:
742
743 try:
744 reader.close()
745 +
746 + # Initialize tracer
747 + rank = kwargs.get("rank", 0)
748 + maybe_init_worker_tracer(
749 + instrumenting_module_name="vllm.worker",
750 + process_kind="worker",
751 + process_name=f"Worker_{rank}",
752 + )
753 +
754 worker = WorkerProc(*args, **kwargs)
755 assert worker.worker_response_mq is not None
756
vllm/v1/worker/cpu_model_runner.py
+3
@@ -9,6 +9,7 @@ import torch.nn as nn
9 from vllm.config import VllmConfig
10 from vllm.logger import init_logger
11 from vllm.model_executor.model_loader import get_model
12 +from vllm.tracing import instrument
13 from vllm.v1.utils import CpuGpuBuffer
14 from vllm.v1.worker.gpu_model_runner import GPUModelRunner
15
@@ -51,6 +52,7 @@ class CPUModelRunner(GPUModelRunner):
52 if isinstance(v, CpuGpuBuffer):
53 v.gpu = v.cpu
54
55 + @instrument(span_name="Loading (CPU)")
56 def load_model(self, eep_scale_up: bool = False) -> None:
57 logger.info("Starting to load model %s...", self.model_config.model)
58 self.model = get_model(vllm_config=self.vllm_config)
@@ -61,6 +63,7 @@ class CPUModelRunner(GPUModelRunner):
63 def get_model(self) -> nn.Module:
64 return self.model
65
66 + @instrument(span_name="Warmup (CPU)")
67 def warming_up_model(self) -> None:
68 logger.info("Warming up model for the compilation...")
69 # Only generate graph for the generic shape
vllm/v1/worker/gpu_model_runner.py
+3
@@ -93,6 +93,7 @@ from vllm.pooling_params import PoolingParams
93 from vllm.sampling_params import SamplingType
94 from vllm.sequence import IntermediateTensors
95 from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
96 +from vllm.tracing import instrument
97 from vllm.utils import length_from_prompt_token_ids_or_embeds
98 from vllm.utils.jsontree import json_map_leaves
99 from vllm.utils.math_utils import cdiv, round_up
@@ -4111,6 +4112,7 @@ class GPUModelRunner(
4112 new_config = update_config(config, config_overrides)
4113 setattr(self, config_name, new_config)
4114
4115 + @instrument(span_name="Loading (GPU)")
4116 def load_model(self, eep_scale_up: bool = False) -> None:
4117 """
4118 Args:
@@ -5165,6 +5167,7 @@ class GPUModelRunner(
5167 self.encoder_cache.clear()
5168 gc.collect()
5169
5170 + @instrument(span_name="Capture model")
5171 def capture_model(self) -> int:
5172 if self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE:
5173 logger.warning(
vllm/v1/worker/gpu_worker.py
+4
@@ -42,6 +42,7 @@ from vllm.platforms import current_platform
42 from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
43 from vllm.sequence import IntermediateTensors
44 from vllm.tasks import SupportedTask
45 +from vllm.tracing import instrument
46 from vllm.utils.mem_utils import MemorySnapshot, format_gib, memory_profiling
47 from vllm.utils.torch_utils import set_random_seed
48 from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
@@ -186,6 +187,7 @@ class Worker(WorkerBase):
187 self.cache_config.num_gpu_blocks = num_gpu_blocks
188 self.cache_config.num_cpu_blocks = num_cpu_blocks
189
190 + @instrument(span_name="Init device")
191 def init_device(self):
192 if self.device_config.device_type == "cuda":
193 # This env var set by Ray causes exceptions with graph building.
@@ -407,6 +409,7 @@ class Worker(WorkerBase):
409 self.model_runner.update_max_model_len(max_model_len)
410 logger.debug("Updated max_model_len to %d", max_model_len)
411
412 + @instrument(span_name="Allocate KV cache")
413 def initialize_from_config(self, kv_cache_config: KVCacheConfig) -> None:
414 """Allocate GPU KV cache with the specified kv_cache_config."""
415
@@ -426,6 +429,7 @@ class Worker(WorkerBase):
429 else:
430 self.model_runner.initialize_kv_cache(kv_cache_config)
431
432 + @instrument(span_name="Warmup (GPU)")
433 def compile_or_warm_up_model(self) -> None:
434 warmup_sizes = []
435
vllm/v1/worker/worker_base.py
+2
@@ -11,6 +11,7 @@ from vllm.config import VllmConfig, set_current_vllm_config
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
13 from vllm.multimodal import MULTIMODAL_REGISTRY
14 +from vllm.tracing import instrument
15 from vllm.utils.import_utils import resolve_obj_by_qualname
16 from vllm.utils.system_utils import update_environment_variables
17 from vllm.v1.kv_cache_interface import KVCacheSpec
@@ -222,6 +223,7 @@ class WorkerWrapperBase:
223 envs = envs_list[self.rpc_rank]
224 update_environment_variables(envs)
225
226 + @instrument(span_name="Worker init")
227 def init_worker(self, all_kwargs: list[dict[str, Any]]) -> None:
228 """
229 Here we inject some common logic before initializing the worker.