A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Misc] Add OpenTelemetry support (#4687)
This PR adds basic support for OpenTelemetry distributed tracing. It includes changes to enable tracing functionality and improve monitoring capabilities. I've also added a markdown with print-screens to guide users how to use this feature. You can find it here
Ronen Schaffer committed
Jun 18, 2024 at 19:17 UTC
7879f24dcce75665d83865ee8281f2ef1bbb7e74
15 files changed
+567
-41
.buildkite/test-pipeline.yaml
+9
@@ -159,6 +159,15 @@ steps:
159
#mirror_hardwares: [amd]
160
command: pytest -v -s quantization
161
162
+- label: Tracing Test
163
+ commands:
164
+ - "pip install \
165
+ opentelemetry-sdk \
166
+ opentelemetry-api \
167
+ opentelemetry-exporter-otlp \
168
+ opentelemetry-semantic-conventions-ai"
169
+ - pytest -v -s tracing
170
+
171
- label: Benchmarks
172
working_dir: "/vllm-workspace/.buildkite"
173
mirror_hardwares: [amd]
benchmarks/benchmark_latency.py
+28
-20
@@ -20,26 +20,29 @@ def main(args: argparse.Namespace):
20
21
# NOTE(woosuk): If the request cannot be processed in a single batch,
22
# the engine will automatically process the request in multiple batches.
23
- llm = LLM(model=args.model,
24
- speculative_model=args.speculative_model,
25
- num_speculative_tokens=args.num_speculative_tokens,
26
- tokenizer=args.tokenizer,
27
- quantization=args.quantization,
28
- tensor_parallel_size=args.tensor_parallel_size,
29
- trust_remote_code=args.trust_remote_code,
30
- dtype=args.dtype,
31
- enforce_eager=args.enforce_eager,
32
- kv_cache_dtype=args.kv_cache_dtype,
33
- quantization_param_path=args.quantization_param_path,
34
- device=args.device,
35
- ray_workers_use_nsight=args.ray_workers_use_nsight,
36
- use_v2_block_manager=args.use_v2_block_manager,
37
- enable_chunked_prefill=args.enable_chunked_prefill,
38
- download_dir=args.download_dir,
39
- block_size=args.block_size,
40
- gpu_memory_utilization=args.gpu_memory_utilization,
41
- load_format=args.load_format,
42
- distributed_executor_backend=args.distributed_executor_backend)
23
+ llm = LLM(
24
+ model=args.model,
25
+ speculative_model=args.speculative_model,
26
+ num_speculative_tokens=args.num_speculative_tokens,
27
+ tokenizer=args.tokenizer,
28
+ quantization=args.quantization,
29
+ tensor_parallel_size=args.tensor_parallel_size,
30
+ trust_remote_code=args.trust_remote_code,
31
+ dtype=args.dtype,
32
+ enforce_eager=args.enforce_eager,
33
+ kv_cache_dtype=args.kv_cache_dtype,
34
+ quantization_param_path=args.quantization_param_path,
35
+ device=args.device,
36
+ ray_workers_use_nsight=args.ray_workers_use_nsight,
37
+ use_v2_block_manager=args.use_v2_block_manager,
38
+ enable_chunked_prefill=args.enable_chunked_prefill,
39
+ download_dir=args.download_dir,
40
+ block_size=args.block_size,
41
+ gpu_memory_utilization=args.gpu_memory_utilization,
42
+ load_format=args.load_format,
43
+ distributed_executor_backend=args.distributed_executor_backend,
44
+ otlp_traces_endpoint=args.otlp_traces_endpoint,
45
+ )
46
47
sampling_params = SamplingParams(
48
n=args.n,
@@ -254,5 +257,10 @@ if __name__ == '__main__':
257
help='Backend to use for distributed serving. When more than 1 GPU '
258
'is used, will be automatically set to "ray" if installed '
259
'or "mp" (multiprocessing) otherwise.')
260
+ parser.add_argument(
261
+ '--otlp-traces-endpoint',
262
+ type=str,
263
+ default=None,
264
+ help='Target URL to which OpenTelemetry traces will be sent.')
265
args = parser.parse_args()
266
main(args)
examples/production_monitoring/Otel.md
new
+82
@@ -0,0 +1,82 @@
1
+# Setup OpenTelemetry POC
2
+
3
+1. Install OpenTelemetry packages:
4
+ ```
5
+ pip install \
6
+ opentelemetry-sdk \
7
+ opentelemetry-api \
8
+ opentelemetry-exporter-otlp \
9
+ opentelemetry-semantic-conventions-ai
10
+ ```
11
+
12
+1. Start Jaeger in a docker container:
13
+ ```
14
+ # From: https://www.jaegertracing.io/docs/1.57/getting-started/
15
+ docker run --rm --name jaeger \
16
+ -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
17
+ -p 6831:6831/udp \
18
+ -p 6832:6832/udp \
19
+ -p 5778:5778 \
20
+ -p 16686:16686 \
21
+ -p 4317:4317 \
22
+ -p 4318:4318 \
23
+ -p 14250:14250 \
24
+ -p 14268:14268 \
25
+ -p 14269:14269 \
26
+ -p 9411:9411 \
27
+ jaegertracing/all-in-one:1.57
28
+ ```
29
+
30
+1. In a new shell, export Jaeger IP:
31
+ ```
32
+ export JAEGER_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' jaeger)
33
+ export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
34
+ ```
35
+ Then set vLLM's service name for OpenTelemetry, enable insecure connections to Jaeger and run vLLM:
36
+ ```
37
+ export OTEL_SERVICE_NAME="vllm-server"
38
+ export OTEL_EXPORTER_OTLP_TRACES_INSECURE=true
39
+ python -m vllm.entrypoints.openai.api_server --model="facebook/opt-125m" --otlp-traces-endpoint="$OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
40
+ ```
41
+
42
+1. In a new shell, send requests with trace context from a dummy client
43
+ ```
44
+ export JAEGER_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' jaeger)
45
+ export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
46
+ export OTEL_EXPORTER_OTLP_TRACES_INSECURE=true
47
+ export OTEL_SERVICE_NAME="client-service"
48
+ python dummy_client.py
49
+ ```
50
+
51
+1. Open Jaeger webui: http://localhost:16686/
52
+
53
+ In the search pane, select `vllm-server` service and hit `Find Traces`. You should get a list of traces, one for each request.
54
+ 
55
+
56
+1. Clicking on a trace will show its spans and their tags. In this demo, each trace has 2 spans. One from the dummy client containing the prompt text and one from vLLM containing metadata about the request.
57
+
58
+
59
+## Exporter Protocol
60
+OpenTelemetry supports either `grpc` or `http/protobuf` as the transport protocol for trace data in the exporter.
61
+By default, `grpc` is used. To set `http/protobuf` as the protocol, configure the `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` environment variable as follows:
62
+```
63
+export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf
64
+export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
65
+python -m vllm.entrypoints.openai.api_server --model="facebook/opt-125m" --otlp-traces-endpoint="$OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
66
+```
67
+
68
+## Instrumentation of FastAPI
69
+OpenTelemetry allows automatic instrumentation of FastAPI.
70
+1. Install the instrumentation library
71
+ ```
72
+ pip install opentelemetry-instrumentation-fastapi
73
+ ```
74
+
75
+1. Run vLLM with `opentelemetry-instrument`
76
+ ```
77
+ opentelemetry-instrument python -m vllm.entrypoints.openai.api_server --model="facebook/opt-125m"
78
+ ```
79
+
80
+1. Send a request to vLLM and find its trace in Jaeger. It should contain spans from FastAPI.
81
+
82
+
\ No newline at end of file
examples/production_monitoring/dummy_client.py
new
+35
@@ -0,0 +1,35 @@
1
+import requests
2
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
3
+ OTLPSpanExporter)
4
+from opentelemetry.sdk.trace import TracerProvider
5
+from opentelemetry.sdk.trace.export import (BatchSpanProcessor,
6
+ ConsoleSpanExporter)
7
+from opentelemetry.trace import SpanKind, set_tracer_provider
8
+from opentelemetry.trace.propagation.tracecontext import (
9
+ TraceContextTextMapPropagator)
10
+
11
+trace_provider = TracerProvider()
12
+set_tracer_provider(trace_provider)
13
+
14
+trace_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
15
+trace_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
16
+
17
+tracer = trace_provider.get_tracer("dummy-client")
18
+
19
+url = "http://localhost:8000/v1/completions"
20
+with tracer.start_as_current_span("client-span", kind=SpanKind.CLIENT) as span:
21
+ prompt = "San Francisco is a"
22
+ span.set_attribute("prompt", prompt)
23
+ headers = {}
24
+ TraceContextTextMapPropagator().inject(headers)
25
+ payload = {
26
+ "model": "facebook/opt-125m",
27
+ "prompt": prompt,
28
+ "max_tokens": 10,
29
+ "best_of": 20,
30
+ "n": 3,
31
+ "use_beam_search": "true",
32
+ "temperature": 0.0,
33
+ # "stream": True,
34
+ }
35
+ response = requests.post(url, headers=headers, json=payload)
tests/tracing/__init__.py
tests/tracing/test_tracing.py
new
+116
@@ -0,0 +1,116 @@
1
+import os
2
+import threading
3
+from concurrent import futures
4
+from typing import Callable, Dict, Iterable, Literal
5
+
6
+import grpc
7
+import pytest
8
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
9
+ ExportTraceServiceResponse)
10
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import (
11
+ TraceServiceServicer, add_TraceServiceServicer_to_server)
12
+from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
13
+from opentelemetry.sdk.environment_variables import (
14
+ OTEL_EXPORTER_OTLP_TRACES_INSECURE)
15
+
16
+from vllm import LLM, SamplingParams
17
+from vllm.tracing import SpanAttributes
18
+
19
+FAKE_TRACE_SERVER_ADDRESS = "localhost:4317"
20
+
21
+FieldName = Literal['bool_value', 'string_value', 'int_value', 'double_value',
22
+ 'array_value']
23
+
24
+
25
+def decode_value(value: AnyValue):
26
+ field_decoders: Dict[FieldName, Callable] = {
27
+ "bool_value": (lambda v: v.bool_value),
28
+ "string_value": (lambda v: v.string_value),
29
+ "int_value": (lambda v: v.int_value),
30
+ "double_value": (lambda v: v.double_value),
31
+ "array_value":
32
+ (lambda v: [decode_value(item) for item in v.array_value.values]),
33
+ }
34
+ for field, decoder in field_decoders.items():
35
+ if value.HasField(field):
36
+ return decoder(value)
37
+ raise ValueError(f"Couldn't decode value: {value}")
38
+
39
+
40
+def decode_attributes(attributes: Iterable[KeyValue]):
41
+ return {kv.key: decode_value(kv.value) for kv in attributes}
42
+
43
+
44
+class FakeTraceService(TraceServiceServicer):
45
+
46
+ def __init__(self):
47
+ self.request = None
48
+ self.evt = threading.Event()
49
+
50
+ def Export(self, request, context):
51
+ self.request = request
52
+ self.evt.set()
53
+ return ExportTraceServiceResponse()
54
+
55
+
56
+@pytest.fixture
57
+def trace_service():
58
+ """Fixture to set up a fake gRPC trace service"""
59
+ server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
60
+ service = FakeTraceService()
61
+ add_TraceServiceServicer_to_server(service, server)
62
+ server.add_insecure_port(FAKE_TRACE_SERVER_ADDRESS)
63
+ server.start()
64
+
65
+ yield service
66
+
67
+ server.stop(None)
68
+
69
+
70
+def test_traces(trace_service):
71
+ os.environ[OTEL_EXPORTER_OTLP_TRACES_INSECURE] = "true"
72
+
73
+ sampling_params = SamplingParams(temperature=0.01,
74
+ top_p=0.1,
75
+ max_tokens=256)
76
+ model = "facebook/opt-125m"
77
+ llm = LLM(
78
+ model=model,
79
+ otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,
80
+ )
81
+ prompts = ["This is a short prompt"]
82
+ outputs = llm.generate(prompts, sampling_params=sampling_params)
83
+
84
+ timeout = 5
85
+ if not trace_service.evt.wait(timeout):
86
+ raise TimeoutError(
87
+ f"The fake trace service didn't receive a trace within "
88
+ f"the {timeout} seconds timeout")
89
+
90
+ attributes = decode_attributes(trace_service.request.resource_spans[0].
91
+ scope_spans[0].spans[0].attributes)
92
+ assert attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) == model
93
+ assert attributes.get(
94
+ SpanAttributes.LLM_REQUEST_ID) == outputs[0].request_id
95
+ assert attributes.get(
96
+ SpanAttributes.LLM_REQUEST_TEMPERATURE) == sampling_params.temperature
97
+ assert attributes.get(
98
+ SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
99
+ assert attributes.get(
100
+ SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
101
+ assert attributes.get(
102
+ SpanAttributes.LLM_REQUEST_BEST_OF) == sampling_params.best_of
103
+ assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
104
+ assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
105
+ outputs[0].prompt_token_ids)
106
+ completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
107
+ assert attributes.get(
108
+ SpanAttributes.LLM_USAGE_COMPLETION_TOKENS) == completion_tokens
109
+ metrics = outputs[0].metrics
110
+ assert attributes.get(
111
+ SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE) == metrics.time_in_queue
112
+ ttft = metrics.first_token_time - metrics.arrival_time
113
+ assert attributes.get(
114
+ SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
115
+ e2e_time = metrics.finished_time - metrics.arrival_time
116
+ assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
vllm/config.py
+13
@@ -10,6 +10,7 @@ from transformers import PretrainedConfig, PreTrainedTokenizerBase
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12
from vllm.model_executor.models import ModelRegistry
13
+from vllm.tracing import is_otel_installed
14
from vllm.transformers_utils.config import get_config, get_hf_text_config
15
from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
16
is_hip, is_neuron, is_tpu, is_xpu)
@@ -1371,6 +1372,17 @@ class DecodingConfig:
1372
f"must be one of {valid_guided_backends}")
1373
1374
1375
+@dataclass
1376
+class ObservabilityConfig:
1377
+ """Configuration for observability."""
1378
+ otlp_traces_endpoint: Optional[str] = None
1379
+
1380
+ def __post_init__(self):
1381
+ if not is_otel_installed() and self.otlp_traces_endpoint is not None:
1382
+ raise ValueError("OpenTelemetry packages must be installed before "
1383
+ "configuring 'otlp_traces_endpoint'")
1384
+
1385
+
1386
@dataclass(frozen=True)
1387
class EngineConfig:
1388
"""Dataclass which contains all engine-related configuration. This
@@ -1387,6 +1399,7 @@ class EngineConfig:
1399
vision_language_config: Optional[VisionLanguageConfig]
1400
speculative_config: Optional[SpeculativeConfig]
1401
decoding_config: Optional[DecodingConfig]
1402
+ observability_config: Optional[ObservabilityConfig]
1403
1404
def __post_init__(self):
1405
"""Verify configs are valid & consistent with each other.
vllm/engine/arg_utils.py
+28
-12
@@ -7,8 +7,9 @@ from typing import List, Optional, Tuple, Union
7
8
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
10
- ParallelConfig, SchedulerConfig, SpeculativeConfig,
11
- TokenizerPoolConfig, VisionLanguageConfig)
10
+ ObservabilityConfig, ParallelConfig, SchedulerConfig,
11
+ SpeculativeConfig, TokenizerPoolConfig,
12
+ VisionLanguageConfig)
13
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
14
from vllm.utils import str_to_int_tuple
15
@@ -101,6 +102,8 @@ class EngineArgs:
102
103
qlora_adapter_name_or_path: Optional[str] = None
104
105
+ otlp_traces_endpoint: Optional[str] = None
106
+
107
def __post_init__(self):
108
if self.tokenizer is None:
109
self.tokenizer = self.model
@@ -599,6 +602,13 @@ class EngineArgs:
602
type=str,
603
default=None,
604
help='Name or path of the QLoRA adapter.')
605
+
606
+ parser.add_argument(
607
+ '--otlp-traces-endpoint',
608
+ type=str,
609
+ default=None,
610
+ help='Target URL to which OpenTelemetry traces will be sent.')
611
+
612
return parser
613
614
@classmethod
@@ -757,6 +767,9 @@ class EngineArgs:
767
decoding_config = DecodingConfig(
768
guided_decoding_backend=self.guided_decoding_backend)
769
770
+ observability_config = ObservabilityConfig(
771
+ otlp_traces_endpoint=self.otlp_traces_endpoint)
772
+
773
if (model_config.get_sliding_window() is not None
774
and scheduler_config.chunked_prefill_enabled
775
and not scheduler_config.use_v2_block_manager):
@@ -764,16 +777,19 @@ class EngineArgs:
777
"Chunked prefill is not supported with sliding window. "
778
"Set --disable-sliding-window to disable sliding window.")
779
767
- return EngineConfig(model_config=model_config,
768
- cache_config=cache_config,
769
- parallel_config=parallel_config,
770
- scheduler_config=scheduler_config,
771
- device_config=device_config,
772
- lora_config=lora_config,
773
- vision_language_config=vision_language_config,
774
- speculative_config=speculative_config,
775
- load_config=load_config,
776
- decoding_config=decoding_config)
780
+ return EngineConfig(
781
+ model_config=model_config,
782
+ cache_config=cache_config,
783
+ parallel_config=parallel_config,
784
+ scheduler_config=scheduler_config,
785
+ device_config=device_config,
786
+ lora_config=lora_config,
787
+ vision_language_config=vision_language_config,
788
+ speculative_config=speculative_config,
789
+ load_config=load_config,
790
+ decoding_config=decoding_config,
791
+ observability_config=observability_config,
792
+ )
793
794
795
@dataclass
vllm/engine/async_llm_engine.py
+22
@@ -244,6 +244,9 @@ class _AsyncLLMEngine(LLMEngine):
244
# Log stats.
245
self.do_log_stats(scheduler_outputs, output)
246
247
+ # Tracing
248
+ self.do_tracing(scheduler_outputs)
249
+
250
if not request_outputs:
251
# Stop the execute model loop in parallel workers until there are
252
# more requests to process. This avoids waiting indefinitely in
@@ -285,6 +288,7 @@ class _AsyncLLMEngine(LLMEngine):
288
params: Union[SamplingParams, PoolingParams],
289
arrival_time: Optional[float] = None,
290
lora_request: Optional[LoRARequest] = None,
291
+ trace_headers: Optional[Dict[str, str]] = None,
292
) -> None:
293
if lora_request is not None and not self.lora_config:
294
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -301,6 +305,7 @@ class _AsyncLLMEngine(LLMEngine):
305
params=params,
306
arrival_time=arrival_time,
307
lora_request=lora_request,
308
+ trace_headers=trace_headers,
309
)
310
311
async def check_health_async(self) -> None:
@@ -556,6 +561,7 @@ class AsyncLLMEngine:
561
params: Union[SamplingParams, PoolingParams],
562
arrival_time: Optional[float] = None,
563
lora_request: Optional[LoRARequest] = None,
564
+ trace_headers: Optional[Dict[str, str]] = None,
565
) -> AsyncStream:
566
if self.log_requests:
567
if isinstance(inputs, str):
@@ -597,6 +603,7 @@ class AsyncLLMEngine:
603
params=params,
604
arrival_time=arrival_time,
605
lora_request=lora_request,
606
+ trace_headers=trace_headers,
607
)
608
609
return stream
@@ -607,6 +614,7 @@ class AsyncLLMEngine:
614
sampling_params: SamplingParams,
615
request_id: str,
616
lora_request: Optional[LoRARequest] = None,
617
+ trace_headers: Optional[Dict[str, str]] = None,
618
) -> AsyncIterator[RequestOutput]:
619
"""Generate outputs for a request.
620
@@ -621,6 +629,7 @@ class AsyncLLMEngine:
629
sampling_params: The sampling parameters of the request.
630
request_id: The unique id of the request.
631
lora_request: LoRA request to use for generation, if any.
632
+ trace_headers: OpenTelemetry trace headers.
633
634
Yields:
635
The output `RequestOutput` objects from the LLMEngine
@@ -674,6 +683,7 @@ class AsyncLLMEngine:
683
inputs,
684
sampling_params,
685
lora_request=lora_request,
686
+ trace_headers=trace_headers,
687
):
688
yield LLMEngine.validate_output(output, RequestOutput)
689
@@ -683,6 +693,7 @@ class AsyncLLMEngine:
693
pooling_params: PoolingParams,
694
request_id: str,
695
lora_request: Optional[LoRARequest] = None,
696
+ trace_headers: Optional[Dict[str, str]] = None,
697
) -> AsyncIterator[EmbeddingRequestOutput]:
698
"""Generate outputs for a request from an embedding model.
699
@@ -697,6 +708,7 @@ class AsyncLLMEngine:
708
pooling_params: The pooling parameters of the request.
709
request_id: The unique id of the request.
710
lora_request: LoRA request to use for generation, if any.
711
+ trace_headers: OpenTelemetry trace headers.
712
713
Yields:
714
The output `EmbeddingRequestOutput` objects from the LLMEngine
@@ -748,6 +760,7 @@ class AsyncLLMEngine:
760
inputs,
761
pooling_params,
762
lora_request=lora_request,
763
+ trace_headers=trace_headers,
764
):
765
yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
766
@@ -758,6 +771,7 @@ class AsyncLLMEngine:
771
params: Union[SamplingParams, PoolingParams],
772
*,
773
lora_request: Optional[LoRARequest] = None,
774
+ trace_headers: Optional[Dict[str, str]] = None,
775
) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
776
"""Common logic to process requests with SamplingParams or
777
PoolingParams."""
@@ -769,6 +783,7 @@ class AsyncLLMEngine:
783
params,
784
arrival_time=arrival_time,
785
lora_request=lora_request,
786
+ trace_headers=trace_headers,
787
)
788
789
try:
@@ -848,3 +863,10 @@ class AsyncLLMEngine:
863
else:
864
await self.engine.check_health_async()
865
logger.debug("Health check took %fs", time.perf_counter() - t)
866
+
867
+ async def is_tracing_enabled(self) -> bool:
868
+ if self.engine_use_ray:
869
+ return await self.engine.is_tracing_enabled.remote( # type: ignore
870
+ )
871
+ else:
872
+ return self.engine.is_tracing_enabled()
vllm/engine/llm_engine.py
+93
-9
@@ -1,14 +1,14 @@
1
import time
2
from contextlib import contextmanager
3
-from typing import TYPE_CHECKING, ClassVar, Iterable, List, Optional
3
+from typing import TYPE_CHECKING, ClassVar, Dict, Iterable, List, Optional
4
from typing import Sequence as GenericSequence
5
from typing import Set, Type, TypeVar, Union
6
7
from transformers import GenerationConfig, PreTrainedTokenizer
8
9
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
10
- LoRAConfig, ModelConfig, ParallelConfig,
11
- SchedulerConfig, SpeculativeConfig,
10
+ LoRAConfig, ModelConfig, ObservabilityConfig,
11
+ ParallelConfig, SchedulerConfig, SpeculativeConfig,
12
VisionLanguageConfig)
13
from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
14
SchedulerOutputs)
@@ -31,6 +31,8 @@ from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
31
PoolerOutput, SamplerOutput, Sequence,
32
SequenceGroup, SequenceGroupMetadata,
33
SequenceStatus)
34
+from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
35
+ init_tracer)
36
from vllm.transformers_utils.detokenizer import Detokenizer
37
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
38
get_tokenizer_group)
@@ -154,6 +156,7 @@ class LLMEngine:
156
vision_language_config: Optional[VisionLanguageConfig],
157
speculative_config: Optional[SpeculativeConfig],
158
decoding_config: Optional[DecodingConfig],
159
+ observability_config: Optional[ObservabilityConfig],
160
executor_class: Type[ExecutorBase],
161
log_stats: bool,
162
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -168,7 +171,8 @@ class LLMEngine:
171
"disable_custom_all_reduce=%s, quantization=%s, "
172
"enforce_eager=%s, kv_cache_dtype=%s, "
173
"quantization_param_path=%s, device_config=%s, "
171
- "decoding_config=%r, seed=%d, served_model_name=%s)",
174
+ "decoding_config=%r, observability_config=%r, "
175
+ "seed=%d, served_model_name=%s)",
176
VLLM_VERSION,
177
model_config.model,
178
speculative_config,
@@ -192,6 +196,7 @@ class LLMEngine:
196
model_config.quantization_param_path,
197
device_config.device,
198
decoding_config,
199
+ observability_config,
200
model_config.seed,
201
model_config.served_model_name,
202
)
@@ -207,6 +212,8 @@ class LLMEngine:
212
self.speculative_config = speculative_config
213
self.load_config = load_config
214
self.decoding_config = decoding_config or DecodingConfig()
215
+ self.observability_config = observability_config or ObservabilityConfig(
216
+ )
217
self.log_stats = log_stats
218
219
if not self.model_config.skip_tokenizer_init:
@@ -288,6 +295,12 @@ class LLMEngine:
295
max_model_len=self.model_config.max_model_len)
296
self.stat_logger.info("cache_config", self.cache_config)
297
298
+ self.tracer = None
299
+ if self.observability_config.otlp_traces_endpoint:
300
+ self.tracer = init_tracer(
301
+ "vllm.llm_engine",
302
+ self.observability_config.otlp_traces_endpoint)
303
+
304
# Create sequence output processor, e.g. for beam search or
305
# speculative decoding.
306
self.output_processor = (
@@ -444,6 +457,7 @@ class LLMEngine:
457
params: Union[SamplingParams, PoolingParams],
458
arrival_time: float,
459
lora_request: Optional[LoRARequest],
460
+ trace_headers: Optional[Dict[str, str]] = None,
461
) -> None:
462
# Create the sequences.
463
block_size = self.cache_config.block_size
@@ -461,6 +475,7 @@ class LLMEngine:
475
params,
476
arrival_time=arrival_time,
477
lora_request=lora_request,
478
+ trace_headers=trace_headers,
479
)
480
elif isinstance(params, PoolingParams):
481
seq_group = self._create_sequence_group_with_pooling(
@@ -507,6 +522,7 @@ class LLMEngine:
522
params: Union[SamplingParams, PoolingParams],
523
arrival_time: Optional[float] = None,
524
lora_request: Optional[LoRARequest] = None,
525
+ trace_headers: Optional[Dict[str, str]] = None,
526
) -> None:
527
"""Add a request to the engine's request pool.
528
@@ -524,6 +540,7 @@ class LLMEngine:
540
:class:`~vllm.PoolingParams` for pooling.
541
arrival_time: The arrival time of the request. If None, we use
542
the current monotonic time.
543
+ trace_headers: OpenTelemetry trace headers.
544
545
Details:
546
- Set arrival_time to the current time if it is None.
@@ -565,6 +582,7 @@ class LLMEngine:
582
params=params,
583
arrival_time=arrival_time,
584
lora_request=lora_request,
585
+ trace_headers=trace_headers,
586
)
587
588
def _create_sequence_group_with_sampling(
@@ -574,6 +592,7 @@ class LLMEngine:
592
sampling_params: SamplingParams,
593
arrival_time: float,
594
lora_request: Optional[LoRARequest],
595
+ trace_headers: Optional[Dict[str, str]] = None,
596
) -> SequenceGroup:
597
"""Creates a SequenceGroup with SamplingParams."""
598
max_logprobs = self.get_model_config().max_logprobs
@@ -595,11 +614,14 @@ class LLMEngine:
614
self.generation_config_fields)
615
616
# Create the sequence group.
598
- seq_group = SequenceGroup(request_id=request_id,
599
- seqs=[seq],
600
- arrival_time=arrival_time,
601
- sampling_params=sampling_params,
602
- lora_request=lora_request)
617
+ seq_group = SequenceGroup(
618
+ request_id=request_id,
619
+ seqs=[seq],
620
+ arrival_time=arrival_time,
621
+ sampling_params=sampling_params,
622
+ lora_request=lora_request,
623
+ trace_headers=trace_headers,
624
+ )
625
626
return seq_group
627
@@ -793,6 +815,9 @@ class LLMEngine:
815
# Log stats.
816
self.do_log_stats(scheduler_outputs, output)
817
818
+ # Tracing
819
+ self.do_tracing(scheduler_outputs)
820
+
821
if not request_outputs:
822
# Stop the execute model loop in parallel workers until there are
823
# more requests to process. This avoids waiting indefinitely in
@@ -986,3 +1011,62 @@ class LLMEngine:
1011
1012
def check_health(self) -> None:
1013
self.model_executor.check_health()
1014
+
1015
+ def is_tracing_enabled(self) -> bool:
1016
+ return self.tracer is not None
1017
+
1018
+ def do_tracing(self, scheduler_outputs: SchedulerOutputs) -> None:
1019
+ if self.tracer is None:
1020
+ return
1021
+
1022
+ for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
1023
+ seq_group = scheduled_seq_group.seq_group
1024
+ if seq_group.is_finished():
1025
+ self.create_trace_span(seq_group)
1026
+
1027
+ def create_trace_span(self, seq_group: SequenceGroup) -> None:
1028
+ if self.tracer is None or seq_group.sampling_params is None:
1029
+ return
1030
+ arrival_time_nano_seconds = int(seq_group.metrics.arrival_time * 1e9)
1031
+
1032
+ trace_context = extract_trace_context(seq_group.trace_headers)
1033
+
1034
+ with self.tracer.start_as_current_span(
1035
+ "llm_request",
1036
+ kind=SpanKind.SERVER,
1037
+ context=trace_context,
1038
+ start_time=arrival_time_nano_seconds) as seq_span:
1039
+ metrics = seq_group.metrics
1040
+ ttft = metrics.first_token_time - metrics.arrival_time
1041
+ e2e_time = metrics.finished_time - metrics.arrival_time
1042
+ # attribute names are based on
1043
+ # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/llm-spans.md
1044
+ seq_span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL,
1045
+ self.model_config.model)
1046
+ seq_span.set_attribute(SpanAttributes.LLM_REQUEST_ID,
1047
+ seq_group.request_id)
1048
+ seq_span.set_attribute(SpanAttributes.LLM_REQUEST_TEMPERATURE,
1049
+ seq_group.sampling_params.temperature)
1050
+ seq_span.set_attribute(SpanAttributes.LLM_REQUEST_TOP_P,
1051
+ seq_group.sampling_params.top_p)
1052
+ seq_span.set_attribute(SpanAttributes.LLM_REQUEST_MAX_TOKENS,
1053
+ seq_group.sampling_params.max_tokens)
1054
+ seq_span.set_attribute(SpanAttributes.LLM_REQUEST_BEST_OF,
1055
+ seq_group.sampling_params.best_of)
1056
+ seq_span.set_attribute(SpanAttributes.LLM_REQUEST_N,
1057
+ seq_group.sampling_params.n)
1058
+ seq_span.set_attribute(SpanAttributes.LLM_USAGE_NUM_SEQUENCES,
1059
+ seq_group.num_seqs())
1060
+ seq_span.set_attribute(SpanAttributes.LLM_USAGE_PROMPT_TOKENS,
1061
+ len(seq_group.prompt_token_ids))
1062
+ seq_span.set_attribute(
1063
+ SpanAttributes.LLM_USAGE_COMPLETION_TOKENS,
1064
+ sum([
1065
+ seq.get_output_len()
1066
+ for seq in seq_group.get_finished_seqs()
1067
+ ]))
1068
+ seq_span.set_attribute(SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE,
1069
+ metrics.time_in_queue)
1070
+ seq_span.set_attribute(
1071
+ SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
1072
+ seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
vllm/entrypoints/openai/serving_chat.py
+11
@@ -31,6 +31,8 @@ from vllm.multimodal.utils import (async_get_and_parse_image,
31
get_full_image_text_prompt)
32
from vllm.outputs import RequestOutput
33
from vllm.sequence import Logprob
34
+from vllm.tracing import (contains_trace_headers, extract_trace_headers,
35
+ log_tracing_disabled_warning)
36
from vllm.utils import random_uuid
37
38
logger = init_logger(__name__)
@@ -267,11 +269,20 @@ class OpenAIServingChat(OpenAIServing):
269
if image_data is not None:
270
inputs["multi_modal_data"] = image_data
271
272
+ is_tracing_enabled = await self.engine.is_tracing_enabled()
273
+ trace_headers = None
274
+ if is_tracing_enabled and raw_request:
275
+ trace_headers = extract_trace_headers(raw_request.headers)
276
+ if not is_tracing_enabled and raw_request and contains_trace_headers(
277
+ raw_request.headers):
278
+ log_tracing_disabled_warning()
279
+
280
result_generator = self.engine.generate(
281
inputs,
282
sampling_params,
283
request_id,
284
lora_request,
285
+ trace_headers=trace_headers,
286
)
287
# Streaming response
288
if request.stream:
vllm/entrypoints/openai/serving_completion.py
+11
@@ -24,6 +24,8 @@ from vllm.model_executor.guided_decoding import (
24
get_guided_decoding_logits_processor)
25
from vllm.outputs import RequestOutput
26
from vllm.sequence import Logprob
27
+from vllm.tracing import (contains_trace_headers, extract_trace_headers,
28
+ log_tracing_disabled_warning)
29
from vllm.utils import merge_async_iterators, random_uuid
30
31
logger = init_logger(__name__)
@@ -125,6 +127,14 @@ class OpenAIServingCompletion(OpenAIServing):
127
truncate_prompt_tokens)
128
prompt_ids, prompt_text = prompt_formats
129
130
+ is_tracing_enabled = await self.engine.is_tracing_enabled()
131
+ trace_headers = None
132
+ if is_tracing_enabled:
133
+ trace_headers = extract_trace_headers(raw_request.headers)
134
+ if not is_tracing_enabled and contains_trace_headers(
135
+ raw_request.headers):
136
+ log_tracing_disabled_warning()
137
+
138
generator = self.engine.generate(
139
{
140
"prompt": prompt_text,
@@ -133,6 +143,7 @@ class OpenAIServingCompletion(OpenAIServing):
143
sampling_params,
144
f"{request_id}-{i}",
145
lora_request=lora_request,
146
+ trace_headers=trace_headers,
147
)
148
149
generators.append(generator)
vllm/sequence.py
+3
@@ -414,6 +414,7 @@ class SequenceGroup:
414
for an embedding model.
415
encoder_seq: Optional, the single encoder sequence. Should be None
416
unless you are working with an encoder/decoder model.
417
+ trace_headers: OpenTelemetry trace headers.
418
"""
419
420
def __init__(
@@ -426,6 +427,7 @@ class SequenceGroup:
427
embeddings: Optional[List[float]] = None,
428
pooling_params: Optional[PoolingParams] = None,
429
encoder_seq: Optional[Sequence] = None,
430
+ trace_headers: Optional[Dict[str, str]] = None,
431
) -> None:
432
self.request_id = request_id
433
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -441,6 +443,7 @@ class SequenceGroup:
443
self.embeddings = embeddings
444
self.pooling_params = pooling_params
445
self.encoder_seq = encoder_seq
446
+ self.trace_headers = trace_headers
447
448
@property
449
def prompt(self) -> Optional[str]:
vllm/tracing.py
new
+104
@@ -0,0 +1,104 @@
1
+import os
2
+from typing import Mapping, Optional
3
+
4
+from vllm.logger import init_logger
5
+from vllm.utils import run_once
6
+
7
+TRACE_HEADERS = ["traceparent", "tracestate"]
8
+
9
+logger = init_logger(__name__)
10
+
11
+_is_otel_installed = False
12
+try:
13
+ from opentelemetry.context.context import Context
14
+ from opentelemetry.sdk.environment_variables import (
15
+ OTEL_EXPORTER_OTLP_TRACES_PROTOCOL)
16
+ from opentelemetry.sdk.trace import TracerProvider
17
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
18
+ from opentelemetry.semconv.ai import SpanAttributes as BaseSpanAttributes
19
+ from opentelemetry.trace import SpanKind, Tracer, set_tracer_provider
20
+ from opentelemetry.trace.propagation.tracecontext import (
21
+ TraceContextTextMapPropagator)
22
+ _is_otel_installed = True
23
+except ImportError:
24
+
25
+ class Context: # type: ignore
26
+ pass
27
+
28
+ class BaseSpanAttributes: # type: ignore
29
+ pass
30
+
31
+ class SpanKind: # type: ignore
32
+ pass
33
+
34
+ class Tracer: # type: ignore
35
+ pass
36
+
37
+
38
+def is_otel_installed() -> bool:
39
+ return _is_otel_installed
40
+
41
+
42
+def init_tracer(instrumenting_module_name: str,
43
+ otlp_traces_endpoint: str) -> Optional[Tracer]:
44
+ assert is_otel_installed(), ("OpenTelemetry packages must be installed "
45
+ "prior to initializing a tracer")
46
+ trace_provider = TracerProvider()
47
+
48
+ span_exporter = get_span_exporter(otlp_traces_endpoint)
49
+ trace_provider.add_span_processor(BatchSpanProcessor(span_exporter))
50
+ set_tracer_provider(trace_provider)
51
+
52
+ tracer = trace_provider.get_tracer(instrumenting_module_name)
53
+ return tracer
54
+
55
+
56
+def get_span_exporter(endpoint):
57
+ protocol = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "grpc")
58
+ if protocol == "grpc":
59
+ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
60
+ OTLPSpanExporter)
61
+ elif protocol == "http/protobuf":
62
+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
63
+ OTLPSpanExporter)
64
+ else:
65
+ raise ValueError(
66
+ f"Unsupported OTLP protocol '{protocol}' is configured")
67
+
68
+ return OTLPSpanExporter(endpoint=endpoint)
69
+
70
+
71
+def extract_trace_context(
72
+ headers: Optional[Mapping[str, str]]) -> Optional[Context]:
73
+ if is_otel_installed():
74
+ headers = headers or {}
75
+ return TraceContextTextMapPropagator().extract(headers)
76
+ else:
77
+ return None
78
+
79
+
80
+def extract_trace_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
81
+
82
+ return {h: headers[h] for h in TRACE_HEADERS if h in headers}
83
+
84
+
85
+class SpanAttributes(BaseSpanAttributes):
86
+ # The following span attribute names are added here because they are missing
87
+ # from the Semantic Conventions for LLM.
88
+ LLM_REQUEST_ID = "gen_ai.request.id"
89
+ LLM_REQUEST_BEST_OF = "gen_ai.request.best_of"
90
+ LLM_REQUEST_N = "gen_ai.request.n"
91
+ LLM_USAGE_NUM_SEQUENCES = "gen_ai.usage.num_sequences"
92
+ LLM_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
93
+ LLM_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token"
94
+ LLM_LATENCY_E2E = "gen_ai.latency.e2e"
95
+
96
+
97
+def contains_trace_headers(headers: Mapping[str, str]) -> bool:
98
+ return any(h in headers for h in TRACE_HEADERS)
99
+
100
+
101
+@run_once
102
+def log_tracing_disabled_warning() -> None:
103
+ logger.warning(
104
+ "Received a request with trace context but tracing is disabled")
vllm/utils.py
+12
@@ -763,3 +763,15 @@ def cuda_device_count_stateless() -> int:
763
# after https://github.com/pytorch/pytorch/pull/122815 is released.
764
765
return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
766
+
767
+
768
+#From: https://stackoverflow.com/a/4104188/2749989
769
+def run_once(f):
770
+
771
+ def wrapper(*args, **kwargs) -> Any:
772
+ if not wrapper.has_run: # type: ignore[attr-defined]
773
+ wrapper.has_run = True # type: ignore[attr-defined]
774
+ return f(*args, **kwargs)
775
+
776
+ wrapper.has_run = False # type: ignore[attr-defined]
777
+ return wrapper