@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7879f24dcc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc] Add OpenTelemetry support (#4687)

This PR adds basic support for OpenTelemetry distributed tracing. It includes changes to enable tracing functionality and improve monitoring capabilities. I've also added a markdown with print-screens to guide users how to use this feature. You can find it here

Ronen Schaffer committed Jun 18, 2024 at 19:17 UTC 7879f24dcce75665d83865ee8281f2ef1bbb7e74
15 files changed +567 -41
.buildkite/test-pipeline.yaml
+9
@@ -159,6 +159,15 @@ steps:
159 #mirror_hardwares: [amd]
160 command: pytest -v -s quantization
161
162 +- label: Tracing Test
163 + commands:
164 + - "pip install \
165 + opentelemetry-sdk \
166 + opentelemetry-api \
167 + opentelemetry-exporter-otlp \
168 + opentelemetry-semantic-conventions-ai"
169 + - pytest -v -s tracing
170 +
171 - label: Benchmarks
172 working_dir: "/vllm-workspace/.buildkite"
173 mirror_hardwares: [amd]
benchmarks/benchmark_latency.py
+28 -20
@@ -20,26 +20,29 @@ def main(args: argparse.Namespace):
20
21 # NOTE(woosuk): If the request cannot be processed in a single batch,
22 # the engine will automatically process the request in multiple batches.
23 - llm = LLM(model=args.model,
24 - speculative_model=args.speculative_model,
25 - num_speculative_tokens=args.num_speculative_tokens,
26 - tokenizer=args.tokenizer,
27 - quantization=args.quantization,
28 - tensor_parallel_size=args.tensor_parallel_size,
29 - trust_remote_code=args.trust_remote_code,
30 - dtype=args.dtype,
31 - enforce_eager=args.enforce_eager,
32 - kv_cache_dtype=args.kv_cache_dtype,
33 - quantization_param_path=args.quantization_param_path,
34 - device=args.device,
35 - ray_workers_use_nsight=args.ray_workers_use_nsight,
36 - use_v2_block_manager=args.use_v2_block_manager,
37 - enable_chunked_prefill=args.enable_chunked_prefill,
38 - download_dir=args.download_dir,
39 - block_size=args.block_size,
40 - gpu_memory_utilization=args.gpu_memory_utilization,
41 - load_format=args.load_format,
42 - distributed_executor_backend=args.distributed_executor_backend)
23 + llm = LLM(
24 + model=args.model,
25 + speculative_model=args.speculative_model,
26 + num_speculative_tokens=args.num_speculative_tokens,
27 + tokenizer=args.tokenizer,
28 + quantization=args.quantization,
29 + tensor_parallel_size=args.tensor_parallel_size,
30 + trust_remote_code=args.trust_remote_code,
31 + dtype=args.dtype,
32 + enforce_eager=args.enforce_eager,
33 + kv_cache_dtype=args.kv_cache_dtype,
34 + quantization_param_path=args.quantization_param_path,
35 + device=args.device,
36 + ray_workers_use_nsight=args.ray_workers_use_nsight,
37 + use_v2_block_manager=args.use_v2_block_manager,
38 + enable_chunked_prefill=args.enable_chunked_prefill,
39 + download_dir=args.download_dir,
40 + block_size=args.block_size,
41 + gpu_memory_utilization=args.gpu_memory_utilization,
42 + load_format=args.load_format,
43 + distributed_executor_backend=args.distributed_executor_backend,
44 + otlp_traces_endpoint=args.otlp_traces_endpoint,
45 + )
46
47 sampling_params = SamplingParams(
48 n=args.n,
@@ -254,5 +257,10 @@ if __name__ == '__main__':
257 help='Backend to use for distributed serving. When more than 1 GPU '
258 'is used, will be automatically set to "ray" if installed '
259 'or "mp" (multiprocessing) otherwise.')
260 + parser.add_argument(
261 + '--otlp-traces-endpoint',
262 + type=str,
263 + default=None,
264 + help='Target URL to which OpenTelemetry traces will be sent.')
265 args = parser.parse_args()
266 main(args)
examples/production_monitoring/Otel.md new
+82
@@ -0,0 +1,82 @@
1 +# Setup OpenTelemetry POC
2 +
3 +1. Install OpenTelemetry packages:
4 + ```
5 + pip install \
6 + opentelemetry-sdk \
7 + opentelemetry-api \
8 + opentelemetry-exporter-otlp \
9 + opentelemetry-semantic-conventions-ai
10 + ```
11 +
12 +1. Start Jaeger in a docker container:
13 + ```
14 + # From: https://www.jaegertracing.io/docs/1.57/getting-started/
15 + docker run --rm --name jaeger \
16 + -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
17 + -p 6831:6831/udp \
18 + -p 6832:6832/udp \
19 + -p 5778:5778 \
20 + -p 16686:16686 \
21 + -p 4317:4317 \
22 + -p 4318:4318 \
23 + -p 14250:14250 \
24 + -p 14268:14268 \
25 + -p 14269:14269 \
26 + -p 9411:9411 \
27 + jaegertracing/all-in-one:1.57
28 + ```
29 +
30 +1. In a new shell, export Jaeger IP:
31 + ```
32 + export JAEGER_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' jaeger)
33 + export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
34 + ```
35 + Then set vLLM's service name for OpenTelemetry, enable insecure connections to Jaeger and run vLLM:
36 + ```
37 + export OTEL_SERVICE_NAME="vllm-server"
38 + export OTEL_EXPORTER_OTLP_TRACES_INSECURE=true
39 + python -m vllm.entrypoints.openai.api_server --model="facebook/opt-125m" --otlp-traces-endpoint="$OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
40 + ```
41 +
42 +1. In a new shell, send requests with trace context from a dummy client
43 + ```
44 + export JAEGER_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' jaeger)
45 + export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
46 + export OTEL_EXPORTER_OTLP_TRACES_INSECURE=true
47 + export OTEL_SERVICE_NAME="client-service"
48 + python dummy_client.py
49 + ```
50 +
51 +1. Open Jaeger webui: http://localhost:16686/
52 +
53 + In the search pane, select `vllm-server` service and hit `Find Traces`. You should get a list of traces, one for each request.
54 + ![Traces](https://i.imgur.com/GYHhFjo.png)
55 +
56 +1. Clicking on a trace will show its spans and their tags. In this demo, each trace has 2 spans. One from the dummy client containing the prompt text and one from vLLM containing metadata about the request.
57 +![Spans details](https://i.imgur.com/OPf6CBL.png)
58 +
59 +## Exporter Protocol
60 +OpenTelemetry supports either `grpc` or `http/protobuf` as the transport protocol for trace data in the exporter.
61 +By default, `grpc` is used. To set `http/protobuf` as the protocol, configure the `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` environment variable as follows:
62 +```
63 +export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf
64 +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
65 +python -m vllm.entrypoints.openai.api_server --model="facebook/opt-125m" --otlp-traces-endpoint="$OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
66 +```
67 +
68 +## Instrumentation of FastAPI
69 +OpenTelemetry allows automatic instrumentation of FastAPI.
70 +1. Install the instrumentation library
71 + ```
72 + pip install opentelemetry-instrumentation-fastapi
73 + ```
74 +
75 +1. Run vLLM with `opentelemetry-instrument`
76 + ```
77 + opentelemetry-instrument python -m vllm.entrypoints.openai.api_server --model="facebook/opt-125m"
78 + ```
79 +
80 +1. Send a request to vLLM and find its trace in Jaeger. It should contain spans from FastAPI.
81 +
82 +![FastAPI Spans](https://i.imgur.com/hywvoOJ.png)
\ No newline at end of file
examples/production_monitoring/dummy_client.py new
+35
@@ -0,0 +1,35 @@
1 +import requests
2 +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
3 + OTLPSpanExporter)
4 +from opentelemetry.sdk.trace import TracerProvider
5 +from opentelemetry.sdk.trace.export import (BatchSpanProcessor,
6 + ConsoleSpanExporter)
7 +from opentelemetry.trace import SpanKind, set_tracer_provider
8 +from opentelemetry.trace.propagation.tracecontext import (
9 + TraceContextTextMapPropagator)
10 +
11 +trace_provider = TracerProvider()
12 +set_tracer_provider(trace_provider)
13 +
14 +trace_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
15 +trace_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
16 +
17 +tracer = trace_provider.get_tracer("dummy-client")
18 +
19 +url = "http://localhost:8000/v1/completions"
20 +with tracer.start_as_current_span("client-span", kind=SpanKind.CLIENT) as span:
21 + prompt = "San Francisco is a"
22 + span.set_attribute("prompt", prompt)
23 + headers = {}
24 + TraceContextTextMapPropagator().inject(headers)
25 + payload = {
26 + "model": "facebook/opt-125m",
27 + "prompt": prompt,
28 + "max_tokens": 10,
29 + "best_of": 20,
30 + "n": 3,
31 + "use_beam_search": "true",
32 + "temperature": 0.0,
33 + # "stream": True,
34 + }
35 + response = requests.post(url, headers=headers, json=payload)
tests/tracing/__init__.py
tests/tracing/test_tracing.py new
+116
@@ -0,0 +1,116 @@
1 +import os
2 +import threading
3 +from concurrent import futures
4 +from typing import Callable, Dict, Iterable, Literal
5 +
6 +import grpc
7 +import pytest
8 +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
9 + ExportTraceServiceResponse)
10 +from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import (
11 + TraceServiceServicer, add_TraceServiceServicer_to_server)
12 +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
13 +from opentelemetry.sdk.environment_variables import (
14 + OTEL_EXPORTER_OTLP_TRACES_INSECURE)
15 +
16 +from vllm import LLM, SamplingParams
17 +from vllm.tracing import SpanAttributes
18 +
19 +FAKE_TRACE_SERVER_ADDRESS = "localhost:4317"
20 +
21 +FieldName = Literal['bool_value', 'string_value', 'int_value', 'double_value',
22 + 'array_value']
23 +
24 +
25 +def decode_value(value: AnyValue):
26 + field_decoders: Dict[FieldName, Callable] = {
27 + "bool_value": (lambda v: v.bool_value),
28 + "string_value": (lambda v: v.string_value),
29 + "int_value": (lambda v: v.int_value),
30 + "double_value": (lambda v: v.double_value),
31 + "array_value":
32 + (lambda v: [decode_value(item) for item in v.array_value.values]),
33 + }
34 + for field, decoder in field_decoders.items():
35 + if value.HasField(field):
36 + return decoder(value)
37 + raise ValueError(f"Couldn't decode value: {value}")
38 +
39 +
40 +def decode_attributes(attributes: Iterable[KeyValue]):
41 + return {kv.key: decode_value(kv.value) for kv in attributes}
42 +
43 +
44 +class FakeTraceService(TraceServiceServicer):
45 +
46 + def __init__(self):
47 + self.request = None
48 + self.evt = threading.Event()
49 +
50 + def Export(self, request, context):
51 + self.request = request
52 + self.evt.set()
53 + return ExportTraceServiceResponse()
54 +
55 +
56 +@pytest.fixture
57 +def trace_service():
58 + """Fixture to set up a fake gRPC trace service"""
59 + server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
60 + service = FakeTraceService()
61 + add_TraceServiceServicer_to_server(service, server)
62 + server.add_insecure_port(FAKE_TRACE_SERVER_ADDRESS)
63 + server.start()
64 +
65 + yield service
66 +
67 + server.stop(None)
68 +
69 +
70 +def test_traces(trace_service):
71 + os.environ[OTEL_EXPORTER_OTLP_TRACES_INSECURE] = "true"
72 +
73 + sampling_params = SamplingParams(temperature=0.01,
74 + top_p=0.1,
75 + max_tokens=256)
76 + model = "facebook/opt-125m"
77 + llm = LLM(
78 + model=model,
79 + otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,
80 + )
81 + prompts = ["This is a short prompt"]
82 + outputs = llm.generate(prompts, sampling_params=sampling_params)
83 +
84 + timeout = 5
85 + if not trace_service.evt.wait(timeout):
86 + raise TimeoutError(
87 + f"The fake trace service didn't receive a trace within "
88 + f"the {timeout} seconds timeout")
89 +
90 + attributes = decode_attributes(trace_service.request.resource_spans[0].
91 + scope_spans[0].spans[0].attributes)
92 + assert attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) == model
93 + assert attributes.get(
94 + SpanAttributes.LLM_REQUEST_ID) == outputs[0].request_id
95 + assert attributes.get(
96 + SpanAttributes.LLM_REQUEST_TEMPERATURE) == sampling_params.temperature
97 + assert attributes.get(
98 + SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
99 + assert attributes.get(
100 + SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
101 + assert attributes.get(
102 + SpanAttributes.LLM_REQUEST_BEST_OF) == sampling_params.best_of
103 + assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
104 + assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
105 + outputs[0].prompt_token_ids)
106 + completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
107 + assert attributes.get(
108 + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS) == completion_tokens
109 + metrics = outputs[0].metrics
110 + assert attributes.get(
111 + SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE) == metrics.time_in_queue
112 + ttft = metrics.first_token_time - metrics.arrival_time
113 + assert attributes.get(
114 + SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
115 + e2e_time = metrics.finished_time - metrics.arrival_time
116 + assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
vllm/config.py
+13
@@ -10,6 +10,7 @@ from transformers import PretrainedConfig, PreTrainedTokenizerBase
10 from vllm.logger import init_logger
11 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12 from vllm.model_executor.models import ModelRegistry
13 +from vllm.tracing import is_otel_installed
14 from vllm.transformers_utils.config import get_config, get_hf_text_config
15 from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
16 is_hip, is_neuron, is_tpu, is_xpu)
@@ -1371,6 +1372,17 @@ class DecodingConfig:
1372 f"must be one of {valid_guided_backends}")
1373
1374
1375 +@dataclass
1376 +class ObservabilityConfig:
1377 + """Configuration for observability."""
1378 + otlp_traces_endpoint: Optional[str] = None
1379 +
1380 + def __post_init__(self):
1381 + if not is_otel_installed() and self.otlp_traces_endpoint is not None:
1382 + raise ValueError("OpenTelemetry packages must be installed before "
1383 + "configuring 'otlp_traces_endpoint'")
1384 +
1385 +
1386 @dataclass(frozen=True)
1387 class EngineConfig:
1388 """Dataclass which contains all engine-related configuration. This
@@ -1387,6 +1399,7 @@ class EngineConfig:
1399 vision_language_config: Optional[VisionLanguageConfig]
1400 speculative_config: Optional[SpeculativeConfig]
1401 decoding_config: Optional[DecodingConfig]
1402 + observability_config: Optional[ObservabilityConfig]
1403
1404 def __post_init__(self):
1405 """Verify configs are valid & consistent with each other.
vllm/engine/arg_utils.py
+28 -12
@@ -7,8 +7,9 @@ from typing import List, Optional, Tuple, Union
7
8 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
10 - ParallelConfig, SchedulerConfig, SpeculativeConfig,
11 - TokenizerPoolConfig, VisionLanguageConfig)
10 + ObservabilityConfig, ParallelConfig, SchedulerConfig,
11 + SpeculativeConfig, TokenizerPoolConfig,
12 + VisionLanguageConfig)
13 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
14 from vllm.utils import str_to_int_tuple
15
@@ -101,6 +102,8 @@ class EngineArgs:
102
103 qlora_adapter_name_or_path: Optional[str] = None
104
105 + otlp_traces_endpoint: Optional[str] = None
106 +
107 def __post_init__(self):
108 if self.tokenizer is None:
109 self.tokenizer = self.model
@@ -599,6 +602,13 @@ class EngineArgs:
602 type=str,
603 default=None,
604 help='Name or path of the QLoRA adapter.')
605 +
606 + parser.add_argument(
607 + '--otlp-traces-endpoint',
608 + type=str,
609 + default=None,
610 + help='Target URL to which OpenTelemetry traces will be sent.')
611 +
612 return parser
613
614 @classmethod
@@ -757,6 +767,9 @@ class EngineArgs:
767 decoding_config = DecodingConfig(
768 guided_decoding_backend=self.guided_decoding_backend)
769
770 + observability_config = ObservabilityConfig(
771 + otlp_traces_endpoint=self.otlp_traces_endpoint)
772 +
773 if (model_config.get_sliding_window() is not None
774 and scheduler_config.chunked_prefill_enabled
775 and not scheduler_config.use_v2_block_manager):
@@ -764,16 +777,19 @@ class EngineArgs:
777 "Chunked prefill is not supported with sliding window. "
778 "Set --disable-sliding-window to disable sliding window.")
779
767 - return EngineConfig(model_config=model_config,
768 - cache_config=cache_config,
769 - parallel_config=parallel_config,
770 - scheduler_config=scheduler_config,
771 - device_config=device_config,
772 - lora_config=lora_config,
773 - vision_language_config=vision_language_config,
774 - speculative_config=speculative_config,
775 - load_config=load_config,
776 - decoding_config=decoding_config)
780 + return EngineConfig(
781 + model_config=model_config,
782 + cache_config=cache_config,
783 + parallel_config=parallel_config,
784 + scheduler_config=scheduler_config,
785 + device_config=device_config,
786 + lora_config=lora_config,
787 + vision_language_config=vision_language_config,
788 + speculative_config=speculative_config,
789 + load_config=load_config,
790 + decoding_config=decoding_config,
791 + observability_config=observability_config,
792 + )
793
794
795 @dataclass
vllm/engine/async_llm_engine.py
+22
@@ -244,6 +244,9 @@ class _AsyncLLMEngine(LLMEngine):
244 # Log stats.
245 self.do_log_stats(scheduler_outputs, output)
246
247 + # Tracing
248 + self.do_tracing(scheduler_outputs)
249 +
250 if not request_outputs:
251 # Stop the execute model loop in parallel workers until there are
252 # more requests to process. This avoids waiting indefinitely in
@@ -285,6 +288,7 @@ class _AsyncLLMEngine(LLMEngine):
288 params: Union[SamplingParams, PoolingParams],
289 arrival_time: Optional[float] = None,
290 lora_request: Optional[LoRARequest] = None,
291 + trace_headers: Optional[Dict[str, str]] = None,
292 ) -> None:
293 if lora_request is not None and not self.lora_config:
294 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -301,6 +305,7 @@ class _AsyncLLMEngine(LLMEngine):
305 params=params,
306 arrival_time=arrival_time,
307 lora_request=lora_request,
308 + trace_headers=trace_headers,
309 )
310
311 async def check_health_async(self) -> None:
@@ -556,6 +561,7 @@ class AsyncLLMEngine:
561 params: Union[SamplingParams, PoolingParams],
562 arrival_time: Optional[float] = None,
563 lora_request: Optional[LoRARequest] = None,
564 + trace_headers: Optional[Dict[str, str]] = None,
565 ) -> AsyncStream:
566 if self.log_requests:
567 if isinstance(inputs, str):
@@ -597,6 +603,7 @@ class AsyncLLMEngine:
603 params=params,
604 arrival_time=arrival_time,
605 lora_request=lora_request,
606 + trace_headers=trace_headers,
607 )
608
609 return stream
@@ -607,6 +614,7 @@ class AsyncLLMEngine:
614 sampling_params: SamplingParams,
615 request_id: str,
616 lora_request: Optional[LoRARequest] = None,
617 + trace_headers: Optional[Dict[str, str]] = None,
618 ) -> AsyncIterator[RequestOutput]:
619 """Generate outputs for a request.
620
@@ -621,6 +629,7 @@ class AsyncLLMEngine:
629 sampling_params: The sampling parameters of the request.
630 request_id: The unique id of the request.
631 lora_request: LoRA request to use for generation, if any.
632 + trace_headers: OpenTelemetry trace headers.
633
634 Yields:
635 The output `RequestOutput` objects from the LLMEngine
@@ -674,6 +683,7 @@ class AsyncLLMEngine:
683 inputs,
684 sampling_params,
685 lora_request=lora_request,
686 + trace_headers=trace_headers,
687 ):
688 yield LLMEngine.validate_output(output, RequestOutput)
689
@@ -683,6 +693,7 @@ class AsyncLLMEngine:
693 pooling_params: PoolingParams,
694 request_id: str,
695 lora_request: Optional[LoRARequest] = None,
696 + trace_headers: Optional[Dict[str, str]] = None,
697 ) -> AsyncIterator[EmbeddingRequestOutput]:
698 """Generate outputs for a request from an embedding model.
699
@@ -697,6 +708,7 @@ class AsyncLLMEngine:
708 pooling_params: The pooling parameters of the request.
709 request_id: The unique id of the request.
710 lora_request: LoRA request to use for generation, if any.
711 + trace_headers: OpenTelemetry trace headers.
712
713 Yields:
714 The output `EmbeddingRequestOutput` objects from the LLMEngine
@@ -748,6 +760,7 @@ class AsyncLLMEngine:
760 inputs,
761 pooling_params,
762 lora_request=lora_request,
763 + trace_headers=trace_headers,
764 ):
765 yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
766
@@ -758,6 +771,7 @@ class AsyncLLMEngine:
771 params: Union[SamplingParams, PoolingParams],
772 *,
773 lora_request: Optional[LoRARequest] = None,
774 + trace_headers: Optional[Dict[str, str]] = None,
775 ) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
776 """Common logic to process requests with SamplingParams or
777 PoolingParams."""
@@ -769,6 +783,7 @@ class AsyncLLMEngine:
783 params,
784 arrival_time=arrival_time,
785 lora_request=lora_request,
786 + trace_headers=trace_headers,
787 )
788
789 try:
@@ -848,3 +863,10 @@ class AsyncLLMEngine:
863 else:
864 await self.engine.check_health_async()
865 logger.debug("Health check took %fs", time.perf_counter() - t)
866 +
867 + async def is_tracing_enabled(self) -> bool:
868 + if self.engine_use_ray:
869 + return await self.engine.is_tracing_enabled.remote( # type: ignore
870 + )
871 + else:
872 + return self.engine.is_tracing_enabled()
vllm/engine/llm_engine.py
+93 -9
@@ -1,14 +1,14 @@
1 import time
2 from contextlib import contextmanager
3 -from typing import TYPE_CHECKING, ClassVar, Iterable, List, Optional
3 +from typing import TYPE_CHECKING, ClassVar, Dict, Iterable, List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Set, Type, TypeVar, Union
6
7 from transformers import GenerationConfig, PreTrainedTokenizer
8
9 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
10 - LoRAConfig, ModelConfig, ParallelConfig,
11 - SchedulerConfig, SpeculativeConfig,
10 + LoRAConfig, ModelConfig, ObservabilityConfig,
11 + ParallelConfig, SchedulerConfig, SpeculativeConfig,
12 VisionLanguageConfig)
13 from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
14 SchedulerOutputs)
@@ -31,6 +31,8 @@ from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
31 PoolerOutput, SamplerOutput, Sequence,
32 SequenceGroup, SequenceGroupMetadata,
33 SequenceStatus)
34 +from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
35 + init_tracer)
36 from vllm.transformers_utils.detokenizer import Detokenizer
37 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
38 get_tokenizer_group)
@@ -154,6 +156,7 @@ class LLMEngine:
156 vision_language_config: Optional[VisionLanguageConfig],
157 speculative_config: Optional[SpeculativeConfig],
158 decoding_config: Optional[DecodingConfig],
159 + observability_config: Optional[ObservabilityConfig],
160 executor_class: Type[ExecutorBase],
161 log_stats: bool,
162 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -168,7 +171,8 @@ class LLMEngine:
171 "disable_custom_all_reduce=%s, quantization=%s, "
172 "enforce_eager=%s, kv_cache_dtype=%s, "
173 "quantization_param_path=%s, device_config=%s, "
171 - "decoding_config=%r, seed=%d, served_model_name=%s)",
174 + "decoding_config=%r, observability_config=%r, "
175 + "seed=%d, served_model_name=%s)",
176 VLLM_VERSION,
177 model_config.model,
178 speculative_config,
@@ -192,6 +196,7 @@ class LLMEngine:
196 model_config.quantization_param_path,
197 device_config.device,
198 decoding_config,
199 + observability_config,
200 model_config.seed,
201 model_config.served_model_name,
202 )
@@ -207,6 +212,8 @@ class LLMEngine:
212 self.speculative_config = speculative_config
213 self.load_config = load_config
214 self.decoding_config = decoding_config or DecodingConfig()
215 + self.observability_config = observability_config or ObservabilityConfig(
216 + )
217 self.log_stats = log_stats
218
219 if not self.model_config.skip_tokenizer_init:
@@ -288,6 +295,12 @@ class LLMEngine:
295 max_model_len=self.model_config.max_model_len)
296 self.stat_logger.info("cache_config", self.cache_config)
297
298 + self.tracer = None
299 + if self.observability_config.otlp_traces_endpoint:
300 + self.tracer = init_tracer(
301 + "vllm.llm_engine",
302 + self.observability_config.otlp_traces_endpoint)
303 +
304 # Create sequence output processor, e.g. for beam search or
305 # speculative decoding.
306 self.output_processor = (
@@ -444,6 +457,7 @@ class LLMEngine:
457 params: Union[SamplingParams, PoolingParams],
458 arrival_time: float,
459 lora_request: Optional[LoRARequest],
460 + trace_headers: Optional[Dict[str, str]] = None,
461 ) -> None:
462 # Create the sequences.
463 block_size = self.cache_config.block_size
@@ -461,6 +475,7 @@ class LLMEngine:
475 params,
476 arrival_time=arrival_time,
477 lora_request=lora_request,
478 + trace_headers=trace_headers,
479 )
480 elif isinstance(params, PoolingParams):
481 seq_group = self._create_sequence_group_with_pooling(
@@ -507,6 +522,7 @@ class LLMEngine:
522 params: Union[SamplingParams, PoolingParams],
523 arrival_time: Optional[float] = None,
524 lora_request: Optional[LoRARequest] = None,
525 + trace_headers: Optional[Dict[str, str]] = None,
526 ) -> None:
527 """Add a request to the engine's request pool.
528
@@ -524,6 +540,7 @@ class LLMEngine:
540 :class:`~vllm.PoolingParams` for pooling.
541 arrival_time: The arrival time of the request. If None, we use
542 the current monotonic time.
543 + trace_headers: OpenTelemetry trace headers.
544
545 Details:
546 - Set arrival_time to the current time if it is None.
@@ -565,6 +582,7 @@ class LLMEngine:
582 params=params,
583 arrival_time=arrival_time,
584 lora_request=lora_request,
585 + trace_headers=trace_headers,
586 )
587
588 def _create_sequence_group_with_sampling(
@@ -574,6 +592,7 @@ class LLMEngine:
592 sampling_params: SamplingParams,
593 arrival_time: float,
594 lora_request: Optional[LoRARequest],
595 + trace_headers: Optional[Dict[str, str]] = None,
596 ) -> SequenceGroup:
597 """Creates a SequenceGroup with SamplingParams."""
598 max_logprobs = self.get_model_config().max_logprobs
@@ -595,11 +614,14 @@ class LLMEngine:
614 self.generation_config_fields)
615
616 # Create the sequence group.
598 - seq_group = SequenceGroup(request_id=request_id,
599 - seqs=[seq],
600 - arrival_time=arrival_time,
601 - sampling_params=sampling_params,
602 - lora_request=lora_request)
617 + seq_group = SequenceGroup(
618 + request_id=request_id,
619 + seqs=[seq],
620 + arrival_time=arrival_time,
621 + sampling_params=sampling_params,
622 + lora_request=lora_request,
623 + trace_headers=trace_headers,
624 + )
625
626 return seq_group
627
@@ -793,6 +815,9 @@ class LLMEngine:
815 # Log stats.
816 self.do_log_stats(scheduler_outputs, output)
817
818 + # Tracing
819 + self.do_tracing(scheduler_outputs)
820 +
821 if not request_outputs:
822 # Stop the execute model loop in parallel workers until there are
823 # more requests to process. This avoids waiting indefinitely in
@@ -986,3 +1011,62 @@ class LLMEngine:
1011
1012 def check_health(self) -> None:
1013 self.model_executor.check_health()
1014 +
1015 + def is_tracing_enabled(self) -> bool:
1016 + return self.tracer is not None
1017 +
1018 + def do_tracing(self, scheduler_outputs: SchedulerOutputs) -> None:
1019 + if self.tracer is None:
1020 + return
1021 +
1022 + for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
1023 + seq_group = scheduled_seq_group.seq_group
1024 + if seq_group.is_finished():
1025 + self.create_trace_span(seq_group)
1026 +
1027 + def create_trace_span(self, seq_group: SequenceGroup) -> None:
1028 + if self.tracer is None or seq_group.sampling_params is None:
1029 + return
1030 + arrival_time_nano_seconds = int(seq_group.metrics.arrival_time * 1e9)
1031 +
1032 + trace_context = extract_trace_context(seq_group.trace_headers)
1033 +
1034 + with self.tracer.start_as_current_span(
1035 + "llm_request",
1036 + kind=SpanKind.SERVER,
1037 + context=trace_context,
1038 + start_time=arrival_time_nano_seconds) as seq_span:
1039 + metrics = seq_group.metrics
1040 + ttft = metrics.first_token_time - metrics.arrival_time
1041 + e2e_time = metrics.finished_time - metrics.arrival_time
1042 + # attribute names are based on
1043 + # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/llm-spans.md
1044 + seq_span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL,
1045 + self.model_config.model)
1046 + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_ID,
1047 + seq_group.request_id)
1048 + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_TEMPERATURE,
1049 + seq_group.sampling_params.temperature)
1050 + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_TOP_P,
1051 + seq_group.sampling_params.top_p)
1052 + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_MAX_TOKENS,
1053 + seq_group.sampling_params.max_tokens)
1054 + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_BEST_OF,
1055 + seq_group.sampling_params.best_of)
1056 + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_N,
1057 + seq_group.sampling_params.n)
1058 + seq_span.set_attribute(SpanAttributes.LLM_USAGE_NUM_SEQUENCES,
1059 + seq_group.num_seqs())
1060 + seq_span.set_attribute(SpanAttributes.LLM_USAGE_PROMPT_TOKENS,
1061 + len(seq_group.prompt_token_ids))
1062 + seq_span.set_attribute(
1063 + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS,
1064 + sum([
1065 + seq.get_output_len()
1066 + for seq in seq_group.get_finished_seqs()
1067 + ]))
1068 + seq_span.set_attribute(SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE,
1069 + metrics.time_in_queue)
1070 + seq_span.set_attribute(
1071 + SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
1072 + seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
vllm/entrypoints/openai/serving_chat.py
+11
@@ -31,6 +31,8 @@ from vllm.multimodal.utils import (async_get_and_parse_image,
31 get_full_image_text_prompt)
32 from vllm.outputs import RequestOutput
33 from vllm.sequence import Logprob
34 +from vllm.tracing import (contains_trace_headers, extract_trace_headers,
35 + log_tracing_disabled_warning)
36 from vllm.utils import random_uuid
37
38 logger = init_logger(__name__)
@@ -267,11 +269,20 @@ class OpenAIServingChat(OpenAIServing):
269 if image_data is not None:
270 inputs["multi_modal_data"] = image_data
271
272 + is_tracing_enabled = await self.engine.is_tracing_enabled()
273 + trace_headers = None
274 + if is_tracing_enabled and raw_request:
275 + trace_headers = extract_trace_headers(raw_request.headers)
276 + if not is_tracing_enabled and raw_request and contains_trace_headers(
277 + raw_request.headers):
278 + log_tracing_disabled_warning()
279 +
280 result_generator = self.engine.generate(
281 inputs,
282 sampling_params,
283 request_id,
284 lora_request,
285 + trace_headers=trace_headers,
286 )
287 # Streaming response
288 if request.stream:
vllm/entrypoints/openai/serving_completion.py
+11
@@ -24,6 +24,8 @@ from vllm.model_executor.guided_decoding import (
24 get_guided_decoding_logits_processor)
25 from vllm.outputs import RequestOutput
26 from vllm.sequence import Logprob
27 +from vllm.tracing import (contains_trace_headers, extract_trace_headers,
28 + log_tracing_disabled_warning)
29 from vllm.utils import merge_async_iterators, random_uuid
30
31 logger = init_logger(__name__)
@@ -125,6 +127,14 @@ class OpenAIServingCompletion(OpenAIServing):
127 truncate_prompt_tokens)
128 prompt_ids, prompt_text = prompt_formats
129
130 + is_tracing_enabled = await self.engine.is_tracing_enabled()
131 + trace_headers = None
132 + if is_tracing_enabled:
133 + trace_headers = extract_trace_headers(raw_request.headers)
134 + if not is_tracing_enabled and contains_trace_headers(
135 + raw_request.headers):
136 + log_tracing_disabled_warning()
137 +
138 generator = self.engine.generate(
139 {
140 "prompt": prompt_text,
@@ -133,6 +143,7 @@ class OpenAIServingCompletion(OpenAIServing):
143 sampling_params,
144 f"{request_id}-{i}",
145 lora_request=lora_request,
146 + trace_headers=trace_headers,
147 )
148
149 generators.append(generator)
vllm/sequence.py
+3
@@ -414,6 +414,7 @@ class SequenceGroup:
414 for an embedding model.
415 encoder_seq: Optional, the single encoder sequence. Should be None
416 unless you are working with an encoder/decoder model.
417 + trace_headers: OpenTelemetry trace headers.
418 """
419
420 def __init__(
@@ -426,6 +427,7 @@ class SequenceGroup:
427 embeddings: Optional[List[float]] = None,
428 pooling_params: Optional[PoolingParams] = None,
429 encoder_seq: Optional[Sequence] = None,
430 + trace_headers: Optional[Dict[str, str]] = None,
431 ) -> None:
432 self.request_id = request_id
433 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -441,6 +443,7 @@ class SequenceGroup:
443 self.embeddings = embeddings
444 self.pooling_params = pooling_params
445 self.encoder_seq = encoder_seq
446 + self.trace_headers = trace_headers
447
448 @property
449 def prompt(self) -> Optional[str]:
vllm/tracing.py new
+104
@@ -0,0 +1,104 @@
1 +import os
2 +from typing import Mapping, Optional
3 +
4 +from vllm.logger import init_logger
5 +from vllm.utils import run_once
6 +
7 +TRACE_HEADERS = ["traceparent", "tracestate"]
8 +
9 +logger = init_logger(__name__)
10 +
11 +_is_otel_installed = False
12 +try:
13 + from opentelemetry.context.context import Context
14 + from opentelemetry.sdk.environment_variables import (
15 + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL)
16 + from opentelemetry.sdk.trace import TracerProvider
17 + from opentelemetry.sdk.trace.export import BatchSpanProcessor
18 + from opentelemetry.semconv.ai import SpanAttributes as BaseSpanAttributes
19 + from opentelemetry.trace import SpanKind, Tracer, set_tracer_provider
20 + from opentelemetry.trace.propagation.tracecontext import (
21 + TraceContextTextMapPropagator)
22 + _is_otel_installed = True
23 +except ImportError:
24 +
25 + class Context: # type: ignore
26 + pass
27 +
28 + class BaseSpanAttributes: # type: ignore
29 + pass
30 +
31 + class SpanKind: # type: ignore
32 + pass
33 +
34 + class Tracer: # type: ignore
35 + pass
36 +
37 +
38 +def is_otel_installed() -> bool:
39 + return _is_otel_installed
40 +
41 +
42 +def init_tracer(instrumenting_module_name: str,
43 + otlp_traces_endpoint: str) -> Optional[Tracer]:
44 + assert is_otel_installed(), ("OpenTelemetry packages must be installed "
45 + "prior to initializing a tracer")
46 + trace_provider = TracerProvider()
47 +
48 + span_exporter = get_span_exporter(otlp_traces_endpoint)
49 + trace_provider.add_span_processor(BatchSpanProcessor(span_exporter))
50 + set_tracer_provider(trace_provider)
51 +
52 + tracer = trace_provider.get_tracer(instrumenting_module_name)
53 + return tracer
54 +
55 +
56 +def get_span_exporter(endpoint):
57 + protocol = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "grpc")
58 + if protocol == "grpc":
59 + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
60 + OTLPSpanExporter)
61 + elif protocol == "http/protobuf":
62 + from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
63 + OTLPSpanExporter)
64 + else:
65 + raise ValueError(
66 + f"Unsupported OTLP protocol '{protocol}' is configured")
67 +
68 + return OTLPSpanExporter(endpoint=endpoint)
69 +
70 +
71 +def extract_trace_context(
72 + headers: Optional[Mapping[str, str]]) -> Optional[Context]:
73 + if is_otel_installed():
74 + headers = headers or {}
75 + return TraceContextTextMapPropagator().extract(headers)
76 + else:
77 + return None
78 +
79 +
80 +def extract_trace_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
81 +
82 + return {h: headers[h] for h in TRACE_HEADERS if h in headers}
83 +
84 +
85 +class SpanAttributes(BaseSpanAttributes):
86 + # The following span attribute names are added here because they are missing
87 + # from the Semantic Conventions for LLM.
88 + LLM_REQUEST_ID = "gen_ai.request.id"
89 + LLM_REQUEST_BEST_OF = "gen_ai.request.best_of"
90 + LLM_REQUEST_N = "gen_ai.request.n"
91 + LLM_USAGE_NUM_SEQUENCES = "gen_ai.usage.num_sequences"
92 + LLM_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
93 + LLM_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token"
94 + LLM_LATENCY_E2E = "gen_ai.latency.e2e"
95 +
96 +
97 +def contains_trace_headers(headers: Mapping[str, str]) -> bool:
98 + return any(h in headers for h in TRACE_HEADERS)
99 +
100 +
101 +@run_once
102 +def log_tracing_disabled_warning() -> None:
103 + logger.warning(
104 + "Received a request with trace context but tracing is disabled")
vllm/utils.py
+12
@@ -763,3 +763,15 @@ def cuda_device_count_stateless() -> int:
763 # after https://github.com/pytorch/pytorch/pull/122815 is released.
764
765 return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
766 +
767 +
768 +#From: https://stackoverflow.com/a/4104188/2749989
769 +def run_once(f):
770 +
771 + def wrapper(*args, **kwargs) -> Any:
772 + if not wrapper.has_run: # type: ignore[attr-defined]
773 + wrapper.has_run = True # type: ignore[attr-defined]
774 + return f(*args, **kwargs)
775 +
776 + wrapper.has_run = False # type: ignore[attr-defined]
777 + return wrapper