@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 347eeebe3b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc] Remove experimental dep from tracing.py (#12007)

Signed-off-by: Adrian Cole <adrian.cole@elastic.co>

Adrian Cole committed Jan 21, 2025 at 11:51 UTC 347eeebe3bd84c1fb59af72be753ec42cc74b799
3 files changed +66 -60
tests/tracing/test_tracing.py
+30 -30
@@ -100,32 +100,32 @@ def test_traces(trace_service):
100
101 attributes = decode_attributes(
102 request.resource_spans[0].scope_spans[0].spans[0].attributes)
103 - assert attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) == model
103 + assert attributes.get(SpanAttributes.GEN_AI_RESPONSE_MODEL) == model
104 assert attributes.get(
105 - SpanAttributes.LLM_REQUEST_ID) == outputs[0].request_id
105 + SpanAttributes.GEN_AI_REQUEST_ID) == outputs[0].request_id
106 + assert attributes.get(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE
107 + ) == sampling_params.temperature
108 assert attributes.get(
107 - SpanAttributes.LLM_REQUEST_TEMPERATURE) == sampling_params.temperature
109 + SpanAttributes.GEN_AI_REQUEST_TOP_P) == sampling_params.top_p
110 assert attributes.get(
109 - SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
110 - assert attributes.get(
111 - SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
112 - assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
113 - assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
111 + SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
112 + assert attributes.get(SpanAttributes.GEN_AI_REQUEST_N) == sampling_params.n
113 + assert attributes.get(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS) == len(
114 outputs[0].prompt_token_ids)
115 completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
116 assert attributes.get(
117 - SpanAttributes.LLM_USAGE_COMPLETION_TOKENS) == completion_tokens
117 + SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS) == completion_tokens
118 metrics = outputs[0].metrics
119 assert attributes.get(
120 - SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE) == metrics.time_in_queue
120 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE) == metrics.time_in_queue
121 ttft = metrics.first_token_time - metrics.arrival_time
122 assert attributes.get(
123 - SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
123 + SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
124 e2e_time = metrics.finished_time - metrics.arrival_time
125 - assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
125 + assert attributes.get(SpanAttributes.GEN_AI_LATENCY_E2E) == e2e_time
126 assert metrics.scheduler_time > 0
127 - assert attributes.get(
128 - SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER) == metrics.scheduler_time
127 + assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_SCHEDULER
128 + ) == metrics.scheduler_time
129 # Model forward and model execute should be none, since detailed traces is
130 # not enabled.
131 assert metrics.model_forward_time is None
@@ -166,37 +166,37 @@ def test_traces_with_detailed_steps(trace_service):
166
167 attributes = decode_attributes(
168 request.resource_spans[0].scope_spans[0].spans[0].attributes)
169 - assert attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) == model
169 + assert attributes.get(SpanAttributes.GEN_AI_RESPONSE_MODEL) == model
170 assert attributes.get(
171 - SpanAttributes.LLM_REQUEST_ID) == outputs[0].request_id
171 + SpanAttributes.GEN_AI_REQUEST_ID) == outputs[0].request_id
172 + assert attributes.get(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE
173 + ) == sampling_params.temperature
174 assert attributes.get(
173 - SpanAttributes.LLM_REQUEST_TEMPERATURE) == sampling_params.temperature
175 + SpanAttributes.GEN_AI_REQUEST_TOP_P) == sampling_params.top_p
176 assert attributes.get(
175 - SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
176 - assert attributes.get(
177 - SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
178 - assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
179 - assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
177 + SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
178 + assert attributes.get(SpanAttributes.GEN_AI_REQUEST_N) == sampling_params.n
179 + assert attributes.get(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS) == len(
180 outputs[0].prompt_token_ids)
181 completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
182 assert attributes.get(
183 - SpanAttributes.LLM_USAGE_COMPLETION_TOKENS) == completion_tokens
183 + SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS) == completion_tokens
184 metrics = outputs[0].metrics
185 assert attributes.get(
186 - SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE) == metrics.time_in_queue
186 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE) == metrics.time_in_queue
187 ttft = metrics.first_token_time - metrics.arrival_time
188 assert attributes.get(
189 - SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
189 + SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
190 e2e_time = metrics.finished_time - metrics.arrival_time
191 - assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
191 + assert attributes.get(SpanAttributes.GEN_AI_LATENCY_E2E) == e2e_time
192 assert metrics.scheduler_time > 0
193 - assert attributes.get(
194 - SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER) == metrics.scheduler_time
193 + assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_SCHEDULER
194 + ) == metrics.scheduler_time
195 assert metrics.model_forward_time > 0
196 assert attributes.get(
197 - SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_FORWARD) == pytest.approx(
197 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_FORWARD) == pytest.approx(
198 metrics.model_forward_time / 1000)
199 assert metrics.model_execute_time > 0
200 - assert attributes.get(SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE
200 + assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_EXECUTE
201 ) == metrics.model_execute_time
202 assert metrics.model_forward_time < 1000 * metrics.model_execute_time
vllm/engine/llm_engine.py
+15 -17
@@ -1857,46 +1857,44 @@ class LLMEngine:
1857 metrics = seq_group.metrics
1858 ttft = metrics.first_token_time - metrics.arrival_time
1859 e2e_time = metrics.finished_time - metrics.arrival_time
1860 - # attribute names are based on
1861 - # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/llm-spans.md
1862 - seq_span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL,
1860 + seq_span.set_attribute(SpanAttributes.GEN_AI_RESPONSE_MODEL,
1861 self.model_config.model)
1864 - seq_span.set_attribute(SpanAttributes.LLM_REQUEST_ID,
1862 + seq_span.set_attribute(SpanAttributes.GEN_AI_REQUEST_ID,
1863 seq_group.request_id)
1866 - seq_span.set_attribute(SpanAttributes.LLM_REQUEST_TEMPERATURE,
1864 + seq_span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE,
1865 seq_group.sampling_params.temperature)
1868 - seq_span.set_attribute(SpanAttributes.LLM_REQUEST_TOP_P,
1866 + seq_span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TOP_P,
1867 seq_group.sampling_params.top_p)
1870 - seq_span.set_attribute(SpanAttributes.LLM_REQUEST_MAX_TOKENS,
1868 + seq_span.set_attribute(SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS,
1869 seq_group.sampling_params.max_tokens)
1872 - seq_span.set_attribute(SpanAttributes.LLM_REQUEST_N,
1870 + seq_span.set_attribute(SpanAttributes.GEN_AI_REQUEST_N,
1871 seq_group.sampling_params.n)
1874 - seq_span.set_attribute(SpanAttributes.LLM_USAGE_NUM_SEQUENCES,
1872 + seq_span.set_attribute(SpanAttributes.GEN_AI_USAGE_NUM_SEQUENCES,
1873 seq_group.num_seqs())
1876 - seq_span.set_attribute(SpanAttributes.LLM_USAGE_PROMPT_TOKENS,
1874 + seq_span.set_attribute(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS,
1875 len(seq_group.prompt_token_ids))
1876 seq_span.set_attribute(
1879 - SpanAttributes.LLM_USAGE_COMPLETION_TOKENS,
1877 + SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS,
1878 sum([
1879 seq.get_output_len()
1880 for seq in seq_group.get_finished_seqs()
1881 ]))
1884 - seq_span.set_attribute(SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE,
1882 + seq_span.set_attribute(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE,
1883 metrics.time_in_queue)
1884 seq_span.set_attribute(
1887 - SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
1888 - seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
1885 + SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
1886 + seq_span.set_attribute(SpanAttributes.GEN_AI_LATENCY_E2E, e2e_time)
1887 if metrics.scheduler_time is not None:
1888 seq_span.set_attribute(
1891 - SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER,
1889 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_SCHEDULER,
1890 metrics.scheduler_time)
1891 if metrics.model_forward_time is not None:
1892 seq_span.set_attribute(
1895 - SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_FORWARD,
1893 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_FORWARD,
1894 metrics.model_forward_time / 1000.0)
1895 if metrics.model_execute_time is not None:
1896 seq_span.set_attribute(
1899 - SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE,
1897 + SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_EXECUTE,
1898 metrics.model_execute_time)
1899
1900 def _validate_model_inputs(self, inputs: ProcessorInputs,
vllm/tracing.py
+21 -13
@@ -16,7 +16,6 @@ try:
16 OTEL_EXPORTER_OTLP_TRACES_PROTOCOL)
17 from opentelemetry.sdk.trace import TracerProvider
18 from opentelemetry.sdk.trace.export import BatchSpanProcessor
19 - from opentelemetry.semconv_ai import SpanAttributes as BaseSpanAttributes
19 from opentelemetry.trace import SpanKind, Tracer, set_tracer_provider
20 from opentelemetry.trace.propagation.tracecontext import (
21 TraceContextTextMapPropagator)
@@ -92,21 +91,30 @@ def extract_trace_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
91 return {h: headers[h] for h in TRACE_HEADERS if h in headers}
92
93
95 -class SpanAttributes(BaseSpanAttributes):
96 - # The following span attribute names are added here because they are missing
97 - # from the Semantic Conventions for LLM.
98 - LLM_REQUEST_ID = "gen_ai.request.id"
99 - LLM_REQUEST_N = "gen_ai.request.n"
100 - LLM_USAGE_NUM_SEQUENCES = "gen_ai.usage.num_sequences"
101 - LLM_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
102 - LLM_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token"
103 - LLM_LATENCY_E2E = "gen_ai.latency.e2e"
104 - LLM_LATENCY_TIME_IN_SCHEDULER = "gen_ai.latency.time_in_scheduler"
94 +class SpanAttributes:
95 + # Attribute names copied from here to avoid version conflicts:
96 + # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md
97 + GEN_AI_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens"
98 + GEN_AI_USAGE_PROMPT_TOKENS = "gen_ai.usage.prompt_tokens"
99 + GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
100 + GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p"
101 + GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"
102 + GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"
103 + # Attribute names added until they are added to the semantic conventions:
104 + GEN_AI_REQUEST_ID = "gen_ai.request.id"
105 + GEN_AI_REQUEST_N = "gen_ai.request.n"
106 + GEN_AI_USAGE_NUM_SEQUENCES = "gen_ai.usage.num_sequences"
107 + GEN_AI_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
108 + GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token"
109 + GEN_AI_LATENCY_E2E = "gen_ai.latency.e2e"
110 + GEN_AI_LATENCY_TIME_IN_SCHEDULER = "gen_ai.latency.time_in_scheduler"
111 # Time taken in the forward pass for this across all workers
106 - LLM_LATENCY_TIME_IN_MODEL_FORWARD = "gen_ai.latency.time_in_model_forward"
112 + GEN_AI_LATENCY_TIME_IN_MODEL_FORWARD = (
113 + "gen_ai.latency.time_in_model_forward")
114 # Time taken in the model execute function. This will include model
115 # forward, block/sync across workers, cpu-gpu sync time and sampling time.
109 - LLM_LATENCY_TIME_IN_MODEL_EXECUTE = "gen_ai.latency.time_in_model_execute"
116 + GEN_AI_LATENCY_TIME_IN_MODEL_EXECUTE = (
117 + "gen_ai.latency.time_in_model_execute")
118
119
120 def contains_trace_headers(headers: Mapping[str, str]) -> bool: