@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cd5601ac37
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[BugFix] Prevent exporting duplicate OpenTelemetry spans (#9017)

Ronen Schaffer committed Oct 22, 2024 at 21:11 UTC cd5601ac37baadb6a6efa3450f1546ddab84c973
2 files changed +36 -7
tests/tracing/test_tracing.py
+26 -4
@@ -87,8 +87,19 @@ def test_traces(trace_service):
87 f"The fake trace service didn't receive a trace within "
88 f"the {timeout} seconds timeout")
89
90 - attributes = decode_attributes(trace_service.request.resource_spans[0].
91 - scope_spans[0].spans[0].attributes)
90 + request = trace_service.request
91 + assert len(request.resource_spans) == 1, (
92 + f"Expected 1 resource span, "
93 + f"but got {len(request.resource_spans)}")
94 + assert len(request.resource_spans[0].scope_spans) == 1, (
95 + f"Expected 1 scope span, "
96 + f"but got {len(request.resource_spans[0].scope_spans)}")
97 + assert len(request.resource_spans[0].scope_spans[0].spans) == 1, (
98 + f"Expected 1 span, "
99 + f"but got {len(request.resource_spans[0].scope_spans[0].spans)}")
100 +
101 + attributes = decode_attributes(
102 + request.resource_spans[0].scope_spans[0].spans[0].attributes)
103 assert attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) == model
104 assert attributes.get(
105 SpanAttributes.LLM_REQUEST_ID) == outputs[0].request_id
@@ -142,8 +153,19 @@ def test_traces_with_detailed_steps(trace_service):
153 f"The fake trace service didn't receive a trace within "
154 f"the {timeout} seconds timeout")
155
145 - attributes = decode_attributes(trace_service.request.resource_spans[0].
146 - scope_spans[0].spans[0].attributes)
156 + request = trace_service.request
157 + assert len(request.resource_spans) == 1, (
158 + f"Expected 1 resource span, "
159 + f"but got {len(request.resource_spans)}")
160 + assert len(request.resource_spans[0].scope_spans) == 1, (
161 + f"Expected 1 scope span, "
162 + f"but got {len(request.resource_spans[0].scope_spans)}")
163 + assert len(request.resource_spans[0].scope_spans[0].spans) == 1, (
164 + f"Expected 1 span, "
165 + f"but got {len(request.resource_spans[0].scope_spans[0].spans)}")
166 +
167 + attributes = decode_attributes(
168 + request.resource_spans[0].scope_spans[0].spans[0].attributes)
169 assert attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) == model
170 assert attributes.get(
171 SpanAttributes.LLM_REQUEST_ID) == outputs[0].request_id
vllm/engine/llm_engine.py
+10 -3
@@ -1245,7 +1245,7 @@ class LLMEngine:
1245 skip)
1246
1247 # Tracing
1248 - self.do_tracing(scheduler_outputs)
1248 + self.do_tracing(scheduler_outputs, finished_before)
1249
1250 return None
1251
@@ -1840,11 +1840,18 @@ class LLMEngine:
1840 def is_tracing_enabled(self) -> bool:
1841 return self.tracer is not None
1842
1843 - def do_tracing(self, scheduler_outputs: SchedulerOutputs) -> None:
1843 + def do_tracing(self,
1844 + scheduler_outputs: SchedulerOutputs,
1845 + finished_before: Optional[List[int]] = None) -> None:
1846 if self.tracer is None:
1847 return
1848
1847 - for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
1849 + for idx, scheduled_seq_group in enumerate(
1850 + scheduler_outputs.scheduled_seq_groups):
1851 + # Skip double tracing when using async output proc
1852 + if finished_before and idx in finished_before:
1853 + continue
1854 +
1855 seq_group = scheduled_seq_group.seq_group
1856 if seq_group.is_finished():
1857 self.create_trace_span(seq_group)