[ROCm][CI] Use spawn around the threaded OTLP test (#47003)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jun 29, 2026 at 16:34 UTC
8632c884dc440e231c8b7aef65a8795b80fe6676
1 file changed
+70
-51
tests/v1/tracing/test_tracing.py
+70
-51
@@ -7,6 +7,8 @@ import time
7
from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_TRACES_INSECURE
8
9
from vllm import LLM, SamplingParams
10
+from vllm.distributed import cleanup_dist_env_and_memory
11
+from vllm.platforms import current_platform
12
from vllm.tracing import SpanAttributes
13
14
# Import shared fixtures from the tracing conftest
@@ -23,6 +25,11 @@ def test_traces(
25
):
26
with monkeypatch.context() as m:
27
m.setenv(OTEL_EXPORTER_OTLP_TRACES_INSECURE, "true")
28
+ if current_platform.is_rocm():
29
+ # The fake OTLP server starts gRPC worker threads before the engine
30
+ # core is launched. On ROCm CI, forking while those threads are
31
+ # active can segfault in gRPC during engine startup or teardown.
32
+ m.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
33
34
sampling_params = SamplingParams(
35
temperature=0.01,
@@ -30,58 +37,70 @@ def test_traces(
37
max_tokens=256,
38
)
39
model = "facebook/opt-125m"
33
- llm = LLM(
34
- model=model,
35
- otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,
36
- gpu_memory_utilization=0.3,
37
- disable_log_stats=False,
38
- )
39
- prompts = ["This is a short prompt"]
40
- outputs = llm.generate(prompts, sampling_params=sampling_params)
41
- print(f"test_traces outputs is : {outputs}")
40
+ llm = None
41
+ try:
42
+ llm = LLM(
43
+ model=model,
44
+ otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,
45
+ gpu_memory_utilization=0.3,
46
+ disable_log_stats=False,
47
+ )
48
+ prompts = ["This is a short prompt"]
49
+ outputs = llm.generate(prompts, sampling_params=sampling_params)
50
+ print(f"test_traces outputs is : {outputs}")
51
43
- # Wait for the "llm_request" span to be exported.
44
- # The BatchSpanProcessor batches spans and exports them periodically,
45
- # so we need to wait specifically for the llm_request span to appear.
46
- timeout = 15
47
- deadline = time.time() + timeout
48
- llm_request_spans = []
49
- while time.time() < deadline:
50
- all_spans = trace_service.get_all_spans()
51
- llm_request_spans = [s for s in all_spans if s["name"] == "llm_request"]
52
- if llm_request_spans:
53
- break
54
- time.sleep(0.5)
52
+ # Wait for the "llm_request" span to be exported.
53
+ # The BatchSpanProcessor batches spans and exports them periodically,
54
+ # so we need to wait specifically for the llm_request span to appear.
55
+ timeout = 15
56
+ deadline = time.time() + timeout
57
+ llm_request_spans = []
58
+ while time.time() < deadline:
59
+ all_spans = trace_service.get_all_spans()
60
+ llm_request_spans = [s for s in all_spans if s["name"] == "llm_request"]
61
+ if llm_request_spans:
62
+ break
63
+ time.sleep(0.5)
64
56
- assert len(llm_request_spans) == 1, (
57
- f"Expected exactly 1 'llm_request' span, but got {len(llm_request_spans)}. "
58
- f"All span names: {[s['name'] for s in all_spans]}"
59
- )
65
+ assert len(llm_request_spans) == 1, (
66
+ f"Expected exactly 1 'llm_request' span, but got "
67
+ f"{len(llm_request_spans)}. "
68
+ f"All span names: {[s['name'] for s in all_spans]}"
69
+ )
70
61
- attributes = llm_request_spans[0]["attributes"]
62
- # assert attributes.get(SpanAttributes.GEN_AI_RESPONSE_MODEL) == model
63
- assert attributes.get(SpanAttributes.GEN_AI_REQUEST_ID) == outputs[0].request_id
64
- assert (
65
- attributes.get(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE)
66
- == sampling_params.temperature
67
- )
68
- assert (
69
- attributes.get(SpanAttributes.GEN_AI_REQUEST_TOP_P) == sampling_params.top_p
70
- )
71
- assert (
72
- attributes.get(SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS)
73
- == sampling_params.max_tokens
74
- )
75
- assert attributes.get(SpanAttributes.GEN_AI_REQUEST_N) == sampling_params.n
76
- assert attributes.get(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS) == len(
77
- outputs[0].prompt_token_ids
78
- )
79
- completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
80
- assert (
81
- attributes.get(SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS)
82
- == completion_tokens
83
- )
71
+ attributes = llm_request_spans[0]["attributes"]
72
+ # assert attributes.get(SpanAttributes.GEN_AI_RESPONSE_MODEL) == model
73
+ assert (
74
+ attributes.get(SpanAttributes.GEN_AI_REQUEST_ID)
75
+ == outputs[0].request_id
76
+ )
77
+ assert (
78
+ attributes.get(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE)
79
+ == sampling_params.temperature
80
+ )
81
+ assert (
82
+ attributes.get(SpanAttributes.GEN_AI_REQUEST_TOP_P)
83
+ == sampling_params.top_p
84
+ )
85
+ assert (
86
+ attributes.get(SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS)
87
+ == sampling_params.max_tokens
88
+ )
89
+ assert attributes.get(SpanAttributes.GEN_AI_REQUEST_N) == sampling_params.n
90
+ assert attributes.get(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS) == len(
91
+ outputs[0].prompt_token_ids
92
+ )
93
+ completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
94
+ assert (
95
+ attributes.get(SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS)
96
+ == completion_tokens
97
+ )
98
85
- assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE) > 0
86
- assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN) > 0
87
- assert attributes.get(SpanAttributes.GEN_AI_LATENCY_E2E) > 0
99
+ assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE) > 0
100
+ assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN) > 0
101
+ assert attributes.get(SpanAttributes.GEN_AI_LATENCY_E2E) > 0
102
+ finally:
103
+ if llm is not None:
104
+ shutdown_timeout = 60.0 if current_platform.is_rocm() else 5.0
105
+ llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
106
+ cleanup_dist_env_and_memory()