[Hardware][AMD][CI] Toggle test coredumps on ROCm debug agent (#47222)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Matt committed
Jun 30, 2026 at 23:30 UTC
3c1396bab62e138c980a4939a0c04f35aabd1463
2 files changed
+16
-1
.buildkite/scripts/hardware_ci/run-amd-test.sh
+15
@@ -534,6 +534,20 @@ else
534
echo "--- Single-node job"
535
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
536
537
+ ulimit_core_hard=$(ulimit -H -c)
538
+ if [[ "$ulimit_core_hard" == "unlimited" ]]; then
539
+ # docker run can't pass "unlimited" to --ulimit
540
+ ulimit_core_hard="-1"
541
+ fi
542
+ # Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled
543
+ coredump_flags="--ulimit core=0:$ulimit_core_hard"
544
+ if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then
545
+ # Works around https://github.com/rocm/rocm-systems/issues/6206
546
+ coredump_flags='-e HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"'
547
+ else
548
+ echo "ROCm debug agent not enabled, coredumps are disabled in the test container."
549
+ fi
550
+
551
docker run \
552
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
553
$RDMA_FLAGS \
@@ -541,6 +555,7 @@ else
555
--shm-size=16gb \
556
--group-add "$render_gid" \
557
--rm \
558
+ $coredump_flags \
559
-e HF_TOKEN \
560
-e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
561
-e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
tests/distributed/test_weight_transfer.py
+1
-1
@@ -62,7 +62,7 @@ def _get_ray_assigned_device() -> torch.device:
62
63
def _set_ray_assigned_device() -> torch.device:
64
device = _get_ray_assigned_device()
65
- torch.accelerator.set_device(device)
65
+ torch.accelerator.set_device_index(device)
66
return device
67
68