[CI] Drop root privileges for the OTel GPU sampler (#56941)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Kevin H. Luu committed
Sep 14, 2026 at 21:20 UTC
a491b6039d8d23314efc30a2113000ea987e409e
2 files changed
+129
-3
.buildkite/scripts/ci-otel/ci_otel.sh
+50
-3
@@ -65,6 +65,39 @@ _ci_otel_python() {
65
fi
66
}
67
68
+# The Docker test commands run as root, while /workdir is owned by the
69
+# Buildkite checkout user. Run the GPU sampler as that unprivileged identity so
70
+# any future project-code import cannot leave root-owned files in the checkout.
71
+_ci_otel_prepare_gpu_user() {
72
+ _CI_INFRA_GPU_UID=""
73
+ _CI_INFRA_GPU_GID=""
74
+ [ "$(id -u 2>/dev/null || :)" = "0" ] || return 0
75
+ command -v setpriv >/dev/null 2>&1 || return 1
76
+
77
+ _CI_INFRA_OTEL_WORKSPACE_DIR="${CI_INFRA_OTEL_WORKSPACE_DIR:-/workdir}"
78
+ [ -d "${_CI_INFRA_OTEL_WORKSPACE_DIR}" ] || return 1
79
+ # The owner fields are numeric and whitespace-free on GNU stat.
80
+ # shellcheck disable=SC2046
81
+ set -- $(stat -c '%u %g' -- "${_CI_INFRA_OTEL_WORKSPACE_DIR}" 2>/dev/null || :)
82
+ [ "$#" -eq 2 ] || return 1
83
+ case "$1:$2" in
84
+ *[!0-9:]* | 0:* | *:0) return 1 ;;
85
+ esac
86
+
87
+ _CI_INFRA_GPU_UID="$1"
88
+ _CI_INFRA_GPU_GID="$2"
89
+ chown "${_CI_INFRA_GPU_UID}:${_CI_INFRA_GPU_GID}" \
90
+ "${CI_INFRA_OTEL_SPOOL_DIR}" || return 1
91
+ chmod 0700 "${CI_INFRA_OTEL_SPOOL_DIR}" || return 1
92
+ if [ "${_CI_INFRA_OTEL_OWNS_RUNTIME}" = "1" ]; then
93
+ # Permit traversal to the private child spool, but not directory listing.
94
+ chmod 0711 "${CI_INFRA_OTEL_RUNTIME_DIR}" || return 1
95
+ fi
96
+ setpriv --reuid="${_CI_INFRA_GPU_UID}" \
97
+ --regid="${_CI_INFRA_GPU_GID}" --clear-groups --no-new-privs \
98
+ -- test -w "${CI_INFRA_OTEL_SPOOL_DIR}"
99
+}
100
+
101
ci_otel_start() {
102
_CI_INFRA_OTEL_COMMAND_INDEX="$1"
103
_CI_INFRA_OTEL_COMMAND_LABEL="${2:-command ${_CI_INFRA_OTEL_COMMAND_INDEX}}"
@@ -87,9 +120,23 @@ ci_otel_start() {
120
if [ "${CI_INFRA_GPU_SAMPLING:-1}" != "0" ] &&
121
[ -f "${CI_INFRA_OTEL_DIR}/ci_gpu.py" ] &&
122
command -v nvidia-smi >/dev/null 2>&1; then
90
- PYTHONDONTWRITEBYTECODE=1 \
91
- "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" </dev/null &
92
- _CI_INFRA_GPU_PID=$!
123
+ if _ci_otel_prepare_gpu_user; then
124
+ if [ -n "${_CI_INFRA_GPU_UID}" ]; then
125
+ PYTHONDONTWRITEBYTECODE=1 setpriv \
126
+ --reuid="${_CI_INFRA_GPU_UID}" --regid="${_CI_INFRA_GPU_GID}" \
127
+ --clear-groups --no-new-privs -- \
128
+ "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" \
129
+ </dev/null &
130
+ else
131
+ PYTHONDONTWRITEBYTECODE=1 \
132
+ "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" \
133
+ </dev/null &
134
+ fi
135
+ _CI_INFRA_GPU_PID=$!
136
+ else
137
+ echo "vLLM CI OTel: non-root GPU sampler unavailable; sampling disabled" \
138
+ >&2 || :
139
+ fi
140
fi
141
}
142
.buildkite/scripts/ci-otel/tests/test_ci_otel.py
+79
@@ -987,6 +987,85 @@ def test_gpu_periodic_batches_are_bounded_even_when_upload_fails(monkeypatch):
987
assert [len(batch[0].events) for batch in batches] == [1, 1, 1]
988
989
990
+def _write_executable(path: Path, body: str) -> None:
991
+ path.write_text(f"#!/bin/sh\n{body}")
992
+ path.chmod(0o755)
993
+
994
+
995
+@pytest.mark.parametrize(
996
+ "workspace_owner,expect_sampler",
997
+ [("997 996", True), ("0 0", False)],
998
+)
999
+def test_root_shell_runs_gpu_sampler_as_checkout_owner(
1000
+ tmp_path, workspace_owner, expect_sampler
1001
+):
1002
+ bin_dir = tmp_path / "bin"
1003
+ workspace = tmp_path / "workdir"
1004
+ runtime = tmp_path / "runtime"
1005
+ log = tmp_path / "calls.log"
1006
+ bin_dir.mkdir()
1007
+ workspace.mkdir()
1008
+
1009
+ _write_executable(
1010
+ bin_dir / "id",
1011
+ '[ "$1" = "-u" ] && { echo 0; exit 0; }; exec /usr/bin/id "$@"\n',
1012
+ )
1013
+ _write_executable(
1014
+ bin_dir / "stat",
1015
+ f"printf '%s\\n' '{workspace_owner}'\n",
1016
+ )
1017
+ for command in ("chown", "chmod"):
1018
+ _write_executable(bin_dir / command, "exit 0\n")
1019
+ _write_executable(
1020
+ bin_dir / "setpriv",
1021
+ 'printf "setpriv %s\\n" "$*" >> "$CI_OTEL_TEST_LOG"\n'
1022
+ 'while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do shift; done\n'
1023
+ '[ "$#" -gt 0 ] && shift\n'
1024
+ 'exec "$@"\n',
1025
+ )
1026
+ _write_executable(
1027
+ bin_dir / "python3",
1028
+ '[ "$1" = "-c" ] && exit 0\n'
1029
+ '[ "$2" = "new-context" ] && {\n'
1030
+ ' echo "01010101010101010101010101010101 0202020202020202 - 1"\n'
1031
+ " exit 0\n"
1032
+ "}\n"
1033
+ 'case "$1" in *ci_gpu.py) echo gpu >> "$CI_OTEL_TEST_LOG";; esac\n'
1034
+ "exit 0\n",
1035
+ )
1036
+ _write_executable(bin_dir / "nvidia-smi", "exit 0\n")
1037
+ shell = (
1038
+ f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '
1039
+ f"ci_otel_start 1 {_quoted('true')}; ci_otel_finish 0"
1040
+ )
1041
+
1042
+ result = subprocess.run(
1043
+ ["/bin/sh", "-c", shell],
1044
+ check=False,
1045
+ capture_output=True,
1046
+ text=True,
1047
+ env={
1048
+ **os.environ,
1049
+ "PATH": f"{bin_dir}:{os.environ['PATH']}",
1050
+ "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),
1051
+ "CI_INFRA_OTEL_RUNTIME_DIR": str(runtime),
1052
+ "CI_INFRA_OTEL_WORKSPACE_DIR": str(workspace),
1053
+ "CI_OTEL_TEST_LOG": str(log),
1054
+ },
1055
+ )
1056
+
1057
+ assert result.returncode == 0, result.stderr
1058
+ calls = log.read_text() if log.exists() else ""
1059
+ if expect_sampler:
1060
+ expected = "--reuid=997 --regid=996 --clear-groups --no-new-privs --"
1061
+ assert calls.count("setpriv") == 2
1062
+ assert calls.count(expected) == 2
1063
+ assert "gpu" in calls
1064
+ else:
1065
+ assert calls == ""
1066
+ assert "non-root GPU sampler unavailable" in result.stderr
1067
+
1068
+
1069
@pytest.mark.parametrize("sampling", ["1", "0"])
1070
def test_gpu_sampler_stops_with_command_and_preserves_failure_status(
1071
tmp_path, sampling