[CI][ROCm] Prevent Model Executor apt stalls (#58607)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Codex <noreply@openai.com>
Andreas Karatzas committed
Sep 24, 2026 at 23:18 UTC
fa94450b743766d466021a40e5b5f5f5746c92b9
5 files changed
+32
-4
.buildkite/test-amd.yaml
-1
@@ -2014,7 +2014,6 @@ steps:
2014
- tests/model_executor/test_jit_warmup_cutedsl_launcher.py
2015
- tests/model_executor/test_jit_warmup_triton_launcher.py
2016
commands:
2017
- - apt-get update && apt-get install -y curl libsodium23
2017
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
2018
- export PYTHONFAULTHANDLER=1
2019
- pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
.buildkite/test_areas/model_executor.yaml
+5
@@ -48,3 +48,8 @@ steps:
48
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
49
- vllm/_aiter_ops.py
50
- vllm/platforms/rocm.py
51
+ commands:
52
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
53
+ - export PYTHONFAULTHANDLER=1
54
+ - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
55
+ - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
docker/Dockerfile.rock
+9
-1
@@ -776,13 +776,21 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
776
uv pip install --system /nixl_install/*.whl /deep_install/*.whl
777
778
# RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec.
779
-RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
779
+RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \
780
+ -o Acquire::Retries=3 \
781
+ -o Acquire::http::Timeout=30 \
782
+ -o Acquire::https::Timeout=30 \
783
+ && apt-get install -q -y --no-install-recommends \
784
+ -o Acquire::Retries=3 \
785
+ -o Acquire::http::Timeout=30 \
786
+ -o Acquire::https::Timeout=30 \
787
librdmacm1 \
788
libibverbs1 \
789
ibverbs-providers \
790
ibverbs-utils \
791
util-linux \
792
unzip \
793
+ libsodium23 \
794
pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \
795
libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \
796
&& rm -rf /var/lib/apt/lists/*
docker/Dockerfile.rocm
+9
-1
@@ -797,13 +797,21 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
797
COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem
798
799
# RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec.
800
-RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
800
+RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \
801
+ -o Acquire::Retries=3 \
802
+ -o Acquire::http::Timeout=30 \
803
+ -o Acquire::https::Timeout=30 \
804
+ && apt-get install -q -y --no-install-recommends \
805
+ -o Acquire::Retries=3 \
806
+ -o Acquire::http::Timeout=30 \
807
+ -o Acquire::https::Timeout=30 \
808
librdmacm1 \
809
libibverbs1 \
810
ibverbs-providers \
811
ibverbs-utils \
812
util-linux \
813
unzip \
814
+ libsodium23 \
815
pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \
816
libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \
817
&& rm -rf /var/lib/apt/lists/*
docker/Dockerfile.rocm_gfx1250
+9
-1
@@ -560,11 +560,19 @@ RUN --mount=type=bind,from=build_rixl,src=/app/install,target=/rixl_install \
560
COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem
561
562
# RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec.
563
-RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
563
+RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \
564
+ -o Acquire::Retries=3 \
565
+ -o Acquire::http::Timeout=30 \
566
+ -o Acquire::https::Timeout=30 \
567
+ && apt-get install -q -y --no-install-recommends \
568
+ -o Acquire::Retries=3 \
569
+ -o Acquire::http::Timeout=30 \
570
+ -o Acquire::https::Timeout=30 \
571
librdmacm1 \
572
libibverbs1 \
573
ibverbs-providers \
574
ibverbs-utils \
575
+ libsodium23 \
576
pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \
577
libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \
578
&& rm -rf /var/lib/apt/lists/*