| 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # Re-generate container recipes |
| 4 | # |
| 5 | # This script uses the "lcitool" available from |
| 6 | # |
| 7 | # https://gitlab.com/libvirt/libvirt-ci |
| 8 | # |
| 9 | # Copyright (c) 2020 Red Hat Inc. |
| 10 | # |
| 11 | # This work is licensed under the terms of the GNU GPL, version 2 |
| 12 | # or (at your option) any later version. See the COPYING file in |
| 13 | # the top-level directory. |
| 14 | |
| 15 | import sys |
| 16 | import subprocess |
| 17 | |
| 18 | from pathlib import Path |
| 19 | |
| 20 | if len(sys.argv) != 1: |
| 21 | print("syntax: %s" % sys.argv[0], file=sys.stderr) |
| 22 | sys.exit(1) |
| 23 | |
| 24 | self_dir = Path(__file__).parent |
| 25 | src_dir = self_dir.parent.parent |
| 26 | dockerfiles_dir = Path(src_dir, "tests", "docker", "dockerfiles") |
| 27 | |
| 28 | lcitool_path = Path(self_dir, "libvirt-ci", "bin", "lcitool") |
| 29 | |
| 30 | lcitool_cmd = [lcitool_path, "--data-dir", self_dir] |
| 31 | |
| 32 | |
| 33 | def atomic_write(filename, content): |
| 34 | tmp = filename.with_suffix(filename.suffix + ".tmp") |
| 35 | try: |
| 36 | with tmp.open("w") as fp: |
| 37 | print(content, file=fp, end="") |
| 38 | tmp.rename(filename) |
| 39 | except Exception as ex: |
| 40 | tmp.unlink() |
| 41 | raise |
| 42 | |
| 43 | |
| 44 | def generate(filename, cmd, trailer): |
| 45 | print("Generate %s" % filename) |
| 46 | lcitool = subprocess.run(cmd, capture_output=True, encoding='utf8') |
| 47 | |
| 48 | if lcitool.returncode != 0: |
| 49 | raise Exception("Failed to generate %s: %s" % (filename, lcitool.stderr)) |
| 50 | |
| 51 | content = lcitool.stdout |
| 52 | if trailer is not None: |
| 53 | content += trailer |
| 54 | atomic_write(filename, content) |
| 55 | |
| 56 | # Optional user setting, this will always be the last thing added |
| 57 | # so maximise the number of layers that are cached |
| 58 | add_user_mapping = [ |
| 59 | "# As a final step configure the user (if env is defined)", |
| 60 | "ARG USER", |
| 61 | "ARG UID", |
| 62 | "RUN if [ \"${USER}\" ]; then \\", |
| 63 | " id ${USER} 2>/dev/null || useradd -u ${UID} -U ${USER}; fi\n" |
| 64 | ] |
| 65 | |
| 66 | def generate_dockerfile(host, target, project="qemu", cross=None, trailer=None, |
| 67 | enable_rust=True): |
| 68 | filename = Path(src_dir, "tests", "docker", "dockerfiles", host + ".docker") |
| 69 | cmd = lcitool_cmd + ["dockerfile"] |
| 70 | if cross is not None: |
| 71 | cmd.extend(["--cross", cross]) |
| 72 | cmd.extend([target, project]) |
| 73 | |
| 74 | if trailer is not None: |
| 75 | trailer = trailer.strip() + "\n" |
| 76 | trailer += "\n".join(add_user_mapping) |
| 77 | else: |
| 78 | trailer = "\n".join(add_user_mapping) |
| 79 | |
| 80 | if enable_rust: |
| 81 | trailer += "\nENV ENABLE_RUST=1\n" |
| 82 | generate(filename, cmd, trailer) |
| 83 | |
| 84 | |
| 85 | def generate_vars(target, trailer=None): |
| 86 | filename = Path(src_dir, ".gitlab-ci.d", target + ".vars") |
| 87 | cmd = lcitool_cmd + ["variables", "--format", "shell", target, "qemu"] |
| 88 | generate(filename, cmd, trailer) |
| 89 | |
| 90 | |
| 91 | def generate_pkglist(vm, target, project="qemu"): |
| 92 | filename = Path(src_dir, "tests", "vm", "generated", vm + ".json") |
| 93 | cmd = lcitool_cmd + ["variables", "--format", "json", target, project] |
| 94 | generate(filename, cmd, None) |
| 95 | |
| 96 | |
| 97 | def generate_yaml(os, target, arch, trailer=None): |
| 98 | filename = Path(src_dir, "scripts", "ci", "setup", os, f"{target}-{arch}.yaml") |
| 99 | cmd = lcitool_cmd + ["variables", "--format", "yaml", "-a", |
| 100 | arch, target, "qemu"] |
| 101 | generate(filename, cmd, trailer) |
| 102 | |
| 103 | |
| 104 | alpine_extras = r""" |
| 105 | # https://gitlab.alpinelinux.org/alpine/aports/-/issues/17463 |
| 106 | RUN apk add clang19-libclang |
| 107 | """ |
| 108 | |
| 109 | # Netmap still needs to be manually built as it is yet to be packaged |
| 110 | # into a distro. We also add cscope and gtags which are used in the CI |
| 111 | # test |
| 112 | debian13_extras = r""" |
| 113 | # netmap/cscope/global |
| 114 | RUN apt update && \ |
| 115 | DEBIAN_FRONTEND=noninteractive eatmydata \ |
| 116 | apt install -y --no-install-recommends \ |
| 117 | cscope\ |
| 118 | global\ |
| 119 | linux-headers-generic |
| 120 | RUN git clone https://github.com/luigirizzo/netmap.git /usr/src/netmap |
| 121 | RUN cd /usr/src/netmap && git checkout v11.3 |
| 122 | RUN cd /usr/src/netmap/LINUX && \ |
| 123 | ./configure --no-drivers --no-apps \ |
| 124 | --kernel-dir=$(ls -d /usr/src/linux-headers-*-$(dpkg --print-architecture)) \ |
| 125 | && make install |
| 126 | ENV QEMU_CONFIGURE_OPTS=--enable-netmap |
| 127 | """ |
| 128 | |
| 129 | # Based on the hub.docker.com/library/rust Dockerfiles |
| 130 | fedora_rustup_nightly_extras = r""" |
| 131 | RUN dnf install -y wget |
| 132 | ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo |
| 133 | ENV RUSTC=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc |
| 134 | ENV RUSTDOC=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustdoc |
| 135 | ENV CARGO=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/cargo |
| 136 | RUN set -eux && \ |
| 137 | rustArch='x86_64-unknown-linux-gnu' && \ |
| 138 | rustupSha256='6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d' && \ |
| 139 | url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" && \ |
| 140 | wget "$url" && \ |
| 141 | echo "${rustupSha256} *rustup-init" | sha256sum -c - && \ |
| 142 | chmod +x rustup-init && \ |
| 143 | ./rustup-init -y --no-modify-path --profile default --default-toolchain nightly --default-host ${rustArch} && \ |
| 144 | chmod -R a+w $RUSTUP_HOME $CARGO_HOME && \ |
| 145 | /usr/local/cargo/bin/rustup --version && \ |
| 146 | /usr/local/cargo/bin/rustup run nightly cargo --version && \ |
| 147 | /usr/local/cargo/bin/rustup run nightly rustc --version && \ |
| 148 | test "$CARGO" = "$(/usr/local/cargo/bin/rustup +nightly which cargo)" && \ |
| 149 | test "$RUSTDOC" = "$(/usr/local/cargo/bin/rustup +nightly which rustdoc)" && \ |
| 150 | test "$RUSTC" = "$(/usr/local/cargo/bin/rustup +nightly which rustc)" |
| 151 | ENV PATH=$CARGO_HOME/bin:$PATH |
| 152 | RUN /usr/local/cargo/bin/rustup run nightly cargo install --locked bindgen-cli |
| 153 | RUN $CARGO --list |
| 154 | """ |
| 155 | |
| 156 | ubuntu2404_rust_extras = r""" |
| 157 | ENV RUSTC=/usr/bin/rustc-1.83 |
| 158 | ENV RUSTDOC=/usr/bin/rustdoc-1.83 |
| 159 | ENV CARGO_HOME=/usr/local/cargo |
| 160 | ENV PATH=$CARGO_HOME/bin:$PATH |
| 161 | RUN apt update && \ |
| 162 | DEBIAN_FRONTEND=noninteractive eatmydata \ |
| 163 | apt install -y --no-install-recommends cargo |
| 164 | RUN cargo install --locked bindgen-cli |
| 165 | """ |
| 166 | |
| 167 | debian_all_test_cross_compilers = r""" |
| 168 | # extras for cross and alternate toolchains |
| 169 | RUN apt update && \ |
| 170 | DEBIAN_FRONTEND=noninteractive eatmydata \ |
| 171 | apt install -y --no-install-recommends \ |
| 172 | clang\ |
| 173 | dpkg-dev\ |
| 174 | gdb-multiarch\ |
| 175 | libclang-rt-dev |
| 176 | ENV AVAILABLE_COMPILERS="\ |
| 177 | gcc-aarch64-linux-gnu \ |
| 178 | libc6-dev-arm64-cross \ |
| 179 | gcc-arm-linux-gnueabihf \ |
| 180 | libc6-dev-armhf-cross \ |
| 181 | gcc-alpha-linux-gnu \ |
| 182 | libc6.1-dev-alpha-cross \ |
| 183 | gcc-mips-linux-gnu \ |
| 184 | libc6-dev-mips-cross \ |
| 185 | gcc-mips64-linux-gnuabi64 \ |
| 186 | libc6-dev-mips64-cross \ |
| 187 | gcc-mips64el-linux-gnuabi64 \ |
| 188 | libc6-dev-mips64el-cross \ |
| 189 | gcc-mipsel-linux-gnu \ |
| 190 | libc6-dev-mipsel-cross \ |
| 191 | gcc-powerpc64le-linux-gnu \ |
| 192 | libc6-dev-ppc64el-cross \ |
| 193 | gcc-riscv64-linux-gnu \ |
| 194 | libc6-dev-riscv64-cross \ |
| 195 | gcc-s390x-linux-gnu \ |
| 196 | libc6-dev-s390x-cross\ |
| 197 | gcc-sh4-linux-gnu \ |
| 198 | libc6-dev-sh4-cross \ |
| 199 | " |
| 200 | RUN if dpkg-architecture -e amd64; then \ |
| 201 | export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-hppa-linux-gnu libc6-dev-hppa-cross"; \ |
| 202 | export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-m68k-linux-gnu libc6-dev-m68k-cross"; \ |
| 203 | export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-powerpc-linux-gnu libc6-dev-powerpc-cross"; \ |
| 204 | export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross"; \ |
| 205 | export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-sparc64-linux-gnu libc6-dev-sparc64-cross"; \ |
| 206 | fi && \ |
| 207 | apt update && \ |
| 208 | DEBIAN_FRONTEND=noninteractive eatmydata \ |
| 209 | apt install -y --no-install-recommends \ |
| 210 | ${AVAILABLE_COMPILERS} && \ |
| 211 | dpkg-query --showformat '${Package}_${Version}_${Architecture}' --show > /packages.txt |
| 212 | ENV QEMU_CONFIGURE_OPTS=--disable-docs |
| 213 | ENV DEF_TARGET_LIST=aarch64-linux-user,alpha-linux-user,arm-linux-user,hppa-linux-user,i386-linux-user,m68k-linux-user,mips-linux-user,mips64-linux-user,mips64el-linux-user,mipsel-linux-user,ppc-linux-user,ppc64-linux-user,ppc64le-linux-user,riscv64-linux-user,s390x-linux-user,sh4-linux-user,sparc64-linux-user |
| 214 | """ |
| 215 | |
| 216 | def cross_build(prefix, targets): |
| 217 | conf = "ENV QEMU_CONFIGURE_OPTS=--cross-prefix=%s\n" % (prefix) |
| 218 | targets = "ENV DEF_TARGET_LIST=%s\n" % (targets) |
| 219 | return "".join([conf, targets]) |
| 220 | |
| 221 | # |
| 222 | # Update all the various build configurations. |
| 223 | # Please keep each group sorted alphabetically for easy reading. |
| 224 | # |
| 225 | |
| 226 | try: |
| 227 | # |
| 228 | # Standard native builds |
| 229 | # |
| 230 | generate_dockerfile("alpine", "alpine-323", |
| 231 | trailer=alpine_extras) |
| 232 | generate_dockerfile("centos9", "centos-stream-9") |
| 233 | generate_dockerfile("debian", "debian-13", |
| 234 | trailer=debian13_extras) |
| 235 | generate_dockerfile("fedora", "fedora-43") |
| 236 | generate_dockerfile("opensuse-leap", "opensuse-leap-16") |
| 237 | generate_dockerfile("ubuntu2404", "ubuntu-2404", |
| 238 | trailer=ubuntu2404_rust_extras) |
| 239 | |
| 240 | # |
| 241 | # Non-fatal Rust-enabled build |
| 242 | # |
| 243 | generate_dockerfile("fedora-rust-nightly", "fedora-43", |
| 244 | trailer=fedora_rustup_nightly_extras) |
| 245 | |
| 246 | # |
| 247 | # Cross compiling builds |
| 248 | # |
| 249 | generate_dockerfile("debian-amd64-cross", "debian-13", |
| 250 | cross="x86_64", |
| 251 | trailer=cross_build("x86_64-linux-gnu-", |
| 252 | "x86_64-softmmu," |
| 253 | "x86_64-linux-user," |
| 254 | "i386-softmmu,i386-linux-user")) |
| 255 | |
| 256 | generate_dockerfile("debian-arm64-cross", "debian-13", |
| 257 | cross="aarch64", |
| 258 | trailer=cross_build("aarch64-linux-gnu-", |
| 259 | "aarch64-softmmu,aarch64-linux-user")) |
| 260 | |
| 261 | generate_dockerfile("debian-armhf-cross", "debian-13", |
| 262 | cross="armv7l", |
| 263 | trailer=cross_build("arm-linux-gnueabihf-", |
| 264 | "arm-softmmu,arm-linux-user")) |
| 265 | |
| 266 | generate_dockerfile("debian-i686-cross", "debian-13", |
| 267 | cross="i686", |
| 268 | trailer=cross_build("i686-linux-gnu-", |
| 269 | "i386-softmmu,i386-linux-user")) |
| 270 | |
| 271 | generate_dockerfile("debian-ppc64el-cross", "debian-13", |
| 272 | cross="ppc64le", |
| 273 | trailer=cross_build("powerpc64le-linux-gnu-", |
| 274 | "ppc64-softmmu,ppc64-linux-user")) |
| 275 | |
| 276 | # while not yet a release architecture the packages are still |
| 277 | # build while part of testing |
| 278 | generate_dockerfile("debian-riscv64-cross", "debian-13", |
| 279 | cross="riscv64", |
| 280 | trailer=cross_build("riscv64-linux-gnu-", |
| 281 | "riscv64-softmmu,riscv64-linux-user")) |
| 282 | |
| 283 | generate_dockerfile("debian-s390x-cross", "debian-13", |
| 284 | cross="s390x", |
| 285 | trailer=cross_build("s390x-linux-gnu-", |
| 286 | "s390x-softmmu,s390x-linux-user")) |
| 287 | |
| 288 | generate_dockerfile("fedora-win64-cross", "fedora-43", |
| 289 | project='qemu,qemu-win-installer', |
| 290 | cross="mingw64", |
| 291 | trailer=cross_build("x86_64-w64-mingw32-", |
| 292 | "x86_64-softmmu")) |
| 293 | |
| 294 | # We also generate some docker files with minimal dependencies and |
| 295 | # as many cross-compilers as Debian will package for building TCG |
| 296 | # tests. |
| 297 | |
| 298 | generate_dockerfile("debian-all-test-cross", "debian-13", |
| 299 | project="qemu-minimal", |
| 300 | enable_rust=False, |
| 301 | trailer=debian_all_test_cross_compilers) |
| 302 | |
| 303 | # |
| 304 | # GitLab packages lists |
| 305 | # |
| 306 | generate_vars("macos-14") |
| 307 | |
| 308 | # |
| 309 | # VM packages lists |
| 310 | # |
| 311 | generate_pkglist("freebsd", "freebsd-14") |
| 312 | generate_pkglist("openbsd", "openbsd-78", project="qemu-minimal") |
| 313 | |
| 314 | # |
| 315 | # Ansible package lists |
| 316 | # |
| 317 | generate_yaml("ubuntu", "ubuntu-2404", "aarch64") |
| 318 | generate_yaml("ubuntu", "ubuntu-2404", "s390x") |
| 319 | generate_yaml("debian", "debian-13", "ppc64le") |
| 320 | |
| 321 | |
| 322 | sys.exit(0) |
| 323 | except Exception as ex: |
| 324 | print(str(ex), file=sys.stderr) |
| 325 | sys.exit(1) |