@cryptotaxi247 / netdata-1 / commits / c42093ff4

Use dagger to build and test the agent. (#16868)

Dag

vkalintiris committed Jan 29, 2024 at 18:39 UTC c42093ff4f062fbbe8bc9f948f8e09c7510ae973
12 files changed +2254
packaging/dag/README.md new
+23
@@ -0,0 +1,23 @@
1 +- Install Dagger CLI:
2 + ```
3 + cd /usr/local
4 + curl -L https://dl.dagger.io/dagger/install.sh | sudo sh
5 + ```
6 +- Install Python's Dagger SDK:
7 + ```
8 + pip install dagger-io
9 + ```
10 +
11 +Now you can run something like this:
12 +
13 +```
14 +dagger run python packaging/dag/main.py build -p linux/x86_64 -d debian12
15 +```
16 +
17 +or
18 +
19 +```
20 +dagger run python packaging/dag/main.py test
21 +```.
22 +
23 +
packaging/dag/build_command.py new
+65
@@ -0,0 +1,65 @@
1 +import click
2 +import asyncio
3 +import sys
4 +import dagger
5 +import pathlib
6 +import uuid
7 +
8 +from nd import (
9 + Distribution,
10 + NetdataInstaller,
11 + FeatureFlags,
12 + Endpoint,
13 + AgentContext,
14 + SUPPORTED_PLATFORMS,
15 + SUPPORTED_DISTRIBUTIONS,
16 +)
17 +
18 +
19 +def run_async(func):
20 + def wrapper(*args, **kwargs):
21 + return asyncio.run(func(*args, **kwargs))
22 +
23 + return wrapper
24 +
25 +
26 +@run_async
27 +async def simple_build(platform, distro):
28 + config = dagger.Config(log_output=sys.stdout)
29 +
30 + async with dagger.Connection(config) as client:
31 + repo_root = pathlib.Path("/netdata")
32 + prefix_path = pathlib.Path("/opt/netdata")
33 +
34 + installer = NetdataInstaller(
35 + platform, distro, repo_root, prefix_path, FeatureFlags.DBEngine
36 + )
37 +
38 + endpoint = Endpoint("node", 19999)
39 + api_key = uuid.uuid4()
40 + allow_children = False
41 +
42 + agent_ctx = AgentContext(
43 + client, platform, distro, installer, endpoint, api_key, allow_children
44 + )
45 +
46 + await agent_ctx.build_container()
47 +
48 +
49 +@click.command()
50 +@click.option(
51 + "--platform",
52 + "-p",
53 + type=click.Choice(sorted([str(p) for p in SUPPORTED_PLATFORMS])),
54 + help="Specify the platform.",
55 +)
56 +@click.option(
57 + "--distribution",
58 + "-d",
59 + type=click.Choice(sorted([str(p) for p in SUPPORTED_DISTRIBUTIONS])),
60 + help="Specify the distribution.",
61 +)
62 +def build(platform, distribution):
63 + platform = dagger.Platform(platform)
64 + distro = Distribution(distribution)
65 + simple_build(platform, distro)
packaging/dag/files/child_stream.conf new
+10
@@ -0,0 +1,10 @@
1 +[stream]
2 + enabled = {{ enabled }}
3 + destination = {{ destination }}
4 + api key = {{ api_key }}
5 + timeout seconds = {{ timeout_seconds }}
6 + default port = {{ default_port }}
7 + send charts matching = {{ send_charts_matching }}
8 + buffer size bytes = {{ buffer_size_bytes }}
9 + reconnect delay seconds = {{ reconnect_delay_seconds }}
10 + initial clock resync iterations = {{ initial_clock_resync_iterations }}
packaging/dag/files/cmake-aarch64.sha256 new
+1
@@ -0,0 +1 @@
1 +a83e01ed1cdf44c2e33e0726513b9a35a8c09e3b5a126fd720b3c8a9d5552368 cmake-aarch64.sh
packaging/dag/files/cmake-x86_64.sha256 new
+1
@@ -0,0 +1 @@
1 +8c449dabb2b2563ec4e6d5e0fb0ae09e729680efab71527b59015131cea4a042 cmake-x86_64.sh
packaging/dag/files/ol8-epel.repo new
+6
@@ -0,0 +1,6 @@
1 +[ol8_developer_EPEL]
2 +name=Oracle Linux $releasever EPEL Packages for Development ($basearch)
3 +baseurl=https://yum$ociregion.$ocidomain/repo/OracleLinux/OL8/developer/EPEL/$basearch/
4 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle
5 +gpgcheck=1
6 +enabled=1
packaging/dag/files/ol9-epel.repo new
+6
@@ -0,0 +1,6 @@
1 +[ol9_developer_EPEL]
2 +name=Oracle Linux $releasever EPEL Packages for Development ($basearch)
3 +baseurl=https://yum$ociregion.$ocidomain/repo/OracleLinux/OL9/developer/EPEL/$basearch/
4 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle
5 +gpgcheck=1
6 +enabled=1
packaging/dag/files/parent_stream.conf new
+7
@@ -0,0 +1,7 @@
1 +[{{ api_key }}]
2 + enabled = {{ enabled }}
3 + allow from = {{ allow_from }}
4 + default history = {{ default_history }}
5 + health enabled by default = {{ health_enabled_by_default }}
6 + default postpone alarms on connect seconds = {{ default_postpone_alarms_on_connect_seconds }}
7 + multiple connections = {{ multiple_connections }}
packaging/dag/imageutils.py new
+1580
@@ -0,0 +1,1580 @@
1 +import os
2 +from pathlib import Path
3 +
4 +import dagger
5 +
6 +
7 +_ALPINE_COMMON_PACKAGES = [
8 + "alpine-sdk",
9 + "autoconf",
10 + "automake",
11 + "bash",
12 + "binutils",
13 + "bison",
14 + "cmake",
15 + "curl",
16 + "curl-static",
17 + "elfutils-dev",
18 + "flex",
19 + "gcc",
20 + "git",
21 + "gnutls-dev",
22 + "gzip",
23 + "jq",
24 + "libelf-static",
25 + "libmnl-dev",
26 + "libmnl-static",
27 + "libtool",
28 + "libuv-dev",
29 + "libuv-static",
30 + "lz4-dev",
31 + "lz4-static",
32 + "make",
33 + "mongo-c-driver-dev",
34 + "mongo-c-driver-static",
35 + "musl-fts-dev",
36 + "ncurses",
37 + "ncurses-static",
38 + "netcat-openbsd",
39 + "ninja",
40 + "openssh",
41 + "pcre2-dev",
42 + "pkgconfig",
43 + "protobuf-dev",
44 + "snappy-dev",
45 + "snappy-static",
46 + "util-linux-dev",
47 + "wget",
48 + "xz",
49 + "yaml-dev",
50 + "yaml-static",
51 + "zlib-dev",
52 + "zlib-static",
53 + "zstd-dev",
54 + "zstd-static",
55 +]
56 +
57 +
58 +def build_alpine_3_18(
59 + client: dagger.Client, platform: dagger.Platform
60 +) -> dagger.Container:
61 + ctr = client.container(platform=platform).from_("alpine:3.18")
62 +
63 + pkgs = [pkg for pkg in _ALPINE_COMMON_PACKAGES]
64 +
65 + ctr = ctr.with_exec(["apk", "add", "--no-cache"] + pkgs)
66 +
67 + return ctr
68 +
69 +
70 +def build_alpine_3_19(
71 + client: dagger.Client, platform: dagger.Platform
72 +) -> dagger.Container:
73 + ctr = client.container(platform=platform).from_("alpine:3.19")
74 +
75 + pkgs = [pkg for pkg in _ALPINE_COMMON_PACKAGES]
76 +
77 + ctr = ctr.with_exec(["apk", "add", "--no-cache"] + pkgs)
78 +
79 + return ctr
80 +
81 +
82 +def static_build_openssl(
83 + client: dagger.Client, ctr: dagger.Container
84 +) -> dagger.Container:
85 + tree = (
86 + client.git(url="https://github.com/openssl/openssl", keep_git_dir=True)
87 + .tag("openssl-3.1.4")
88 + .tree()
89 + )
90 +
91 + #
92 + # TODO: verify 32-bit builds
93 + #
94 + ctr = (
95 + ctr.with_directory("/openssl", tree)
96 + .with_workdir("/openssl")
97 + .with_env_variable("CFLAGS", "-fno-lto -pipe")
98 + .with_env_variable("LDFLAGS", "-static")
99 + .with_env_variable("PKG_CONFIG", "pkg-config --static")
100 + .with_exec(
101 + [
102 + "sed",
103 + "-i",
104 + "s/disable('static', 'pic', 'threads');/disable('static', 'pic');/",
105 + "Configure",
106 + ]
107 + )
108 + .with_exec(
109 + [
110 + "./config",
111 + "-static",
112 + "threads",
113 + "no-tests",
114 + "--prefix=/openssl-static",
115 + "--openssldir=/opt/netdata/etc/ssl",
116 + ]
117 + )
118 + .with_exec(["make", "V=1", "-j", str(os.cpu_count())])
119 + .with_exec(["make", "V=1", "-j", str(os.cpu_count()), "install_sw"])
120 + .with_exec(["ln", "-s", "/openssl-static/lib", "/openssl-static/lib64"])
121 + .with_exec(["perl", "configdata.pm", "--dump"])
122 + )
123 +
124 + return ctr
125 +
126 +
127 +def static_build_bash(client: dagger.Client, ctr: dagger.Container) -> dagger.Container:
128 + tree = (
129 + client.git(url="https://git.savannah.gnu.org/git/bash.git", keep_git_dir=True)
130 + .tag("bash-5.1")
131 + .tree()
132 + )
133 +
134 + ctr = (
135 + ctr.with_directory("/bash", tree)
136 + .with_workdir("/bash")
137 + .with_env_variable("CFLAGS", "-pipe")
138 + .with_env_variable("LDFLAGS", "")
139 + .with_env_variable("PKG_CONFIG", "pkg-config --static")
140 + .with_env_variable("PKG_CONFIG_PATH", "/openssl-static/lib64/pkgconfig")
141 + .with_exec(
142 + [
143 + "./configure",
144 + "--prefix",
145 + "/opt/netdata",
146 + "--without-bash-malloc",
147 + "--enable-static-link",
148 + "--enable-net-redirections",
149 + "--enable-array-variables",
150 + "--disable-progcomp",
151 + "--disable-profiling",
152 + "--disable-nls",
153 + "--disable-dependency-tracking",
154 + ]
155 + )
156 + .with_exec(
157 + [
158 + "echo",
159 + "-e",
160 + "all:\nclean:\ninstall:\n",
161 + ">",
162 + "examples/loadables/Makefile",
163 + ]
164 + )
165 + .with_exec(["make", "clean"])
166 + # see: https://gitweb.gentoo.org/repo/gentoo.git/tree/app-shells/bash/files/bash-5.1-parallel_make.patch?id=4c2ebbf4b8bc660beb98cc2d845c73375d6e4f50
167 + .with_exec(["make", "V=1", "-j", "2", "install"])
168 + .with_exec(["strip", "/opt/netdata/bin/bash"])
169 + )
170 +
171 + return ctr
172 +
173 +
174 +def static_build_curl(client: dagger.Client, ctr: dagger.Container) -> dagger.Container:
175 + tree = (
176 + client.git(url="https://github.com/curl/curl", keep_git_dir=True)
177 + .tag("curl-8_4_0")
178 + .tree()
179 + )
180 +
181 + ctr = (
182 + ctr.with_directory("/curl", tree)
183 + .with_workdir("/curl")
184 + .with_env_variable("CFLAGS", "-I/openssl-static/include -pipe")
185 + .with_env_variable("LDFLAGS", "-static -L/openssl-static/lib64")
186 + .with_env_variable("PKG_CONFIG", "pkg-config --static")
187 + .with_env_variable("PKG_CONFIG_PATH", "/openssl-static/lib64/pkgconfig")
188 + .with_exec(["autoreconf", "-ifv"])
189 + .with_exec(
190 + [
191 + "./configure",
192 + "--prefix=/curl-static",
193 + "--enable-optimize",
194 + "--disable-shared",
195 + "--enable-static",
196 + "--enable-http",
197 + "--disable-ldap",
198 + "--disable-ldaps",
199 + "--enable-proxy",
200 + "--disable-dict",
201 + "--disable-telnet",
202 + "--disable-tftp",
203 + "--disable-pop3",
204 + "--disable-imap",
205 + "--disable-smb",
206 + "--disable-smtp",
207 + "--disable-gopher",
208 + "--enable-ipv6",
209 + "--enable-cookies",
210 + "--with-ca-fallback",
211 + "--with-openssl",
212 + "--disable-dependency-tracking",
213 + ]
214 + )
215 + .with_exec(
216 + ["sed", "-i", "-e", "s/LDFLAGS =/LDFLAGS = -all-static/", "src/Makefile"]
217 + )
218 + .with_exec(["make", "clean"])
219 + .with_exec(["make", "V=1", "-j", str(os.cpu_count()), "install"])
220 + .with_exec(["cp", "/curl-static/bin/curl", "/opt/netdata/bin/curl"])
221 + .with_exec(["strip", "/opt/netdata/bin/curl"])
222 + )
223 +
224 + return ctr
225 +
226 +
227 +def static_build_ioping(
228 + client: dagger.Client, ctr: dagger.Container
229 +) -> dagger.Container:
230 + tree = (
231 + client.git(url="https://github.com/koct9i/ioping", keep_git_dir=True)
232 + .tag("v1.3")
233 + .tree()
234 + )
235 +
236 + ctr = (
237 + ctr.with_directory("/ioping", tree)
238 + .with_workdir("/ioping")
239 + .with_env_variable("CFLAGS", "-static -pipe")
240 + .with_exec(["mkdir", "-p", "/opt/netdata/usr/libexec/netdata/plugins.d"])
241 + .with_exec(["make", "V=1"])
242 + .with_exec(
243 + [
244 + "install",
245 + "-o",
246 + "root",
247 + "-g",
248 + "root",
249 + "-m",
250 + "4750",
251 + "ioping",
252 + "/opt/netdata/usr/libexec/netdata/plugins.d",
253 + ]
254 + )
255 + .with_exec(["strip", "/opt/netdata/usr/libexec/netdata/plugins.d/ioping"])
256 + )
257 +
258 + return ctr
259 +
260 +
261 +def static_build_libnetfilter_acct(
262 + client: dagger.Client, ctr: dagger.Container
263 +) -> dagger.Container:
264 + tree = (
265 + client.git(url="git://git.netfilter.org/libnetfilter_acct", keep_git_dir=True)
266 + .tag("libnetfilter_acct-1.0.3")
267 + .tree()
268 + )
269 +
270 + ctr = (
271 + ctr.with_directory("/libnetfilter_acct", tree)
272 + .with_workdir("/libnetfilter_acct")
273 + .with_env_variable("CFLAGS", "-static -I/usr/include/libmnl -pipe")
274 + .with_env_variable("LDFLAGS", "-static -L/usr/lib -lmnl")
275 + .with_env_variable("PKG_CONFIG", "pkg-config --static")
276 + .with_env_variable("PKG_CONFIG_PATH", "/usr/lib/pkgconfig")
277 + .with_exec(["autoreconf", "-ifv"])
278 + .with_exec(
279 + [
280 + "./configure",
281 + "--prefix=/libnetfilter_acct-static",
282 + "--exec-prefix=/libnetfilter_acct-static",
283 + ]
284 + )
285 + .with_exec(["make", "clean"])
286 + .with_exec(["make", "V=1", "-j", str(os.cpu_count()), "install"])
287 + )
288 +
289 + return ctr
290 +
291 +
292 +def static_build_netdata(
293 + client: dagger.Client, ctr: dagger.Container
294 +) -> dagger.Container:
295 + CFLAGS = [
296 + "-ffunction-sections",
297 + "-fdata-sections",
298 + "-static",
299 + "-O2",
300 + "-funroll-loops",
301 + "-I/openssl-static/include",
302 + "-I/libnetfilter_acct-static/include/libnetfilter_acct",
303 + "-I/curl-local/include/curl",
304 + "-I/usr/include/libmnl",
305 + "-pipe",
306 + ]
307 +
308 + LDFLAGS = [
309 + "-Wl,--gc-sections",
310 + "-static",
311 + "-L/openssl-static/lib64",
312 + "-L/libnetfilter_acct-static/lib",
313 + "-lnetfilter_acct",
314 + "-L/usr/lib",
315 + "-lmnl",
316 + "-L/usr/lib",
317 + "-lzstd",
318 + "-L/curl-local/lib",
319 + ]
320 +
321 + PKG_CONFIG = [
322 + "pkg-config",
323 + "--static",
324 + ]
325 +
326 + PKG_CONFIG_PATH = [
327 + "/openssl-static/lib64/pkgconfig",
328 + "/libnetfilter_acct-static/lib/pkgconfig",
329 + "/usr/lib/pkgconfig",
330 + "/curl-local/lib/pkgconfig",
331 + ]
332 +
333 + CMAKE_FLAGS = [
334 + "-DOPENSSL_ROOT_DIR=/openssl-static",
335 + "-DOPENSSL_LIBRARIES=/openssl-static/lib64",
336 + "-DCMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE=/openssl-static",
337 + "-DLWS_OPENSSL_INCLUDE_DIRS=/openssl-static/include",
338 + "-DLWS_OPENSSL_LIBRARIES=/openssl-static/lib64/libssl.a;/openssl-static/lib64/libcrypto.a",
339 + ]
340 +
341 + NETDATA_INSTALLER_CMD = [
342 + "./netdata-installer.sh",
343 + "--install-prefix",
344 + "/opt",
345 + "--dont-wait",
346 + "--dont-start-it",
347 + "--disable-exporting-mongodb",
348 + "--require-cloud",
349 + "--use-system-protobuf",
350 + "--dont-scrub-cflags-even-though-it-may-break-things",
351 + "--one-time-build",
352 + "--enable-lto",
353 + ]
354 +
355 + ctr = (
356 + ctr.with_workdir("/netdata")
357 + .with_env_variable("NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug")
358 + .with_env_variable("CFLAGS", " ".join(CFLAGS))
359 + .with_env_variable("LDFLAGS", " ".join(LDFLAGS))
360 + .with_env_variable("PKG_CONFIG", " ".join(PKG_CONFIG))
361 + .with_env_variable("PKG_CONFIG_PATH", ":".join(PKG_CONFIG_PATH))
362 + .with_env_variable("CMAKE_FLAGS", " ".join(CMAKE_FLAGS))
363 + .with_env_variable("EBPF_LIBC", "static")
364 + .with_env_variable("IS_NETDATA_STATIC_BINARY", "yes")
365 + .with_exec(NETDATA_INSTALLER_CMD)
366 + )
367 +
368 + return ctr
369 +
370 +
371 +def static_build(client, repo_path):
372 + cmake_build_release_path = os.path.join(repo_path, "cmake-build-release")
373 +
374 + ctr = build_alpine_3_18(client, dagger.Platform("linux/x86_64"))
375 + ctr = static_build_openssl(client, ctr)
376 + ctr = static_build_bash(client, ctr)
377 + ctr = static_build_curl(client, ctr)
378 + ctr = static_build_ioping(client, ctr)
379 + ctr = static_build_libnetfilter_acct(client, ctr)
380 +
381 + ctr = ctr.with_directory(
382 + "/netdata",
383 + client.host().directory(repo_path),
384 + exclude=[
385 + f"{cmake_build_release_path}/*",
386 + "fluent-bit/build",
387 + ],
388 + )
389 +
390 + # TODO: link bin/sbin
391 +
392 + ctr = static_build_netdata(client, ctr)
393 +
394 + build_dir = ctr.directory("/opt/netdata")
395 + artifact_dir = os.path.join(Path.home(), "ci/netdata-static")
396 + output_task = build_dir.export(artifact_dir)
397 + return output_task
398 +
399 +
400 +_CENTOS_COMMON_PACKAGES = [
401 + "autoconf",
402 + "autoconf-archive",
403 + "autogen",
404 + "automake",
405 + "bison",
406 + "bison-devel",
407 + "cmake",
408 + "cups-devel",
409 + "curl",
410 + "diffutils",
411 + "elfutils-libelf-devel",
412 + "findutils",
413 + "flex",
414 + "flex-devel",
415 + "freeipmi-devel",
416 + "gcc",
417 + "gcc-c++",
418 + "git-core",
419 + "golang",
420 + "json-c-devel",
421 + "libyaml-devel",
422 + "libatomic",
423 + "libcurl-devel",
424 + "libmnl-devel",
425 + "libnetfilter_acct-devel",
426 + "libtool",
427 + "libuuid-devel",
428 + "libuv-devel",
429 + "libzstd-devel",
430 + "lm_sensors",
431 + "lz4-devel",
432 + "make",
433 + "ninja-build",
434 + "openssl-devel",
435 + "openssl-perl",
436 + "patch",
437 + "pcre2-devel",
438 + "pkgconfig",
439 + "pkgconfig(libmongoc-1.0)",
440 + "procps",
441 + "protobuf-c-devel",
442 + "protobuf-compiler",
443 + "protobuf-devel",
444 + "rpm-build",
445 + "rpm-devel",
446 + "rpmdevtools",
447 + "snappy-devel",
448 + "systemd-devel",
449 + "wget",
450 + "zlib-devel",
451 +]
452 +
453 +
454 +def build_amazon_linux_2(
455 + client: dagger.Client, platform: dagger.Platform
456 +) -> dagger.Container:
457 + ctr = client.container(platform=platform).from_("amazonlinux:2")
458 +
459 + pkgs = [pkg for pkg in _CENTOS_COMMON_PACKAGES]
460 +
461 + ctr = (
462 + ctr.with_exec(["yum", "update", "-y"])
463 + .with_exec(["yum", "install", "-y"] + pkgs)
464 + .with_exec(["yum", "clean", "all"])
465 + .with_exec(["c_rehash"])
466 + .with_exec(
467 + [
468 + "mkdir",
469 + "-p",
470 + "/root/rpmbuild/BUILD",
471 + "/root/rpmbuild/RPMS",
472 + "/root/rpmbuild/SOURCES",
473 + "/root/rpmbuild/SPECS",
474 + "/root/rpmbuild/SRPMS",
475 + ]
476 + )
477 + )
478 +
479 + if platform == "linux/x86_64":
480 + machine = "x86_64"
481 + elif platform == "linux/arm64":
482 + machine = "aarch64"
483 + else:
484 + raise Exception(
485 + "Amaxon Linux 2 supports only linux/amd64 and linux/arm64 platforms."
486 + )
487 +
488 +
489 + checksum_path = Path(__file__).parent / f"files/cmake-{machine}.sha256"
490 +
491 + ctr = (
492 + ctr.with_file(
493 + f"cmake-{machine}.sha256",
494 + client.host().file(checksum_path.as_posix()),
495 + )
496 + .with_exec(
497 + [
498 + "curl",
499 + "--fail",
500 + "-sSL",
501 + "--connect-timeout",
502 + "20",
503 + "--retry",
504 + "3",
505 + "--output",
506 + f"cmake-{machine}.sh",
507 + f"https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-{machine}.sh",
508 + ]
509 + )
510 + .with_exec(["sha256sum", "-c", f"cmake-{machine}.sha256"])
511 + .with_exec(["chmod", "u+x", f"./cmake-{machine}.sh"])
512 + .with_exec([f"./cmake-{machine}.sh", "--skip-license", "--prefix=/usr/local"])
513 + )
514 +
515 + return ctr
516 +
517 +
518 +def build_centos_7(
519 + client: dagger.Client, platform: dagger.Platform
520 +) -> dagger.Container:
521 + ctr = client.container(platform=platform).from_("centos:7")
522 +
523 + pkgs = [pkg for pkg in _CENTOS_COMMON_PACKAGES] + ["bash"]
524 +
525 + ctr = (
526 + ctr.with_exec(["yum", "install", "-y", "epel-release"])
527 + .with_exec(["yum", "update", "-y"])
528 + .with_exec(["yum", "install", "-y"] + pkgs)
529 + .with_exec(["yum", "clean", "all"])
530 + .with_exec(["c_rehash"])
531 + .with_exec(
532 + [
533 + "mkdir",
534 + "-p",
535 + "/root/rpmbuild/BUILD",
536 + "/root/rpmbuild/RPMS",
537 + "/root/rpmbuild/SOURCES",
538 + "/root/rpmbuild/SPECS",
539 + "/root/rpmbuild/SRPMS",
540 + ]
541 + )
542 + )
543 +
544 + if platform == "linux/x86_64":
545 + machine = "x86_64"
546 + elif platform == "linux/arm64":
547 + machine = "aarch64"
548 + else:
549 + raise Exception("CentOS 7 supports only linux/amd64 and linux/arm64 platforms.")
550 +
551 + checksum_path = Path(__file__).parent / f"files/cmake-{machine}.sha256"
552 +
553 + ctr = (
554 + ctr.with_file(
555 + f"cmake-{machine}.sha256",
556 + client.host().file(checksum_path.as_posix()),
557 + )
558 + .with_exec(
559 + [
560 + "curl",
561 + "--fail",
562 + "-sSL",
563 + "--connect-timeout",
564 + "20",
565 + "--retry",
566 + "3",
567 + "--output",
568 + f"cmake-{machine}.sh",
569 + f"https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-{machine}.sh",
570 + ]
571 + )
572 + .with_exec(["sha256sum", "-c", f"cmake-{machine}.sha256"])
573 + .with_exec(["chmod", "u+x", f"./cmake-{machine}.sh"])
574 + .with_exec([f"./cmake-{machine}.sh", "--skip-license", "--prefix=/usr/local"])
575 + )
576 +
577 + return ctr
578 +
579 +
580 +_ROCKY_LINUX_COMMON_PACKAGES = [
581 + "autoconf",
582 + "autoconf-archive",
583 + "automake",
584 + "bash",
585 + "bison",
586 + "cmake",
587 + "cups-devel",
588 + "curl",
589 + "libcurl-devel",
590 + "diffutils",
591 + "elfutils-libelf-devel",
592 + "findutils",
593 + "flex",
594 + "freeipmi-devel",
595 + "gcc",
596 + "gcc-c++",
597 + "git",
598 + "golang",
599 + "json-c-devel",
600 + "libatomic",
601 + "libmnl-devel",
602 + "libtool",
603 + "libuuid-devel",
604 + "libuv-devel",
605 + "libyaml-devel",
606 + "libzstd-devel",
607 + "lm_sensors",
608 + "lz4-devel",
609 + "make",
610 + "ninja-build",
611 + "nc",
612 + "openssl-devel",
613 + "openssl-perl",
614 + "patch",
615 + "pcre2-devel",
616 + "pkgconfig",
617 + "pkgconfig(libmongoc-1.0)",
618 + "procps",
619 + "protobuf-c-devel",
620 + "protobuf-compiler",
621 + "protobuf-devel",
622 + "python3",
623 + "python3-pyyaml",
624 + "rpm-build",
625 + "rpm-devel",
626 + "rpmdevtools",
627 + "snappy-devel",
628 + "systemd-devel",
629 + "wget",
630 + "zlib-devel",
631 +]
632 +
633 +
634 +def build_rocky_linux_8(
635 + client: dagger.Client, platform: dagger.Platform
636 +) -> dagger.Container:
637 + ctr = client.container(platform=platform).from_("rockylinux:8")
638 +
639 + pkgs = [pkg for pkg in _ROCKY_LINUX_COMMON_PACKAGES] + ["autogen"]
640 +
641 + ctr = (
642 + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"])
643 + .with_exec(
644 + [
645 + "dnf",
646 + "install",
647 + "-y",
648 + "--nodocs",
649 + "dnf-command(config-manager)",
650 + "epel-release",
651 + ]
652 + )
653 + .with_exec(["dnf", "config-manager", "--set-enabled", "powertools"])
654 + .with_exec(["dnf", "clean", "packages"])
655 + .with_exec(
656 + [
657 + "dnf",
658 + "install",
659 + "-y",
660 + "--nodocs",
661 + "--setopt=install_weak_deps=False",
662 + "--setopt=diskspacecheck=False",
663 + ]
664 + + pkgs
665 + )
666 + .with_exec(["rm", "-rf", "/var/cache/dnf"])
667 + .with_exec(["c_rehash"])
668 + .with_exec(
669 + [
670 + "mkdir",
671 + "-p",
672 + "/root/rpmbuild/BUILD",
673 + "/root/rpmbuild/RPMS",
674 + "/root/rpmbuild/SOURCES",
675 + "/root/rpmbuild/SPECS",
676 + "/root/rpmbuild/SRPMS",
677 + ]
678 + )
679 + )
680 +
681 + return ctr
682 +
683 +
684 +def build_rocky_linux_9(
685 + client: dagger.Client, platform: dagger.Platform
686 +) -> dagger.Container:
687 + ctr = client.container(platform=platform).from_("rockylinux:9")
688 +
689 + pkgs = [pkg for pkg in _ROCKY_LINUX_COMMON_PACKAGES]
690 +
691 + ctr = (
692 + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"])
693 + .with_exec(
694 + [
695 + "dnf",
696 + "install",
697 + "-y",
698 + "--nodocs",
699 + "dnf-command(config-manager)",
700 + "epel-release",
701 + ]
702 + )
703 + .with_exec(["dnf", "config-manager", "--set-enabled", "crb"])
704 + .with_exec(["dnf", "clean", "packages"])
705 + .with_exec(
706 + [
707 + "dnf",
708 + "install",
709 + "-y",
710 + "--allowerasing",
711 + "--nodocs",
712 + "--setopt=install_weak_deps=False",
713 + "--setopt=diskspacecheck=False",
714 + ]
715 + + pkgs
716 + )
717 + .with_exec(["rm", "-rf", "/var/cache/dnf"])
718 + .with_exec(["c_rehash"])
719 + .with_exec(
720 + [
721 + "mkdir",
722 + "-p",
723 + "/root/rpmbuild/BUILD",
724 + "/root/rpmbuild/RPMS",
725 + "/root/rpmbuild/SOURCES",
726 + "/root/rpmbuild/SPECS",
727 + "/root/rpmbuild/SRPMS",
728 + ]
729 + )
730 + )
731 +
732 + return ctr
733 +
734 +
735 +_CENTOS_STREAM_COMMON_PACKAGES = [
736 + "autoconf",
737 + "autoconf-archive",
738 + "automake",
739 + "bash",
740 + "bison",
741 + "cmake",
742 + "cups-devel",
743 + "curl",
744 + "libcurl-devel",
745 + "libyaml-devel",
746 + "diffutils",
747 + "elfutils-libelf-devel",
748 + "findutils",
749 + "flex",
750 + "freeipmi-devel",
751 + "gcc",
752 + "gcc-c++",
753 + "git",
754 + "golang",
755 + "json-c-devel",
756 + "libatomic",
757 + "libmnl-devel",
758 + "libtool",
759 + "libuuid-devel",
760 + "libuv-devel",
761 + # "libzstd-devel",
762 + "lm_sensors",
763 + "lz4-devel",
764 + "make",
765 + "ninja-build",
766 + "nc",
767 + "openssl-devel",
768 + "openssl-perl",
769 + "patch",
770 + "pcre2-devel",
771 + "pkgconfig",
772 + "pkgconfig(libmongoc-1.0)",
773 + "procps",
774 + "protobuf-c-devel",
775 + "protobuf-compiler",
776 + "protobuf-devel",
777 + "python3",
778 + "python3-pyyaml",
779 + "rpm-build",
780 + "rpm-devel",
781 + "rpmdevtools",
782 + "snappy-devel",
783 + "systemd-devel",
784 + "wget",
785 + "zlib-devel",
786 +]
787 +
788 +
789 +def build_centos_stream_8(
790 + client: dagger.Client, platform: dagger.Platform
791 +) -> dagger.Container:
792 + ctr = client.container(platform=platform).from_("quay.io/centos/centos:stream8")
793 +
794 + pkgs = [pkg for pkg in _CENTOS_STREAM_COMMON_PACKAGES] + ["autogen"]
795 +
796 + ctr = (
797 + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"])
798 + .with_exec(
799 + [
800 + "dnf",
801 + "install",
802 + "-y",
803 + "--nodocs",
804 + "dnf-command(config-manager)",
805 + "epel-release",
806 + ]
807 + )
808 + .with_exec(["dnf", "config-manager", "--set-enabled", "powertools"])
809 + .with_exec(["dnf", "clean", "packages"])
810 + .with_exec(
811 + [
812 + "dnf",
813 + "install",
814 + "-y",
815 + "--nodocs",
816 + "--setopt=install_weak_deps=False",
817 + "--setopt=diskspacecheck=False",
818 + ]
819 + + pkgs
820 + )
821 + .with_exec(["rm", "-rf", "/var/cache/dnf"])
822 + .with_exec(["c_rehash"])
823 + .with_exec(
824 + [
825 + "mkdir",
826 + "-p",
827 + "/root/rpmbuild/BUILD",
828 + "/root/rpmbuild/RPMS",
829 + "/root/rpmbuild/SOURCES",
830 + "/root/rpmbuild/SPECS",
831 + "/root/rpmbuild/SRPMS",
832 + ]
833 + )
834 + )
835 +
836 + return ctr
837 +
838 +
839 +def build_centos_stream_9(
840 + client: dagger.Client, platform: dagger.Platform
841 +) -> dagger.Container:
842 + ctr = client.container(platform=platform).from_("quay.io/centos/centos:stream9")
843 +
844 + pkgs = [pkg for pkg in _CENTOS_STREAM_COMMON_PACKAGES]
845 +
846 + ctr = (
847 + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"])
848 + .with_exec(
849 + [
850 + "dnf",
851 + "install",
852 + "-y",
853 + "--nodocs",
854 + "dnf-command(config-manager)",
855 + "epel-release",
856 + ]
857 + )
858 + .with_exec(["dnf", "config-manager", "--set-enabled", "crb"])
859 + .with_exec(["dnf", "clean", "packages"])
860 + .with_exec(
861 + [
862 + "dnf",
863 + "install",
864 + "-y",
865 + "--allowerasing",
866 + "--nodocs",
867 + "--setopt=install_weak_deps=False",
868 + "--setopt=diskspacecheck=False",
869 + ]
870 + + pkgs
871 + )
872 + .with_exec(["rm", "-rf", "/var/cache/dnf"])
873 + .with_exec(["c_rehash"])
874 + .with_exec(
875 + [
876 + "mkdir",
877 + "-p",
878 + "/root/rpmbuild/BUILD",
879 + "/root/rpmbuild/RPMS",
880 + "/root/rpmbuild/SOURCES",
881 + "/root/rpmbuild/SPECS",
882 + "/root/rpmbuild/SRPMS",
883 + ]
884 + )
885 + )
886 +
887 + return ctr
888 +
889 +
890 +_ORACLE_LINUX_COMMON_PACKAGES = list(_ROCKY_LINUX_COMMON_PACKAGES)
891 +
892 +
893 +def build_oracle_linux_9(
894 + client: dagger.Client, platform: dagger.Platform
895 +) -> dagger.Container:
896 + ctr = client.container(platform=platform).from_("oraclelinux:9")
897 +
898 + pkgs = [pkg for pkg in _ORACLE_LINUX_COMMON_PACKAGES]
899 +
900 + repo_path = str(Path(__file__).parent.parent.parent)
901 + this_path = os.path.join(repo_path, "packaging/dag")
902 +
903 + ctr = (
904 + ctr.with_file(
905 + "/etc/yum.repos.d/ol9-epel.repo",
906 + client.host().file(f"{this_path}/ol9-epel.repo"),
907 + )
908 + .with_exec(["dnf", "config-manager", "--set-enabled", "ol9_codeready_builder"])
909 + .with_exec(["dnf", "config-manager", "--set-enabled", "ol9_developer_EPEL"])
910 + .with_exec(["dnf", "distro-sync", "-y", "--nodocs"])
911 + .with_exec(["dnf", "clean", "-y", "packages"])
912 + .with_exec(
913 + [
914 + "dnf",
915 + "install",
916 + "-y",
917 + "--nodocs",
918 + "--setopt=install_weak_deps=False",
919 + "--setopt=diskspacecheck=False",
920 + ]
921 + + pkgs
922 + )
923 + .with_exec(["rm", "-rf", "/var/cache/dnf"])
924 + .with_exec(["c_rehash"])
925 + .with_exec(
926 + [
927 + "mkdir",
928 + "-p",
929 + "/root/rpmbuild/BUILD",
930 + "/root/rpmbuild/RPMS",
931 + "/root/rpmbuild/SOURCES",
932 + "/root/rpmbuild/SPECS",
933 + "/root/rpmbuild/SRPMS",
934 + ]
935 + )
936 + )
937 +
938 + return ctr
939 +
940 +
941 +def build_oracle_linux_8(
942 + client: dagger.Client, platform: dagger.Platform
943 +) -> dagger.Container:
944 + ctr = client.container(platform=platform).from_("oraclelinux:8")
945 +
946 + pkgs = [pkg for pkg in _ORACLE_LINUX_COMMON_PACKAGES] + ["autogen"]
947 +
948 + repo_path = str(Path(__file__).parent.parent.parent)
949 + this_path = os.path.join(repo_path, "packaging/dag")
950 +
951 + ctr = (
952 + ctr.with_file(
953 + "/etc/yum.repos.d/ol8-epel.repo",
954 + client.host().file(f"{this_path}/ol8-epel.repo"),
955 + )
956 + .with_exec(["dnf", "config-manager", "--set-enabled", "ol8_codeready_builder"])
957 + .with_exec(["dnf", "distro-sync", "-y", "--nodocs"])
958 + .with_exec(["dnf", "clean", "-y", "packages"])
959 + .with_exec(
960 + [
961 + "dnf",
962 + "install",
963 + "-y",
964 + "--nodocs",
965 + "--setopt=install_weak_deps=False",
966 + "--setopt=diskspacecheck=False",
967 + ]
968 + + pkgs
969 + )
970 + .with_exec(["rm", "-rf", "/var/cache/dnf"])
971 + .with_exec(["c_rehash"])
972 + .with_exec(
973 + [
974 + "mkdir",
975 + "-p",
976 + "/root/rpmbuild/BUILD",
977 + "/root/rpmbuild/RPMS",
978 + "/root/rpmbuild/SOURCES",
979 + "/root/rpmbuild/SPECS",
980 + "/root/rpmbuild/SRPMS",
981 + ]
982 + )
983 + )
984 +
985 + return ctr
986 +
987 +
988 +_OPENSUSE_COMMON_PACKAGES = [
989 + "autoconf",
990 + "autoconf-archive",
991 + "autogen",
992 + "automake",
993 + "bison",
994 + "cmake",
995 + "cups",
996 + "cups-devel",
997 + "curl",
998 + "diffutils",
999 + "flex",
1000 + "freeipmi-devel",
1001 + "gcc",
1002 + "gcc-c++",
1003 + "git-core",
1004 + "go",
1005 + "json-glib-devel",
1006 + "judy-devel",
1007 + "libatomic1",
1008 + "libcurl-devel",
1009 + "libelf-devel",
1010 + "liblz4-devel",
1011 + "libjson-c-devel",
1012 + "libyaml-devel",
1013 + "libmnl0",
1014 + "libmnl-devel",
1015 + "libnetfilter_acct1",
1016 + "libnetfilter_acct-devel",
1017 + "libpcre2-8-0",
1018 + "libopenssl-devel",
1019 + "libtool",
1020 + "libuv-devel",
1021 + "libuuid-devel",
1022 + "libzstd-devel",
1023 + "make",
1024 + "ninja",
1025 + "patch",
1026 + "pkg-config",
1027 + "protobuf-devel",
1028 + "rpm-build",
1029 + "rpm-devel",
1030 + "rpmdevtools",
1031 + "snappy-devel",
1032 + "systemd-devel",
1033 + "tar",
1034 + "wget",
1035 + "xen-devel",
1036 +]
1037 +
1038 +
1039 +def build_opensuse_tumbleweed(
1040 + client: dagger.Client, platform: dagger.Platform
1041 +) -> dagger.Container:
1042 + ctr = client.container(platform=platform).from_("opensuse/tumbleweed:latest")
1043 +
1044 + pkgs = [pkg for pkg in _OPENSUSE_COMMON_PACKAGES] + ["protobuf-c"]
1045 +
1046 + ctr = (
1047 + ctr.with_exec(["zypper", "update", "-y"])
1048 + .with_exec(
1049 + [
1050 + "zypper",
1051 + "install",
1052 + "-y",
1053 + "--allow-downgrade",
1054 + ]
1055 + + pkgs
1056 + )
1057 + .with_exec(["zypper", "clean"])
1058 + .with_exec(["rm", "-rf", "/var/cache/zypp/*/*"])
1059 + .with_exec(["c_rehash"])
1060 + .with_exec(
1061 + [
1062 + "mkdir",
1063 + "-p",
1064 + "/usr/src/packages/BUILD",
1065 + "/usr/src/packages/RPMS",
1066 + "/usr/src/packages/SOURCES",
1067 + "/usr/src/packages/SPECS",
1068 + "/usr/src/packages/SRPMS",
1069 + ]
1070 + )
1071 + )
1072 +
1073 + return ctr
1074 +
1075 +
1076 +def build_opensuse_15_5(
1077 + client: dagger.Client, platform: dagger.Platform
1078 +) -> dagger.Container:
1079 + ctr = client.container(platform=platform).from_("opensuse/leap:15.5")
1080 +
1081 + pkgs = [pkg for pkg in _OPENSUSE_COMMON_PACKAGES] + ["libprotobuf-c-devel"]
1082 +
1083 + ctr = (
1084 + ctr.with_exec(["zypper", "update", "-y"])
1085 + .with_exec(
1086 + [
1087 + "zypper",
1088 + "install",
1089 + "-y",
1090 + "--allow-downgrade",
1091 + ]
1092 + + pkgs
1093 + )
1094 + .with_exec(["zypper", "clean"])
1095 + .with_exec(["rm", "-rf", "/var/cache/zypp/*/*"])
1096 + .with_exec(["c_rehash"])
1097 + .with_exec(
1098 + [
1099 + "mkdir",
1100 + "-p",
1101 + "/usr/src/packages/BUILD",
1102 + "/usr/src/packages/RPMS",
1103 + "/usr/src/packages/SOURCES",
1104 + "/usr/src/packages/SPECS",
1105 + "/usr/src/packages/SRPMS",
1106 + ]
1107 + )
1108 + )
1109 +
1110 + return ctr
1111 +
1112 +
1113 +def build_opensuse_15_4(
1114 + client: dagger.Client, platform: dagger.Platform
1115 +) -> dagger.Container:
1116 + crt = client.container(platform=platform).from_("opensuse/leap:15.4")
1117 +
1118 + pkgs = [pkg for pkg in _OPENSUSE_COMMON_PACKAGES] + ["libprotobuf-c-devel"]
1119 +
1120 + crt = (
1121 + crt.with_exec(["zypper", "update", "-y"])
1122 + .with_exec(
1123 + [
1124 + "zypper",
1125 + "install",
1126 + "-y",
1127 + "--allow-downgrade",
1128 + ]
1129 + + pkgs
1130 + )
1131 + .with_exec(["zypper", "clean"])
1132 + .with_exec(["rm", "-rf", "/var/cache/zypp/*/*"])
1133 + .with_exec(["c_rehash"])
1134 + .with_exec(
1135 + [
1136 + "mkdir",
1137 + "-p",
1138 + "/usr/src/packages/BUILD",
1139 + "/usr/src/packages/RPMS",
1140 + "/usr/src/packages/SOURCES",
1141 + "/usr/src/packages/SPECS",
1142 + "/usr/src/packages/SRPMS",
1143 + ]
1144 + )
1145 + )
1146 +
1147 + return crt
1148 +
1149 +
1150 +_FEDORA_COMMON_PACKAGES = [
1151 + "autoconf",
1152 + "autoconf-archive",
1153 + "autogen",
1154 + "automake",
1155 + "bash",
1156 + "bison",
1157 + "cmake",
1158 + "cups-devel",
1159 + "curl",
1160 + "diffutils",
1161 + "elfutils-libelf-devel",
1162 + "findutils",
1163 + "flex",
1164 + "freeipmi-devel",
1165 + "gcc",
1166 + "gcc-c++",
1167 + "git-core",
1168 + "golang",
1169 + "json-c-devel",
1170 + "libcurl-devel",
1171 + "libyaml-devel",
1172 + "Judy-devel",
1173 + "libatomic",
1174 + "libmnl-devel",
1175 + "libnetfilter_acct-devel",
1176 + "libtool",
1177 + "libuuid-devel",
1178 + "libuv-devel",
1179 + "libzstd-devel",
1180 + "lz4-devel",
1181 + "make",
1182 + "ninja-build",
1183 + "openssl-devel",
1184 + "openssl-perl",
1185 + "patch",
1186 + "pcre2-devel",
1187 + "pkgconfig",
1188 +]
1189 +
1190 +
1191 +def build_fedora_37(
1192 + client: dagger.Client, platform: dagger.Platform
1193 +) -> dagger.Container:
1194 + ctr = client.container(platform=platform).from_("fedora:37")
1195 +
1196 + pkgs = [pkg for pkg in _FEDORA_COMMON_PACKAGES]
1197 +
1198 + ctr = (
1199 + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"])
1200 + .with_exec(["dnf", "clean", "-y", "packages"])
1201 + .with_exec(
1202 + [
1203 + "dnf",
1204 + "install",
1205 + "-y",
1206 + "--nodocs",
1207 + "--setopt=install_weak_deps=False",
1208 + "--setopt=diskspacecheck=False",
1209 + ]
1210 + + pkgs
1211 + )
1212 + .with_exec(["rm", "-rf", "/var/cache/dnf"])
1213 + .with_exec(["c_rehash"])
1214 + .with_exec(
1215 + [
1216 + "mkdir",
1217 + "-p",
1218 + "/root/rpmbuild/BUILD",
1219 + "/root/rpmbuild/RPMS",
1220 + "/root/rpmbuild/SOURCES",
1221 + "/root/rpmbuild/SPECS",
1222 + "/root/rpmbuild/SRPMS",
1223 + ]
1224 + )
1225 + )
1226 +
1227 + return ctr
1228 +
1229 +
1230 +def build_fedora_38(
1231 + client: dagger.Client, platform: dagger.Platform
1232 +) -> dagger.Container:
1233 + ctr = client.container(platform=platform).from_("fedora:38")
1234 +
1235 + pkgs = [pkg for pkg in _FEDORA_COMMON_PACKAGES]
1236 +
1237 + ctr = (
1238 + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"])
1239 + .with_exec(["dnf", "clean", "-y", "packages"])
1240 + .with_exec(
1241 + [
1242 + "dnf",
1243 + "install",
1244 + "-y",
1245 + "--nodocs",
1246 + "--setopt=install_weak_deps=False",
1247 + "--setopt=diskspacecheck=False",
1248 + ]
1249 + + pkgs
1250 + )
1251 + .with_exec(["rm", "-rf", "/var/cache/dnf"])
1252 + .with_exec(["c_rehash"])
1253 + .with_exec(
1254 + [
1255 + "mkdir",
1256 + "-p",
1257 + "/root/rpmbuild/BUILD",
1258 + "/root/rpmbuild/RPMS",
1259 + "/root/rpmbuild/SOURCES",
1260 + "/root/rpmbuild/SPECS",
1261 + "/root/rpmbuild/SRPMS",
1262 + ]
1263 + )
1264 + )
1265 +
1266 + return ctr
1267 +
1268 +
1269 +def build_fedora_39(
1270 + client: dagger.Client, platform: dagger.Platform
1271 +) -> dagger.Container:
1272 + ctr = client.container(platform=platform).from_("fedora:39")
1273 +
1274 + pkgs = [pkg for pkg in _FEDORA_COMMON_PACKAGES]
1275 +
1276 + ctr = (
1277 + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"])
1278 + .with_exec(["dnf", "clean", "-y", "packages"])
1279 + .with_exec(
1280 + [
1281 + "dnf",
1282 + "install",
1283 + "-y",
1284 + "--nodocs",
1285 + "--setopt=install_weak_deps=False",
1286 + "--setopt=diskspacecheck=False",
1287 + ]
1288 + + pkgs
1289 + )
1290 + .with_exec(["rm", "-rf", "/var/cache/dnf"])
1291 + .with_exec(["c_rehash"])
1292 + .with_exec(
1293 + [
1294 + "mkdir",
1295 + "-p",
1296 + "/root/rpmbuild/BUILD",
1297 + "/root/rpmbuild/RPMS",
1298 + "/root/rpmbuild/SOURCES",
1299 + "/root/rpmbuild/SPECS",
1300 + "/root/rpmbuild/SRPMS",
1301 + ]
1302 + )
1303 + )
1304 +
1305 + return ctr
1306 +
1307 +
1308 +_DEBIAN_COMMON_PACKAGES = [
1309 + "autoconf",
1310 + "autoconf-archive",
1311 + "autogen",
1312 + "automake",
1313 + "bison",
1314 + "build-essential",
1315 + "ca-certificates",
1316 + "cmake",
1317 + "curl",
1318 + "dh-autoreconf",
1319 + "dh-make",
1320 + "dpkg-dev",
1321 + "flex",
1322 + "g++",
1323 + "gcc",
1324 + "git-buildpackage",
1325 + "git-core",
1326 + "golang",
1327 + "libatomic1",
1328 + "libcurl4-openssl-dev",
1329 + "libcups2-dev",
1330 + "libdistro-info-perl",
1331 + "libelf-dev",
1332 + "libipmimonitoring-dev",
1333 + "libjson-c-dev",
1334 + "libyaml-dev",
1335 + "libjudy-dev",
1336 + "liblz4-dev",
1337 + "libmnl-dev",
1338 + "libmongoc-dev",
1339 + "libnetfilter-acct-dev",
1340 + "libpcre2-dev",
1341 + "libprotobuf-dev",
1342 + "libprotoc-dev",
1343 + "libsnappy-dev",
1344 + "libsystemd-dev",
1345 + "libssl-dev",
1346 + "libtool",
1347 + "libuv1-dev",
1348 + "libzstd-dev",
1349 + "make",
1350 + "ninja-build",
1351 + "pkg-config",
1352 + "protobuf-compiler",
1353 + "systemd",
1354 + "uuid-dev",
1355 + "wget",
1356 + "zlib1g-dev",
1357 +]
1358 +
1359 +
1360 +def build_debian_10(
1361 + client: dagger.Client, platform: dagger.Platform
1362 +) -> dagger.Container:
1363 + ctr = client.container(platform=platform).from_("debian:buster")
1364 +
1365 + pkgs = [pkg for pkg in _DEBIAN_COMMON_PACKAGES] + ["dh-systemd", "libxen-dev"]
1366 +
1367 + ctr = (
1368 + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive")
1369 + .with_exec(["apt-get", "update"])
1370 + .with_exec(["apt-get", "upgrade", "-y"])
1371 + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs)
1372 + .with_exec(["apt-get", "clean"])
1373 + .with_exec(["c_rehash"])
1374 + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"])
1375 + )
1376 +
1377 + return ctr
1378 +
1379 +
1380 +def build_debian_11(
1381 + client: dagger.Client, platform: dagger.Platform
1382 +) -> dagger.Container:
1383 + ctr = client.container(platform=platform).from_("debian:bullseye")
1384 +
1385 + pkgs = [pkg for pkg in _DEBIAN_COMMON_PACKAGES] + ["libxen-dev"]
1386 +
1387 + ctr = (
1388 + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive")
1389 + .with_exec(["apt-get", "update"])
1390 + .with_exec(["apt-get", "upgrade", "-y"])
1391 + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs)
1392 + .with_exec(["apt-get", "clean"])
1393 + .with_exec(["c_rehash"])
1394 + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"])
1395 + )
1396 +
1397 + return ctr
1398 +
1399 +
1400 +def build_debian_12(
1401 + client: dagger.Client, platform: dagger.Platform
1402 +) -> dagger.Container:
1403 + ctr = client.container(platform=platform).from_("debian:bookworm")
1404 +
1405 + pkgs = [pkg for pkg in _DEBIAN_COMMON_PACKAGES]
1406 +
1407 + if platform != dagger.Platform("linux/i386"):
1408 + pkgs.append("libxen-dev")
1409 +
1410 + ctr = (
1411 + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive")
1412 + .with_exec(["apt-get", "update"])
1413 + .with_exec(["apt-get", "upgrade", "-y"])
1414 + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs)
1415 + .with_exec(["apt-get", "clean"])
1416 + .with_exec(["c_rehash"])
1417 + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"])
1418 + )
1419 +
1420 + return ctr
1421 +
1422 +
1423 +_UBUNTU_COMMON_PACKAGES = [
1424 + "autoconf",
1425 + "autoconf-archive",
1426 + "autogen",
1427 + "automake",
1428 + "bison",
1429 + "build-essential",
1430 + "ca-certificates",
1431 + "cmake",
1432 + "curl",
1433 + "dh-autoreconf",
1434 + "dh-make",
1435 + "dpkg-dev",
1436 + "flex",
1437 + "g++",
1438 + "gcc",
1439 + "git-buildpackage",
1440 + "git-core",
1441 + "golang",
1442 + "libatomic1",
1443 + "libcurl4-openssl-dev",
1444 + "libcups2-dev",
1445 + "libdistro-info-perl",
1446 + "libelf-dev",
1447 + "libipmimonitoring-dev",
1448 + "libjson-c-dev",
1449 + "libyaml-dev",
1450 + "libjudy-dev",
1451 + "liblz4-dev",
1452 + "libmnl-dev",
1453 + "libmongoc-dev",
1454 + "libnetfilter-acct-dev",
1455 + "libpcre2-dev",
1456 + "libprotobuf-dev",
1457 + "libprotoc-dev",
1458 + "libsnappy-dev",
1459 + "libsystemd-dev",
1460 + "libssl-dev",
1461 + "libtool",
1462 + "libuv1-dev",
1463 + "libxen-dev",
1464 + "libzstd-dev",
1465 + "make",
1466 + "ninja-build",
1467 + "pkg-config",
1468 + "protobuf-compiler",
1469 + "systemd",
1470 + "uuid-dev",
1471 + "wget",
1472 + "zlib1g-dev",
1473 +]
1474 +
1475 +
1476 +def build_ubuntu_20_04(
1477 + client: dagger.Client, platform: dagger.Platform
1478 +) -> dagger.Container:
1479 + ctr = client.container(platform=platform).from_("ubuntu:20.04")
1480 +
1481 + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES] + ["dh-systemd"]
1482 +
1483 + ctr = (
1484 + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive")
1485 + .with_exec(["apt-get", "update"])
1486 + .with_exec(["apt-get", "upgrade", "-y"])
1487 + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs)
1488 + .with_exec(["apt-get", "clean"])
1489 + .with_exec(["c_rehash"])
1490 + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"])
1491 + )
1492 +
1493 + #
1494 + # FIXME: add kitware for cmake on arm-hf
1495 + #
1496 +
1497 + return ctr
1498 +
1499 +
1500 +def build_ubuntu_22_04(
1501 + client: dagger.Client, platform: dagger.Platform
1502 +) -> dagger.Container:
1503 + ctr = client.container(platform=platform).from_("ubuntu:22.04")
1504 +
1505 + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES]
1506 +
1507 + ctr = (
1508 + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive")
1509 + .with_exec(["apt-get", "update"])
1510 + .with_exec(["apt-get", "upgrade", "-y"])
1511 + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs)
1512 + .with_exec(["apt-get", "clean"])
1513 + .with_exec(["c_rehash"])
1514 + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"])
1515 + )
1516 +
1517 + return ctr
1518 +
1519 +
1520 +def build_ubuntu_23_04(
1521 + client: dagger.Client, platform: dagger.Platform
1522 +) -> dagger.Container:
1523 + ctr = client.container(platform=platform).from_("ubuntu:23.04")
1524 +
1525 + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES]
1526 +
1527 + ctr = (
1528 + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive")
1529 + .with_exec(["apt-get", "update"])
1530 + .with_exec(["apt-get", "upgrade", "-y"])
1531 + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs)
1532 + .with_exec(["apt-get", "clean"])
1533 + .with_exec(["c_rehash"])
1534 + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"])
1535 + )
1536 +
1537 + return ctr
1538 +
1539 +
1540 +def build_ubuntu_23_10(
1541 + client: dagger.Client, platform: dagger.Platform
1542 +) -> dagger.Container:
1543 + ctr = client.container(platform=platform).from_("ubuntu:23.10")
1544 +
1545 + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES]
1546 +
1547 + ctr = (
1548 + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive")
1549 + .with_exec(["apt-get", "update"])
1550 + .with_exec(["apt-get", "upgrade", "-y"])
1551 + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs)
1552 + .with_exec(["apt-get", "clean"])
1553 + .with_exec(["c_rehash"])
1554 + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"])
1555 + )
1556 +
1557 + return ctr
1558 +
1559 +
1560 +def install_cargo(ctr: dagger.Container) -> dagger.Container:
1561 + bin_paths = [
1562 + "/root/.cargo/bin",
1563 + "/usr/local/sbin",
1564 + "/usr/local/bin",
1565 + "/usr/sbin",
1566 + "/usr/bin",
1567 + "/sbin",
1568 + "/bin",
1569 + ]
1570 +
1571 + ctr = (
1572 + ctr.with_workdir("/")
1573 + .with_exec(["sh", "-c", "curl https://sh.rustup.rs -sSf | sh -s -- -y"])
1574 + .with_env_variable("PATH", ":".join(bin_paths))
1575 + .with_exec(["cargo", "new", "--bin", "hello"])
1576 + .with_workdir("/hello")
1577 + .with_exec(["cargo", "run", "-v", "-v"])
1578 + )
1579 +
1580 + return ctr
packaging/dag/main.py new
+18
@@ -0,0 +1,18 @@
1 +#!/usr/bin/env python3
2 +
3 +import click
4 +
5 +from test_command import test
6 +from build_command import build
7 +
8 +
9 +@click.group()
10 +def cli():
11 + pass
12 +
13 +
14 +cli.add_command(test)
15 +cli.add_command(build)
16 +
17 +if __name__ == "__main__":
18 + cli()
packaging/dag/nd.py new
+409
@@ -0,0 +1,409 @@
1 +from typing import List
2 +
3 +import enum
4 +import os
5 +import pathlib
6 +import uuid
7 +
8 +import dagger
9 +import jinja2
10 +
11 +import imageutils
12 +
13 +
14 +class Platform:
15 + def __init__(self, platform: str):
16 + self.platform = dagger.Platform(platform)
17 +
18 + def escaped(self) -> str:
19 + return str(self.platform).removeprefix("linux/").replace("/", "_")
20 +
21 + def __eq__(self, other):
22 + if isinstance(other, Platform):
23 + return self.platform == other.platform
24 + elif isinstance(other, dagger.Platform):
25 + return self.platform == other
26 + else:
27 + return NotImplemented
28 +
29 + def __ne__(self, other):
30 + return not (self == other)
31 +
32 + def __hash__(self):
33 + return hash(self.platform)
34 +
35 + def __str__(self) -> str:
36 + return str(self.platform)
37 +
38 +
39 +SUPPORTED_PLATFORMS = set(
40 + [
41 + Platform("linux/x86_64"),
42 + Platform("linux/arm64"),
43 + Platform("linux/i386"),
44 + Platform("linux/arm/v7"),
45 + Platform("linux/arm/v6"),
46 + Platform("linux/ppc64le"),
47 + Platform("linux/s390x"),
48 + Platform("linux/riscv64"),
49 + ]
50 +)
51 +
52 +
53 +SUPPORTED_DISTRIBUTIONS = set(
54 + [
55 + "alpine_3_18",
56 + "alpine_3_19",
57 + "amazonlinux2",
58 + "centos7",
59 + "centos-stream8",
60 + "centos-stream9",
61 + "debian10",
62 + "debian11",
63 + "debian12",
64 + "fedora37",
65 + "fedora38",
66 + "fedora39",
67 + "opensuse15.4",
68 + "opensuse15.5",
69 + "opensusetumbleweed",
70 + "oraclelinux8",
71 + "oraclelinux9",
72 + "rockylinux8",
73 + "rockylinux9",
74 + "ubuntu20.04",
75 + "ubuntu22.04",
76 + "ubuntu23.04",
77 + "ubuntu23.10",
78 + ]
79 +)
80 +
81 +
82 +class Distribution:
83 + def __init__(self, display_name):
84 + self.display_name = display_name
85 +
86 + if self.display_name == "alpine_3_18":
87 + self.docker_tag = "alpine:3.18"
88 + self.builder = imageutils.build_alpine_3_18
89 + self.platforms = SUPPORTED_PLATFORMS
90 + elif self.display_name == "alpine_3_19":
91 + self.docker_tag = "alpine:3.19"
92 + self.builder = imageutils.build_alpine_3_19
93 + self.platforms = SUPPORTED_PLATFORMS
94 + elif self.display_name == "amazonlinux2":
95 + self.docker_tag = "amazonlinux:2"
96 + self.builder = imageutils.build_amazon_linux_2
97 + self.platforms = SUPPORTED_PLATFORMS
98 + elif self.display_name == "centos7":
99 + self.docker_tag = "centos:7"
100 + self.builder = imageutils.build_centos_7
101 + self.platforms = SUPPORTED_PLATFORMS
102 + elif self.display_name == "centos-stream8":
103 + self.docker_tag = "quay.io/centos/centos:stream8"
104 + self.builder = imageutils.build_centos_stream_8
105 + self.platforms = SUPPORTED_PLATFORMS
106 + elif self.display_name == "centos-stream9":
107 + self.docker_tag = "quay.io/centos/centos:stream9"
108 + self.builder = imageutils.build_centos_stream_9
109 + self.platforms = SUPPORTED_PLATFORMS
110 + elif self.display_name == "debian10":
111 + self.docker_tag = "debian:10"
112 + self.builder = imageutils.build_debian_10
113 + self.platforms = SUPPORTED_PLATFORMS
114 + elif self.display_name == "debian11":
115 + self.docker_tag = "debian:11"
116 + self.builder = imageutils.build_debian_11
117 + self.platforms = SUPPORTED_PLATFORMS
118 + elif self.display_name == "debian12":
119 + self.docker_tag = "debian:12"
120 + self.builder = imageutils.build_debian_12
121 + self.platforms = SUPPORTED_PLATFORMS
122 + elif self.display_name == "fedora37":
123 + self.docker_tag = "fedora:37"
124 + self.builder = imageutils.build_fedora_37
125 + self.platforms = SUPPORTED_PLATFORMS
126 + elif self.display_name == "fedora38":
127 + self.docker_tag = "fedora:38"
128 + self.builder = imageutils.build_fedora_38
129 + self.platforms = SUPPORTED_PLATFORMS
130 + elif self.display_name == "fedora39":
131 + self.docker_tag = "fedora:39"
132 + self.platforms = SUPPORTED_PLATFORMS
133 + self.builder = imageutils.build_fedora_39
134 + elif self.display_name == "opensuse15.4":
135 + self.docker_tag = "opensuse/leap:15.4"
136 + self.builder = imageutils.build_opensuse_15_4
137 + self.platforms = SUPPORTED_PLATFORMS
138 + elif self.display_name == "opensuse15.5":
139 + self.docker_tag = "opensuse/leap:15.5"
140 + self.builder = imageutils.build_opensuse_15_5
141 + self.platforms = SUPPORTED_PLATFORMS
142 + elif self.display_name == "opensusetumbleweed":
143 + self.docker_tag = "opensuse/tumbleweed:latest"
144 + self.builder = imageutils.build_opensuse_tumbleweed
145 + self.platforms = SUPPORTED_PLATFORMS
146 + elif self.display_name == "oraclelinux8":
147 + self.docker_tag = "oraclelinux:8"
148 + self.builder = imageutils.build_oracle_linux_8
149 + self.platforms = SUPPORTED_PLATFORMS
150 + elif self.display_name == "oraclelinux9":
151 + self.docker_tag = "oraclelinux:9"
152 + self.builder = imageutils.build_oracle_linux_9
153 + self.platforms = SUPPORTED_PLATFORMS
154 + elif self.display_name == "rockylinux8":
155 + self.docker_tag = "rockylinux:8"
156 + self.builder = imageutils.build_rocky_linux_8
157 + self.platforms = SUPPORTED_PLATFORMS
158 + elif self.display_name == "rockylinux9":
159 + self.docker_tag = "rockylinux:9"
160 + self.builder = imageutils.build_rocky_linux_9
161 + self.platforms = SUPPORTED_PLATFORMS
162 + elif self.display_name == "ubuntu20.04":
163 + self.docker_tag = "ubuntu:20.04"
164 + self.builder = imageutils.build_ubuntu_20_04
165 + self.platforms = SUPPORTED_PLATFORMS
166 + elif self.display_name == "ubuntu22.04":
167 + self.docker_tag = "ubuntu:22.04"
168 + self.builder = imageutils.build_ubuntu_22_04
169 + self.platforms = SUPPORTED_PLATFORMS
170 + elif self.display_name == "ubuntu23.04":
171 + self.docker_tag = "ubuntu:23.04"
172 + self.builder = imageutils.build_ubuntu_23_04
173 + self.platforms = SUPPORTED_PLATFORMS
174 + elif self.display_name == "ubuntu23.10":
175 + self.docker_tag = "ubuntu:23.10"
176 + self.builder = imageutils.build_ubuntu_23_10
177 + self.platforms = SUPPORTED_PLATFORMS
178 + else:
179 + raise ValueError(f"Unknown distribution: {self.display_name}")
180 +
181 + def _cache_volume(
182 + self, client: dagger.Client, platform: dagger.Platform, path: str
183 + ) -> dagger.CacheVolume:
184 + tag = "_".join([self.display_name, Platform(platform).escaped()])
185 + return client.cache_volume(f"{path}-{tag}")
186 +
187 + def build(
188 + self, client: dagger.Client, platform: dagger.Platform
189 + ) -> dagger.Container:
190 + if platform not in self.platforms:
191 + raise ValueError(
192 + f"Building {self.display_name} is not supported on {platform}."
193 + )
194 +
195 + ctr = self.builder(client, platform)
196 + ctr = imageutils.install_cargo(ctr)
197 +
198 + return ctr
199 +
200 +
201 +class FeatureFlags(enum.Flag):
202 + DBEngine = enum.auto()
203 + GoPlugin = enum.auto()
204 + ExtendedBPF = enum.auto()
205 + LogsManagement = enum.auto()
206 + MachineLearning = enum.auto()
207 + BundledProtobuf = enum.auto()
208 +
209 +
210 +class NetdataInstaller:
211 + def __init__(
212 + self,
213 + platform: Platform,
214 + distro: Distribution,
215 + repo_root: pathlib.Path,
216 + prefix: pathlib.Path,
217 + features: FeatureFlags,
218 + ):
219 + self.platform = platform
220 + self.distro = distro
221 + self.repo_root = repo_root
222 + self.prefix = prefix
223 + self.features = features
224 +
225 + def _mount_repo(
226 + self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path
227 + ) -> dagger.Container:
228 + host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix()
229 + exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"]
230 +
231 + # The installer builds/stores intermediate artifacts under externaldeps/
232 + # We add a volume to speed up rebuilds. The volume has to be unique
233 + # per platform/distro in order to avoid mixing unrelated artifacts
234 + # together.
235 + externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps")
236 +
237 + ctr = (
238 + ctr.with_directory(
239 + self.repo_root.as_posix(), client.host().directory(host_repo_root)
240 + )
241 + .with_workdir(self.repo_root.as_posix())
242 + .with_mounted_cache(
243 + os.path.join(self.repo_root, "externaldeps"), externaldeps
244 + )
245 + )
246 +
247 + return ctr
248 +
249 + def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container:
250 + args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"]
251 +
252 + if FeatureFlags.DBEngine not in self.features:
253 + args.append("--disable-dbengine")
254 +
255 + if FeatureFlags.GoPlugin not in self.features:
256 + args.append("--disable-go")
257 +
258 + if FeatureFlags.ExtendedBPF not in self.features:
259 + args.append("--disable-ebpf")
260 +
261 + if FeatureFlags.LogsManagement not in self.features:
262 + args.append("--disable-logsmanagement")
263 +
264 + if FeatureFlags.MachineLearning not in self.features:
265 + args.append("--disable-ml")
266 +
267 + if FeatureFlags.BundledProtobuf not in self.features:
268 + args.append("--use-system-protobuf")
269 +
270 + args.extend(["--install-prefix", self.prefix.parent.as_posix()])
271 +
272 + ctr = self._mount_repo(client, ctr, self.repo_root.as_posix())
273 +
274 + ctr = ctr.with_env_variable(
275 + "NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug"
276 + ).with_exec(["./netdata-installer.sh"] + args)
277 +
278 + return ctr
279 +
280 +
281 +class Endpoint:
282 + def __init__(self, hostname: str, port: int):
283 + self.hostname = hostname
284 + self.port = port
285 +
286 + def __str__(self):
287 + return ":".join([self.hostname, str(self.port)])
288 +
289 +
290 +class ChildStreamConf:
291 + def __init__(
292 + self,
293 + installer: NetdataInstaller,
294 + destinations: List[Endpoint],
295 + api_key: uuid.UUID,
296 + ):
297 + self.installer = installer
298 + self.substitutions = {
299 + "enabled": "yes",
300 + "destination": " ".join([str(dst) for dst in destinations]),
301 + "api_key": api_key,
302 + "timeout_seconds": 60,
303 + "default_port": 19999,
304 + "send_charts_matching": "*",
305 + "buffer_size_bytes": 1024 * 1024,
306 + "reconnect_delay_seconds": 5,
307 + "initial_clock_resync_iterations": 60,
308 + }
309 +
310 + def render(self) -> str:
311 + tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf"
312 + with open(tmpl_path) as fp:
313 + tmpl = jinja2.Template(fp.read())
314 +
315 + return tmpl.render(**self.substitutions)
316 +
317 +
318 +class ParentStreamConf:
319 + def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID):
320 + self.installer = installer
321 + self.substitutions = {
322 + "api_key": str(api_key),
323 + "enabled": "yes",
324 + "allow_from": "*",
325 + "default_history": 3600,
326 + "health_enabled_by_default": "auto",
327 + "default_postpone_alarms_on_connect_seconds": 60,
328 + "multiple_connections": "allow",
329 + }
330 +
331 + def render(self) -> str:
332 + tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf"
333 + with open(tmpl_path) as fp:
334 + tmpl = jinja2.Template(fp.read())
335 +
336 + return tmpl.render(**self.substitutions)
337 +
338 +
339 +class StreamConf:
340 + def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf):
341 + self.child_conf = child_conf
342 + self.parent_conf = parent_conf
343 +
344 + def render(self) -> str:
345 + child_section = self.child_conf.render() if self.child_conf else ""
346 + parent_section = self.parent_conf.render() if self.parent_conf else ""
347 + return "\n".join([child_section, parent_section])
348 +
349 +
350 +class AgentContext:
351 + def __init__(
352 + self,
353 + client: dagger.Client,
354 + platform: dagger.Platform,
355 + distro: Distribution,
356 + installer: NetdataInstaller,
357 + endpoint: Endpoint,
358 + api_key: uuid.UUID,
359 + allow_children: bool,
360 + ):
361 + self.client = client
362 + self.platform = platform
363 + self.distro = distro
364 + self.installer = installer
365 + self.endpoint = endpoint
366 + self.api_key = api_key
367 + self.allow_children = allow_children
368 +
369 + self.parent_contexts = []
370 +
371 + self.built_distro = False
372 + self.built_agent = False
373 +
374 + def add_parent(self, parent_context: "AgentContext"):
375 + self.parent_contexts.append(parent_context)
376 +
377 + def build_container(self) -> dagger.Container:
378 + ctr = self.distro.build(self.client, self.platform)
379 + ctr = self.installer.install(self.client, ctr)
380 +
381 + if len(self.parent_contexts) == 0 and not self.allow_children:
382 + return ctr.with_exposed_port(self.endpoint.port)
383 +
384 + destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts]
385 + child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key)
386 +
387 + parent_stream_conf = None
388 + if self.allow_children:
389 + parent_stream_conf = ParentStreamConf(self.installer, self.api_key)
390 +
391 + stream_conf = StreamConf(child_stream_conf, parent_stream_conf)
392 +
393 + # write the stream conf to localhost and cp it in the container
394 + host_stream_conf_path = pathlib.Path(
395 + f"/tmp/{self.endpoint.hostname}_stream.conf"
396 + )
397 + with open(host_stream_conf_path, "w") as fp:
398 + fp.write(stream_conf.render())
399 +
400 + ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf"
401 +
402 + ctr = ctr.with_file(
403 + ctr_stream_conf_path.as_posix(),
404 + self.client.host().file(host_stream_conf_path.as_posix()),
405 + )
406 +
407 + ctr = ctr.with_exposed_port(self.endpoint.port)
408 +
409 + return ctr
packaging/dag/test_command.py new
+128
@@ -0,0 +1,128 @@
1 +import click
2 +import asyncio
3 +import sys
4 +import pathlib
5 +import dagger
6 +import uuid
7 +import httpx
8 +
9 +from nd import Distribution, NetdataInstaller, FeatureFlags, Endpoint, AgentContext
10 +
11 +
12 +def run_async(func):
13 + def wrapper(*args, **kwargs):
14 + return asyncio.run(func(*args, **kwargs))
15 +
16 + return wrapper
17 +
18 +
19 +@run_async
20 +async def simple_test():
21 + config = dagger.Config(log_output=sys.stdout)
22 +
23 + async with dagger.Connection(config) as client:
24 + platform = dagger.Platform("linux/x86_64")
25 + distro = Distribution("debian10")
26 +
27 + repo_root = pathlib.Path("/netdata")
28 + prefix_path = pathlib.Path("/opt/netdata")
29 + installer = NetdataInstaller(
30 + platform, distro, repo_root, prefix_path, FeatureFlags.DBEngine
31 + )
32 +
33 + api_key = uuid.uuid4()
34 +
35 + #
36 + # parent
37 + #
38 + parent_endpoint = Endpoint("parent1", 22000)
39 + parent_ctx = AgentContext(
40 + client, platform, distro, installer, parent_endpoint, api_key, True
41 + )
42 + parent_cmd = installer.prefix / "usr/sbin/netdata"
43 + parent_args = [
44 + parent_cmd.as_posix(),
45 + "-D",
46 + "-i",
47 + "0.0.0.0",
48 + "-p",
49 + str(parent_endpoint.port),
50 + ]
51 +
52 + parent_ctr = parent_ctx.build_container()
53 + parent_ctr = parent_ctr.with_exec(parent_args)
54 + parent_svc = parent_ctr.as_service()
55 +
56 + #
57 + # child
58 + #
59 + child_endpoint = Endpoint("child1", 21000)
60 + child_ctx = AgentContext(
61 + client, platform, distro, installer, child_endpoint, api_key, False
62 + )
63 + child_ctx.add_parent(parent_ctx)
64 + child_cmd = installer.prefix / "usr/sbin/netdata"
65 + child_args = [
66 + child_cmd.as_posix(),
67 + "-D",
68 + "-i",
69 + "0.0.0.0",
70 + "-p",
71 + str(child_endpoint.port),
72 + ]
73 +
74 + child_ctr = child_ctx.build_container()
75 + child_ctr = child_ctr.with_service_binding(parent_endpoint.hostname, parent_svc)
76 + child_ctr = child_ctr.with_exec(child_args)
77 + child_svc = child_ctr.as_service()
78 +
79 + #
80 + # endpoints
81 + #
82 + parent_tunnel, child_tunnel = await asyncio.gather(
83 + client.host().tunnel(parent_svc, native=True).start(),
84 + client.host().tunnel(child_svc, native=True).start(),
85 + )
86 +
87 + parent_endpoint, child_endpoint = await asyncio.gather(
88 + parent_tunnel.endpoint(),
89 + child_tunnel.endpoint(),
90 + )
91 +
92 + await asyncio.sleep(10)
93 +
94 + #
95 + # run tests
96 + #
97 +
98 + async with httpx.AsyncClient() as http:
99 + resp = await http.get(f"http://{parent_endpoint}/api/v1/info")
100 +
101 + #
102 + # Check that the child was connected
103 + #
104 + jd = resp.json()
105 + assert (
106 + "hosts-available" in jd
107 + ), "Could not find 'host-available' key in api/v1/info"
108 + assert jd["hosts-available"] == 2, "Child did not connect to parent"
109 +
110 + #
111 + # Check bearer protection
112 + #
113 + forbidden_urls = [
114 + f"http://{parent_endpoint}/api/v2/bearer_protection",
115 + f"http://{parent_endpoint}/api/v2/bearer_get_token",
116 + ]
117 +
118 + for url in forbidden_urls:
119 + async with httpx.AsyncClient() as http:
120 + resp = await http.get(url)
121 + assert (
122 + resp.status_code == httpx.codes.UNAVAILABLE_FOR_LEGAL_REASONS
123 + ), "Bearer protection is broken"
124 +
125 +
126 +@click.command(help="Run a simple parent/child test")
127 +def test():
128 + simple_test()