| 1 | from typing import List |
| 2 | |
| 3 | import enum |
| 4 | import os |
| 5 | import pathlib |
| 6 | import uuid |
| 7 | |
| 8 | import dagger |
| 9 | import jinja2 |
| 10 | |
| 11 | import imageutils |
| 12 | |
| 13 | |
| 14 | class Platform: |
| 15 | def __init__(self, platform: str): |
| 16 | self.platform = dagger.Platform(platform) |
| 17 | |
| 18 | def escaped(self) -> str: |
| 19 | return str(self.platform).removeprefix("linux/").replace("/", "_") |
| 20 | |
| 21 | def __eq__(self, other): |
| 22 | if isinstance(other, Platform): |
| 23 | return self.platform == other.platform |
| 24 | elif isinstance(other, dagger.Platform): |
| 25 | return self.platform == other |
| 26 | else: |
| 27 | return NotImplemented |
| 28 | |
| 29 | def __ne__(self, other): |
| 30 | return not (self == other) |
| 31 | |
| 32 | def __hash__(self): |
| 33 | return hash(self.platform) |
| 34 | |
| 35 | def __str__(self) -> str: |
| 36 | return str(self.platform) |
| 37 | |
| 38 | |
| 39 | SUPPORTED_PLATFORMS = set( |
| 40 | [ |
| 41 | Platform("linux/x86_64"), |
| 42 | Platform("linux/arm64"), |
| 43 | Platform("linux/i386"), |
| 44 | Platform("linux/arm/v7"), |
| 45 | Platform("linux/arm/v6"), |
| 46 | Platform("linux/ppc64le"), |
| 47 | Platform("linux/s390x"), |
| 48 | Platform("linux/riscv64"), |
| 49 | ] |
| 50 | ) |
| 51 | |
| 52 | |
| 53 | SUPPORTED_DISTRIBUTIONS = set( |
| 54 | [ |
| 55 | "alpine_3_18", |
| 56 | "alpine_3_19", |
| 57 | "amazonlinux2", |
| 58 | "centos7", |
| 59 | "centos-stream8", |
| 60 | "centos-stream9", |
| 61 | "debian10", |
| 62 | "debian11", |
| 63 | "debian12", |
| 64 | "fedora37", |
| 65 | "fedora38", |
| 66 | "fedora39", |
| 67 | "opensuse15.4", |
| 68 | "opensuse15.5", |
| 69 | "opensusetumbleweed", |
| 70 | "oraclelinux8", |
| 71 | "oraclelinux9", |
| 72 | "rockylinux8", |
| 73 | "rockylinux9", |
| 74 | "ubuntu20.04", |
| 75 | "ubuntu22.04", |
| 76 | "ubuntu23.04", |
| 77 | "ubuntu23.10", |
| 78 | ] |
| 79 | ) |
| 80 | |
| 81 | |
| 82 | class Distribution: |
| 83 | def __init__(self, display_name): |
| 84 | self.display_name = display_name |
| 85 | |
| 86 | if self.display_name == "alpine_3_18": |
| 87 | self.docker_tag = "alpine:3.18" |
| 88 | self.builder = imageutils.build_alpine_3_18 |
| 89 | self.platforms = SUPPORTED_PLATFORMS |
| 90 | elif self.display_name == "alpine_3_19": |
| 91 | self.docker_tag = "alpine:3.19" |
| 92 | self.builder = imageutils.build_alpine_3_19 |
| 93 | self.platforms = SUPPORTED_PLATFORMS |
| 94 | elif self.display_name == "amazonlinux2": |
| 95 | self.docker_tag = "amazonlinux:2" |
| 96 | self.builder = imageutils.build_amazon_linux_2 |
| 97 | self.platforms = SUPPORTED_PLATFORMS |
| 98 | elif self.display_name == "centos7": |
| 99 | self.docker_tag = "centos:7" |
| 100 | self.builder = imageutils.build_centos_7 |
| 101 | self.platforms = SUPPORTED_PLATFORMS |
| 102 | elif self.display_name == "centos-stream8": |
| 103 | self.docker_tag = "quay.io/centos/centos:stream8" |
| 104 | self.builder = imageutils.build_centos_stream_8 |
| 105 | self.platforms = SUPPORTED_PLATFORMS |
| 106 | elif self.display_name == "centos-stream9": |
| 107 | self.docker_tag = "quay.io/centos/centos:stream9" |
| 108 | self.builder = imageutils.build_centos_stream_9 |
| 109 | self.platforms = SUPPORTED_PLATFORMS |
| 110 | elif self.display_name == "debian10": |
| 111 | self.docker_tag = "debian:10" |
| 112 | self.builder = imageutils.build_debian_10 |
| 113 | self.platforms = SUPPORTED_PLATFORMS |
| 114 | elif self.display_name == "debian11": |
| 115 | self.docker_tag = "debian:11" |
| 116 | self.builder = imageutils.build_debian_11 |
| 117 | self.platforms = SUPPORTED_PLATFORMS |
| 118 | elif self.display_name == "debian12": |
| 119 | self.docker_tag = "debian:12" |
| 120 | self.builder = imageutils.build_debian_12 |
| 121 | self.platforms = SUPPORTED_PLATFORMS |
| 122 | elif self.display_name == "fedora37": |
| 123 | self.docker_tag = "fedora:37" |
| 124 | self.builder = imageutils.build_fedora_37 |
| 125 | self.platforms = SUPPORTED_PLATFORMS |
| 126 | elif self.display_name == "fedora38": |
| 127 | self.docker_tag = "fedora:38" |
| 128 | self.builder = imageutils.build_fedora_38 |
| 129 | self.platforms = SUPPORTED_PLATFORMS |
| 130 | elif self.display_name == "fedora39": |
| 131 | self.docker_tag = "fedora:39" |
| 132 | self.platforms = SUPPORTED_PLATFORMS |
| 133 | self.builder = imageutils.build_fedora_39 |
| 134 | elif self.display_name == "opensuse15.4": |
| 135 | self.docker_tag = "opensuse/leap:15.4" |
| 136 | self.builder = imageutils.build_opensuse_15_4 |
| 137 | self.platforms = SUPPORTED_PLATFORMS |
| 138 | elif self.display_name == "opensuse15.5": |
| 139 | self.docker_tag = "opensuse/leap:15.5" |
| 140 | self.builder = imageutils.build_opensuse_15_5 |
| 141 | self.platforms = SUPPORTED_PLATFORMS |
| 142 | elif self.display_name == "opensusetumbleweed": |
| 143 | self.docker_tag = "opensuse/tumbleweed:latest" |
| 144 | self.builder = imageutils.build_opensuse_tumbleweed |
| 145 | self.platforms = SUPPORTED_PLATFORMS |
| 146 | elif self.display_name == "oraclelinux8": |
| 147 | self.docker_tag = "oraclelinux:8" |
| 148 | self.builder = imageutils.build_oracle_linux_8 |
| 149 | self.platforms = SUPPORTED_PLATFORMS |
| 150 | elif self.display_name == "oraclelinux9": |
| 151 | self.docker_tag = "oraclelinux:9" |
| 152 | self.builder = imageutils.build_oracle_linux_9 |
| 153 | self.platforms = SUPPORTED_PLATFORMS |
| 154 | elif self.display_name == "rockylinux8": |
| 155 | self.docker_tag = "rockylinux:8" |
| 156 | self.builder = imageutils.build_rocky_linux_8 |
| 157 | self.platforms = SUPPORTED_PLATFORMS |
| 158 | elif self.display_name == "rockylinux9": |
| 159 | self.docker_tag = "rockylinux:9" |
| 160 | self.builder = imageutils.build_rocky_linux_9 |
| 161 | self.platforms = SUPPORTED_PLATFORMS |
| 162 | elif self.display_name == "ubuntu20.04": |
| 163 | self.docker_tag = "ubuntu:20.04" |
| 164 | self.builder = imageutils.build_ubuntu_20_04 |
| 165 | self.platforms = SUPPORTED_PLATFORMS |
| 166 | elif self.display_name == "ubuntu22.04": |
| 167 | self.docker_tag = "ubuntu:22.04" |
| 168 | self.builder = imageutils.build_ubuntu_22_04 |
| 169 | self.platforms = SUPPORTED_PLATFORMS |
| 170 | elif self.display_name == "ubuntu23.04": |
| 171 | self.docker_tag = "ubuntu:23.04" |
| 172 | self.builder = imageutils.build_ubuntu_23_04 |
| 173 | self.platforms = SUPPORTED_PLATFORMS |
| 174 | elif self.display_name == "ubuntu23.10": |
| 175 | self.docker_tag = "ubuntu:23.10" |
| 176 | self.builder = imageutils.build_ubuntu_23_10 |
| 177 | self.platforms = SUPPORTED_PLATFORMS |
| 178 | else: |
| 179 | raise ValueError(f"Unknown distribution: {self.display_name}") |
| 180 | |
| 181 | def _cache_volume( |
| 182 | self, client: dagger.Client, platform: dagger.Platform, path: str |
| 183 | ) -> dagger.CacheVolume: |
| 184 | tag = "_".join([self.display_name, Platform(platform).escaped()]) |
| 185 | return client.cache_volume(f"{path}-{tag}") |
| 186 | |
| 187 | def build( |
| 188 | self, client: dagger.Client, platform: dagger.Platform |
| 189 | ) -> dagger.Container: |
| 190 | if platform not in self.platforms: |
| 191 | raise ValueError( |
| 192 | f"Building {self.display_name} is not supported on {platform}." |
| 193 | ) |
| 194 | |
| 195 | ctr = self.builder(client, platform) |
| 196 | ctr = imageutils.install_cargo(ctr) |
| 197 | |
| 198 | return ctr |
| 199 | |
| 200 | |
| 201 | class FeatureFlags(enum.Flag): |
| 202 | DBEngine = enum.auto() |
| 203 | GoPlugin = enum.auto() |
| 204 | ExtendedBPF = enum.auto() |
| 205 | LogsManagement = enum.auto() |
| 206 | MachineLearning = enum.auto() |
| 207 | BundledProtobuf = enum.auto() |
| 208 | |
| 209 | |
| 210 | class NetdataInstaller: |
| 211 | def __init__( |
| 212 | self, |
| 213 | platform: Platform, |
| 214 | distro: Distribution, |
| 215 | repo_root: pathlib.Path, |
| 216 | prefix: pathlib.Path, |
| 217 | features: FeatureFlags, |
| 218 | ): |
| 219 | self.platform = platform |
| 220 | self.distro = distro |
| 221 | self.repo_root = repo_root |
| 222 | self.prefix = prefix |
| 223 | self.features = features |
| 224 | |
| 225 | def _mount_repo( |
| 226 | self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path |
| 227 | ) -> dagger.Container: |
| 228 | host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix() |
| 229 | exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"] |
| 230 | |
| 231 | # The installer builds/stores intermediate artifacts under externaldeps/ |
| 232 | # We add a volume to speed up rebuilds. The volume has to be unique |
| 233 | # per platform/distro in order to avoid mixing unrelated artifacts |
| 234 | # together. |
| 235 | externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps") |
| 236 | |
| 237 | ctr = ( |
| 238 | ctr.with_directory( |
| 239 | self.repo_root.as_posix(), client.host().directory(host_repo_root) |
| 240 | ) |
| 241 | .with_workdir(self.repo_root.as_posix()) |
| 242 | .with_mounted_cache( |
| 243 | os.path.join(self.repo_root, "externaldeps"), externaldeps |
| 244 | ) |
| 245 | ) |
| 246 | |
| 247 | return ctr |
| 248 | |
| 249 | def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container: |
| 250 | args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"] |
| 251 | |
| 252 | if FeatureFlags.DBEngine not in self.features: |
| 253 | args.append("--disable-dbengine") |
| 254 | |
| 255 | if FeatureFlags.GoPlugin not in self.features: |
| 256 | args.append("--disable-go") |
| 257 | |
| 258 | if FeatureFlags.ExtendedBPF not in self.features: |
| 259 | args.append("--disable-ebpf") |
| 260 | |
| 261 | if FeatureFlags.MachineLearning not in self.features: |
| 262 | args.append("--disable-ml") |
| 263 | |
| 264 | if FeatureFlags.BundledProtobuf not in self.features: |
| 265 | args.append("--use-system-protobuf") |
| 266 | |
| 267 | args.extend(["--install-prefix", self.prefix.parent.as_posix()]) |
| 268 | |
| 269 | ctr = self._mount_repo(client, ctr, self.repo_root.as_posix()) |
| 270 | |
| 271 | ctr = ctr.with_env_variable( |
| 272 | "NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug" |
| 273 | ).with_exec(["./netdata-installer.sh"] + args) |
| 274 | |
| 275 | return ctr |
| 276 | |
| 277 | |
| 278 | class Endpoint: |
| 279 | def __init__(self, hostname: str, port: int): |
| 280 | self.hostname = hostname |
| 281 | self.port = port |
| 282 | |
| 283 | def __str__(self): |
| 284 | return ":".join([self.hostname, str(self.port)]) |
| 285 | |
| 286 | |
| 287 | class ChildStreamConf: |
| 288 | def __init__( |
| 289 | self, |
| 290 | installer: NetdataInstaller, |
| 291 | destinations: List[Endpoint], |
| 292 | api_key: uuid.UUID, |
| 293 | ): |
| 294 | self.installer = installer |
| 295 | self.substitutions = { |
| 296 | "enabled": "yes", |
| 297 | "destination": " ".join([str(dst) for dst in destinations]), |
| 298 | "api_key": api_key, |
| 299 | "timeout_seconds": 60, |
| 300 | "default_port": 19999, |
| 301 | "send_charts_matching": "*", |
| 302 | "buffer_size_bytes": 1024 * 1024, |
| 303 | "reconnect_delay_seconds": 5, |
| 304 | "initial_clock_resync_iterations": 60, |
| 305 | } |
| 306 | |
| 307 | def render(self) -> str: |
| 308 | tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf" |
| 309 | with open(tmpl_path) as fp: |
| 310 | tmpl = jinja2.Template(fp.read()) |
| 311 | |
| 312 | return tmpl.render(**self.substitutions) |
| 313 | |
| 314 | |
| 315 | class ParentStreamConf: |
| 316 | def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID): |
| 317 | self.installer = installer |
| 318 | self.substitutions = { |
| 319 | "api_key": str(api_key), |
| 320 | "enabled": "yes", |
| 321 | "allow_from": "*", |
| 322 | "default_history": 3600, |
| 323 | "health_enabled_by_default": "auto", |
| 324 | "default_postpone_alarms_on_connect_seconds": 60, |
| 325 | "multiple_connections": "allow", |
| 326 | } |
| 327 | |
| 328 | def render(self) -> str: |
| 329 | tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf" |
| 330 | with open(tmpl_path) as fp: |
| 331 | tmpl = jinja2.Template(fp.read()) |
| 332 | |
| 333 | return tmpl.render(**self.substitutions) |
| 334 | |
| 335 | |
| 336 | class StreamConf: |
| 337 | def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf): |
| 338 | self.child_conf = child_conf |
| 339 | self.parent_conf = parent_conf |
| 340 | |
| 341 | def render(self) -> str: |
| 342 | child_section = self.child_conf.render() if self.child_conf else "" |
| 343 | parent_section = self.parent_conf.render() if self.parent_conf else "" |
| 344 | return "\n".join([child_section, parent_section]) |
| 345 | |
| 346 | |
| 347 | class AgentContext: |
| 348 | def __init__( |
| 349 | self, |
| 350 | client: dagger.Client, |
| 351 | platform: dagger.Platform, |
| 352 | distro: Distribution, |
| 353 | installer: NetdataInstaller, |
| 354 | endpoint: Endpoint, |
| 355 | api_key: uuid.UUID, |
| 356 | allow_children: bool, |
| 357 | ): |
| 358 | self.client = client |
| 359 | self.platform = platform |
| 360 | self.distro = distro |
| 361 | self.installer = installer |
| 362 | self.endpoint = endpoint |
| 363 | self.api_key = api_key |
| 364 | self.allow_children = allow_children |
| 365 | |
| 366 | self.parent_contexts = [] |
| 367 | |
| 368 | self.built_distro = False |
| 369 | self.built_agent = False |
| 370 | |
| 371 | def add_parent(self, parent_context: "AgentContext"): |
| 372 | self.parent_contexts.append(parent_context) |
| 373 | |
| 374 | def build_container(self) -> dagger.Container: |
| 375 | ctr = self.distro.build(self.client, self.platform) |
| 376 | ctr = self.installer.install(self.client, ctr) |
| 377 | |
| 378 | if len(self.parent_contexts) == 0 and not self.allow_children: |
| 379 | return ctr.with_exposed_port(self.endpoint.port) |
| 380 | |
| 381 | destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts] |
| 382 | child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key) |
| 383 | |
| 384 | parent_stream_conf = None |
| 385 | if self.allow_children: |
| 386 | parent_stream_conf = ParentStreamConf(self.installer, self.api_key) |
| 387 | |
| 388 | stream_conf = StreamConf(child_stream_conf, parent_stream_conf) |
| 389 | |
| 390 | # write the stream conf to localhost and cp it in the container |
| 391 | host_stream_conf_path = pathlib.Path( |
| 392 | f"/tmp/{self.endpoint.hostname}_stream.conf" |
| 393 | ) |
| 394 | with open(host_stream_conf_path, "w") as fp: |
| 395 | fp.write(stream_conf.render()) |
| 396 | |
| 397 | ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf" |
| 398 | |
| 399 | ctr = ctr.with_file( |
| 400 | ctr_stream_conf_path.as_posix(), |
| 401 | self.client.host().file(host_stream_conf_path.as_posix()), |
| 402 | ) |
| 403 | |
| 404 | ctr = ctr.with_exposed_port(self.endpoint.port) |
| 405 | |
| 406 | return ctr |