hydra-packet-importer: drop
Martin Weinelt committed
Jan 12, 2025 at 16:21 UTC
a3f56bfc38bfa5eef39e654cda725f5cd3f8408a
7 files changed
-269
.github/CODEOWNERS
-1
@@ -3,7 +3,6 @@
3
/.github/ @NixOS/infra-build
4
/build/ @NixOS/infra-build
5
/builders/ @NixOS/infra-build
6
-/hydra-packet-importer/ @NixOS/infra-build
6
/lib/ @NixOS/infra-build
7
/macs/ @NixOS/infra-build
8
/metrics/ @NixOS/infra-build
hydra-packet-importer/README.md
deleted
-27
@@ -1,27 +0,0 @@
1
-Imports builders' and their SSH keys from the Packet API. Requires the builder
2
-run this script at startup:
3
-
4
-```bash
5
-#!/usr/bin/env nix-shell
6
-#!nix-shell -i bash -p curl jq
7
-
8
-set -eux
9
-
10
-root_url=$(curl https://metadata.packet.net/metadata | jq -r .phone_home_url | rev | cut -d '/' -f2- | rev)
11
-url="$root_url/events"
12
-
13
-
14
-tell() {
15
- data=$(
16
- echo "{}" \
17
- | jq '.state = $state | .code = ($code | tonumber) | .message = $message' \
18
- --arg state "$1" \
19
- --arg code "$2" \
20
- --arg message "$3"
21
- )
22
-
23
- curl -v -X POST -d "$data" "$url"
24
-}
25
-
26
-tell succeeded 1001 "$(cat /etc/ssh/ssh_host_ed25519_key.pub)"
27
-```
hydra-packet-importer/config-example.json
deleted
-34
@@ -1,34 +0,0 @@
1
-{
2
- "token": "You read only API token",
3
- "project_id": "Your project's unique ID",
4
- "mandatory_tags": ["tags which each", "server", "must have"],
5
- "skip_tags": ["tags which will", "exclude a server from", "being imported"],
6
- "plans": {
7
- "c2.large.arm": {
8
- "user": "root",
9
- "system_types": ["aarch64-linux"],
10
- "ssh_key": "/path/to/hydras-ssh-key",
11
- "max_jobs": 15,
12
- "speed_factor": 1,
13
- "features": ["kvm", "nixos-test", "big-parallel"],
14
- "mandatory_features": []
15
- },
16
- "c2.medium.x86": {
17
- "user": "root",
18
- "system_types": ["x86_64-linux", "i686-linux"],
19
- "ssh_key": "/var/lib/hydra/queue-runner/.ssh/id_buildfarm_rsa",
20
- "max_jobs": 18,
21
- "speed_factor": 1,
22
- "features": ["kvm", "nixos-test", "big-parallel"]
23
- }
24
- },
25
- "name_overrides": {
26
- "machines-with-this-name-override-defaults-in-plan": {
27
- "max_jobs": 20,
28
- "system_types": ["x86_64-linux"],
29
- "max_jobs": 1,
30
- "speed_factor": 100,
31
- "features": ["kvm", "nixos-test", "big-parallel"]
32
- }
33
- }
34
-}
hydra-packet-importer/default.nix
deleted
-17
@@ -1,17 +0,0 @@
1
-{ python3 }:
2
-python3.pkgs.buildPythonApplication {
3
- name = "hydra-packet-importer";
4
- src = ./.;
5
-
6
- format = "other";
7
-
8
- nativeBuildInputs = [ python3.pkgs.mypy ];
9
-
10
- propagatedBuildInputs = [ python3.pkgs.packet-python ];
11
-
12
- installPhase = ''
13
- mypy --ignore-missing-imports ./import.py
14
- mkdir -p $out/bin
15
- mv import.py $out/bin/hydra-packet-importer
16
- '';
17
-}
hydra-packet-importer/import.py
deleted
-182
@@ -1,182 +0,0 @@
1
-#!/usr/bin/env python3
2
-
3
-import base64
4
-import json
5
-import sys
6
-from typing import Any, TypedDict
7
-
8
-import packet
9
-
10
-DeviceKeys = list[dict[str, Any]]
11
-
12
-
13
-class Metadata(TypedDict):
14
- user: str | None
15
- features: list[str]
16
- mandatory_features: list[str]
17
- max_jobs: int
18
- system_types: list[str]
19
- speed_factor: int | None
20
-
21
-
22
-class RemoteBuilder(TypedDict):
23
- metadata: Metadata
24
- ssh_key: str
25
-
26
-
27
-class Builder(TypedDict):
28
- hostname: str
29
- address: str
30
- remote_builder_info: RemoteBuilder
31
-
32
-
33
-class HostKey(TypedDict):
34
- system: str
35
- port: int
36
- key: str
37
-
38
-
39
-class Plan(TypedDict):
40
- name: str
41
-
42
-
43
-class Device(TypedDict):
44
- state: str
45
- tags: str
46
- id: str
47
- hostname: str
48
- short_id: str
49
- plan: Plan
50
-
51
-
52
-class ProjectDeviceList(TypedDict):
53
- meta: dict[str, Any]
54
- devices: list[Device]
55
-
56
-
57
-def debug(*args: Any, **kwargs: Any) -> None:
58
- print(*args, file=sys.stderr, **kwargs)
59
-
60
-
61
-def get_builders(manager: Any) -> list[Builder]:
62
- builders: list[Builder] = []
63
-
64
- page: str | None = "projects/{}/devices?page={}".format(config["project_id"], 1)
65
- while page is not None:
66
- debug(page)
67
- data: ProjectDeviceList = manager.call_api(page)
68
- page = None if data["meta"]["next"] is None else data["meta"]["next"]["href"]
69
-
70
- for device in data["devices"]:
71
- if device["state"] != "active":
72
- continue
73
-
74
- if not set(config["mandatory_tags"]).issubset(device["tags"]):
75
- continue
76
-
77
- if not set(device["tags"]).isdisjoint(config["skip_tags"]):
78
- continue
79
-
80
- remote_builder_info = get_remote_builder_info(manager, device["id"])
81
- if remote_builder_info is None:
82
- continue
83
-
84
- builders.append(
85
- {
86
- "hostname": device["hostname"],
87
- "address": "{}.packethost.net".format(device["short_id"]),
88
- "remote_builder_info": remote_builder_info,
89
- }
90
- )
91
-
92
- return builders
93
-
94
-
95
-def get_remote_builder_info(manager, device_id: str) -> RemoteBuilder | None:
96
- # ... 50 is probably enough.
97
- try:
98
- events_url = f"devices/{device_id}/events?per_page=50"
99
- debug(events_url)
100
- data = manager.call_api(events_url)
101
- except Exception:
102
- # 404 probably
103
- return None
104
-
105
- host_key: HostKey | None = None
106
- ssh_key: str | None = None
107
- metadata: Metadata | None = None
108
- for event in data["events"]:
109
- if event["type"] == "provisioning.104.01":
110
- # we reached a "Device connected to DHCP system" event,
111
- # indicating a reboot.
112
- #
113
- # The most first SSH key after DHCP is the one we want,
114
- # in case someone sends a bogus SSH key to the metadata
115
- # API after the post-boot hook.
116
- #
117
- # If we receive a LOT of spam (> 50 spams!) like that, we
118
- # will return None because we never reach this message.
119
- if host_key is not None:
120
- ssh_key = strip_ssh_key_comment(host_key["key"])
121
- if ssh_key is not None and metadata is not None:
122
- return {"metadata": metadata, "ssh_key": ssh_key}
123
- return None
124
- if event["type"] == "user.1001":
125
- try:
126
- host_keys: list[HostKey] = [
127
- key for key in json.loads(event["body"]) if key["port"] == 22
128
- ]
129
- host_key = host_keys[0]
130
- except Exception:
131
- pass
132
- if event["type"] == "user.1002":
133
- metadata = json.loads(event["body"])
134
-
135
- return None
136
-
137
-
138
-def strip_ssh_key_comment(key: str) -> str | None:
139
- ssh_key_parts = key.rsplit(" ", 1)
140
- if len(ssh_key_parts) == 2:
141
- return ssh_key_parts[0]
142
- debug("# Skipped due keyscan failed to split on ' '")
143
- return None
144
-
145
-
146
-def main(config: dict[str, Any]) -> None:
147
- rows = []
148
- manager = packet.Manager(auth_token=config["token"])
149
- found = 0
150
- for builder in get_builders(manager):
151
- found += 1
152
- debug("# {} ({})".format(builder["hostname"], builder["address"]))
153
-
154
- builder_info = builder["remote_builder_info"]
155
-
156
- # build@address system,list /var/lib/ssh.key maxJobs speedFactor feature,list mandatory,features public-host-key
157
- rows.append(
158
- " ".join(
159
- [
160
- "{user}@{host}".format(
161
- user="build",
162
- host=builder["address"],
163
- ),
164
- ",".join(builder_info["metadata"]["system_types"]),
165
- str(config["ssh_key"]),
166
- str(builder_info["metadata"]["max_jobs"]),
167
- str(builder_info["metadata"].get("speed_factor", 1)),
168
- ",".join(builder_info["metadata"]["features"]),
169
- ",".join(builder_info["metadata"].get("mandatory_features", ["-"])),
170
- base64.b64encode(builder_info["ssh_key"].encode()).decode("utf-8"),
171
- ]
172
- )
173
- )
174
-
175
- debug(f"# {len(rows)} / {found}")
176
- print("\n".join(rows))
177
-
178
-
179
-if __name__ == "__main__":
180
- with open(sys.argv[1]) as config_file:
181
- config = json.load(config_file)
182
- main(config)
hydra-packet-importer/shell.nix
deleted
-1
@@ -1 +0,0 @@
1
-(import <nixpkgs> { }).callPackage ./default.nix { }
pyproject.toml
-7
@@ -67,12 +67,6 @@ lint.ignore = [
67
"PTH118",
68
"PTH120"
69
]
70
-"hydra-packet-importer/import.py" = [
71
- "ANN001",
72
- "PTH123",
73
- "BLE001",
74
- "S110"
75
-]
70
"build/pluto/prometheus/exporters/**.py" = [
71
"ANN"
72
]
@@ -95,6 +89,5 @@ lint.ignore = [
89
]
90
91
[[tool.mypy.overrides]]
98
-module = "packet.*"
92
ignore_missing_imports = true
93