hydra-packet-importer: init
Graham Christensen committed
Apr 26, 2019 at 16:26 UTC
ecec43038755bcee152632e26495b1fcf2994a2d
4 files changed
+154
hydra-packet-importer/config-example.json
new
+33
@@ -0,0 +1,33 @@
1
+{
2
+ "token": "You read only API token",
3
+ "project_id": "Your project's unique ID",
4
+ "skip_tags": [ "tags which will", "exclude a server from", "being imported" ],
5
+ "plans": {
6
+ "c2.large.arm": {
7
+ "user": "root",
8
+ "system_types": ["aarch64-linux"],
9
+ "ssh_key": "/path/to/hydras-ssh-key",
10
+ "max_jobs": 15,
11
+ "speed_factor": 1,
12
+ "features": ["kvm","nixos-test","big-parallel"],
13
+ "mandatory_features": []
14
+ },
15
+ "c2.medium.x86": {
16
+ "user": "root",
17
+ "system_types": ["x86_64-linux", "i686-linux"],
18
+ "ssh_key": "/var/lib/hydra/queue-runner/.ssh/id_buildfarm_rsa",
19
+ "max_jobs": 18,
20
+ "speed_factor": 1,
21
+ "features": [ "kvm","nixos-test","big-parallel" ]
22
+ }
23
+ },
24
+ "name_overrides": {
25
+ "machines-with-this-name-override-defaults-in-plan": {
26
+ "max_jobs": 20,
27
+ "system_types": [ "x86_64-linux" ],
28
+ "max_jobs": 1,
29
+ "speed_factor": 100,
30
+ "features": [ "kvm","nixos-test","big-parallel" ]
31
+ }
32
+ }
33
+}
hydra-packet-importer/default.nix
new
+20
@@ -0,0 +1,20 @@
1
+{ stdenv, python3 }:
2
+stdenv.mkDerivation {
3
+ name = "hydra-packet-importer";
4
+ src = ./.;
5
+
6
+ buildInputs = [
7
+ (python3.withPackages (ps: [
8
+ ps.packet-python
9
+ ]))
10
+ ];
11
+
12
+ buildPhase = ''
13
+ patchShebangs ./import.py
14
+ '';
15
+
16
+ installPhase = ''
17
+ mkdir -p $out/bin
18
+ mv ./import.py $out/bin/hydra-packet-importer
19
+ '';
20
+}
hydra-packet-importer/import.py
new
+100
@@ -0,0 +1,100 @@
1
+#!/usr/bin/env python3
2
+
3
+import json
4
+import packet
5
+import base64
6
+from pprint import pprint
7
+import subprocess
8
+import sys
9
+
10
+def debug(*args, **kwargs):
11
+ print(*args, file=sys.stderr, **kwargs)
12
+
13
+
14
+def get_devices(manager):
15
+ devices = []
16
+
17
+ page = 'projects/%s/devices?page=%d' % (config['project_id'], 1)
18
+ while page is not None:
19
+ debug(page)
20
+ data = manager.call_api(page)
21
+ if data['meta']['next'] is None:
22
+ page = None
23
+ else:
24
+ page = data['meta']['next']['href']
25
+
26
+ for device in data['devices']:
27
+ if device['state'] != 'active':
28
+ continue
29
+ if 'spot_instance' not in device:
30
+ continue
31
+ if device['spot_instance'] != True:
32
+ continue
33
+
34
+ if not set(device['tags']).isdisjoint(config['skip_tags']):
35
+ continue
36
+
37
+ devices.append({
38
+ "hostname": device['hostname'],
39
+ "address": "{}.packethost.net".format(device['short_id']),
40
+ "type": device['plan']['name']
41
+ })
42
+
43
+ return devices
44
+
45
+def main(config):
46
+ rows = []
47
+ manager = packet.Manager(auth_token=config['token'])
48
+ found = 0
49
+ for device in get_devices(manager):
50
+ found += 1
51
+ debug("# {} ({})".format(device['hostname'], device['address']))
52
+ if device['type'] not in config['plans']:
53
+ debug("# Skipping {} (type {}) as it has no configured plan".format(
54
+ device['hostname'],
55
+ device['type'])
56
+ )
57
+ continue
58
+
59
+ default_stats = config['plans'][device['type']]
60
+ if device['hostname'] in config['name_overrides']:
61
+ specific_stats = config['name_overrides'][device['hostname']]
62
+ else:
63
+ specific_stats = {}
64
+
65
+ lookup = lambda key: specific_stats.get(key, device.get(key, default_stats.get(key)))
66
+ lookup_default = lambda key, default: default if not lookup(key) else lookup(key)
67
+
68
+ r = subprocess.check_output([
69
+ "ssh-keyscan",
70
+ "-4", # force IPv4
71
+ "-T", "5", # Timeout 5 seconds
72
+ "-t", "ed25519", # Only ed25519 keys
73
+ lookup("address")
74
+ ]).decode("utf-8")
75
+
76
+ elems = r.split(" ", 1)
77
+ if len(elems) != 2:
78
+ debug("# Skipped due keyscan failed to split on ' '")
79
+ continue
80
+ key = elems[1]
81
+
82
+ # root@address system,list /var/lib/ssh.key maxJobs speedFactor feature,list mandatory,features public-host-key
83
+ rows.append(" ".join([
84
+ "{user}@{host}".format(user=lookup("user"),host=lookup("address")),
85
+ ",".join(lookup("system_types")),
86
+ str(lookup("ssh_key")),
87
+ str(lookup("max_jobs")),
88
+ str(lookup("speed_factor")),
89
+ ",".join(lookup_default("features", ["-"])),
90
+ ",".join(lookup_default("mandatory_features", ["-"])),
91
+ base64.b64encode(key.encode()).decode("utf-8")
92
+ ]))
93
+
94
+ debug("# {} / {}".format(len(rows),found))
95
+ print("\n".join(rows))
96
+
97
+if __name__ == "__main__":
98
+ with open(sys.argv[1]) as config_file:
99
+ config = json.load(config_file)
100
+ main(config)
hydra-packet-importer/shell.nix
new
+1
@@ -0,0 +1 @@
1
+(import <nixpkgs> {}).callPackage ./default.nix {}