@cryptotaxi247 / infra-1 / commits / e2ea8e52

Initial check-in, todo: break this up :)

Graham Christensen committed Oct 19, 2018 at 23:41 UTC e2ea8e5200c546cf197a1ccf3e8997373b5a8b99
11 files changed +1022
macs/README.md new
+91
@@ -0,0 +1,91 @@
1 +# MacOS Infrastructure
2 +
3 +Contained are Nix expression for deploying the hydra.nixos.org macOS
4 +infrastructure. Each computer is genuine Apple hardware running NixOS
5 +on the host, with macOS using almost all of the host's resources in an
6 +immutable QEMU virtual machine.
7 +
8 +The virtualisation of macOS seems to be a less error-prone, and easier
9 +to recover from problems.
10 +
11 +
12 +## Bootstrapping a new mac
13 +
14 +### Initial Setup
15 +
16 +We distribute the macOS image with `zfs send` / `zfs receive`. First
17 +enable ZFS in the installation environment.
18 +
19 +1. Add `boot.supportedFilesystems = [ "zfs" ];` to
20 + `/etc/nixos/configuration.nix`
21 +2. Run `nixos-rebuild switch`
22 +3. `modprobe zfs`
23 +
24 +### Partitioning, Formatting, Mounting
25 +
26 +1. Partition the disk:
27 +
28 +```
29 +parted /dev/sda -- mklabel gpt
30 +parted /dev/sda -- mkpart primary 512MiB -16GiB
31 +parted /dev/sda -- mkpart primary linux-swap -16GiB -1MiB
32 +parted /dev/sda -- mkpart ESP fat32 1MiB 512MiB
33 +parted /dev/sda -- set 3 boot on
34 +```
35 +
36 +2. Create a zpool with `/dev/sda` and mount it:
37 +```
38 +zpool create -o ashift=12 -o altroot=/mnt rpool /dev/sda1
39 +zfs create -o mountpoint=legacy rpool/root
40 +mount -t zfs rpool/root/nixos /mnt
41 +```
42 +_note: ashift=12 is copypasta, maybe somebody knows better_
43 +
44 +3. Create the EFI System Partition and mount it:
45 +
46 +```
47 +mkfs.fat -F 32 -n boot /dev/sda3
48 +mkdir /mnt/boot
49 +mount /dev/sda3 /mnt/boot
50 +```
51 +
52 +4. Create and enable swap:
53 +
54 +```
55 +mkswap -L swap /dev/sda2
56 +swapon /dev/sda2
57 +```
58 +
59 +### Generate Configuration
60 +
61 +1. Generate the config
62 +
63 +```
64 +nixos-generate-config --root /mnt
65 +```
66 +
67 +2. Generate a host ID with `head -c 8 /etc/machine-id` , we'll refer
68 + to it soon.
69 +
70 +3. Edit `/mnt/etc/nixos/hardware-configuration.nix` and:
71 +
72 + - change the `/boot` fs device to `/dev/disk/by-label/boot`
73 + - change the `swap` device to `/dev/disk/by-label/swap`
74 + - delete the `cpuFreqGovernor` line
75 + - add `boot.supportedFilesystems = [ "zfs" ];`
76 + - add `networking.hostId = "the-host-id-you-generated";`
77 + - add `nixpkgs.config.allowUnfree = true;` if the `broadcom-sta`
78 + kernel module is enabled.
79 +
80 +### Install
81 +
82 +Run `nixos-install` and reboot.
83 +
84 +### Addition to the NixOps Network
85 +
86 +calculate your own `-smp` line like this:
87 +
88 + - cores: # of cores per socket
89 + - threads: # of threads per core, ie: hyperthreading? set to 2, none? set to 1
90 + - sockets: # of physical sockets in the system
91 + - cpus = * cores * threads * sockets
macs/build.nix new
+32
@@ -0,0 +1,32 @@
1 +import <nixpkgs/nixos> {
2 +configuration = { pkgs, lib, config, ... }: {
3 + imports = [
4 + ./host/default.nix
5 + ];
6 + nixpkgs.config.allowUnfree = true;
7 +
8 + fileSystems."/" =
9 + { device = "rpool/root";
10 + fsType = "zfs";
11 + };
12 +
13 + networking.hostId = "aaaaaaaa";
14 + boot.loader.systemd-boot.enable = true;
15 + boot.loader.efi.canTouchEfiVariables = true;
16 +
17 + networking.hostName = "dummy";
18 + macosGuest.guest = {
19 + cores = 2;
20 + threads = 2;
21 + sockets = 2;
22 + memoryInMegs = 6 * 1024;
23 +
24 + zvolName = "rpool/example";
25 + guestConfigDir = ./guest;
26 +
27 + ovmfCodeFile = ./dist/OVMF_CODE.fd;
28 + ovmfVarsFile = ./dist/OVMF_VARS-1024x768.fd;
29 + cloverImage = ./dist/Clover.qcow2;
30 + };
31 + };
32 +}
macs/dist/Clover.qcow2
Binary files /dev/null and b/macs/dist/Clover.qcow2 differ
macs/dist/OVMF_CODE.fd
Binary files /dev/null and b/macs/dist/OVMF_CODE.fd differ
macs/dist/OVMF_VARS-1024x768.fd
Binary files /dev/null and b/macs/dist/OVMF_VARS-1024x768.fd differ
macs/guest/apply.sh new
+83
@@ -0,0 +1,83 @@
1 +#!/usr/bin/env bash
2 +
3 +PS4='${BASH_SOURCE}::${FUNCNAME[0]}::$LINENO '
4 +set -o pipefail
5 +set -ex
6 +date
7 +
8 +function finish {
9 + set +e
10 + cd /
11 + sleep 1
12 + umount -f /Volumes/CONFIG
13 +}
14 +trap finish EXIT
15 +
16 +cat <<EOF | tee -a /etc/ssh/sshd_config
17 +PermitRootLogin prohibit-password
18 +PasswordAuthentication no
19 +PermitEmptyPasswords no
20 +ChallengeResponseAuthentication no
21 +EOF
22 +
23 +launchctl stop com.openssh.sshd
24 +launchctl start com.openssh.sshd
25 +
26 +
27 +cd /Volumes/CONFIG
28 +
29 +cp -r ./etc/ssh/ssh_host_* /etc/ssh
30 +chown root:wheel /etc/ssh/ssh_host_*
31 +chmod 600 /etc/ssh/ssh_host_*
32 +cd /
33 +
34 +echo "%admin ALL = NOPASSWD: ALL" | tee /etc/sudoers.d/passwordless
35 +
36 +(
37 + # Make this thing work as root
38 + export USER=root
39 + export HOME=~root
40 + export ALLOW_PREEXISTING_INSTALLATION=1
41 + env
42 + curl https://nixos.org/releases/nix/nix-2.1.3/install > ~nixos/install-nix
43 + chmod +rwx ~nixos/install-nix
44 + cat /dev/null | sudo -i -H -u nixos -- sh ~nixos/install-nix --daemon
45 +)
46 +
47 +(
48 + # Make this thing work as root
49 + export USER=root
50 + export HOME=~root
51 +
52 + . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh'
53 + env
54 + ls -la /private || true
55 + ls -la /private/var || true
56 + ls -la /private/var/run || true
57 + ln -s /private/var/run /run || true
58 + nix-channel --add https://github.com/LnL7/nix-darwin/archive/master.tar.gz darwin
59 + nix-channel --update
60 +
61 + sudo -i -H -u nixos -- nix-channel --add https://github.com/LnL7/nix-darwin/archive/master.tar.gz darwin
62 + sudo -i -H -u nixos -- nix-channel --update
63 +
64 + export NIX_PATH=$NIX_PATH:darwin=https://github.com/LnL7/nix-darwin/archive/master.tar.gz
65 +
66 + installer=$(nix-build https://github.com/LnL7/nix-darwin/archive/master.tar.gz -A installer --no-out-link)
67 + set +e
68 + yes | sudo -i -H -u nixos -- $installer/bin/darwin-installer;
69 + echo $?
70 + set -e
71 +)
72 +
73 +(
74 + export USER=root
75 + export HOME=~root
76 +
77 + rm -f /etc/nix/nix.conf
78 + rm -f /etc/bashrc
79 + ln -s /etc/static/bashrc /etc/bashrc
80 + . /etc/static/bashrc
81 + cat /Volumes/CONFIG/darwin-configuration.nix | sudo -u nixos -- tee ~nixos/.nixpkgs/darwin-configuration.nix
82 + sudo -i -H -u nixos -- darwin-rebuild switch
83 +)
macs/guest/darwin-configuration.nix new
+83
@@ -0,0 +1,83 @@
1 +{ config, lib, pkgs, ... }:
2 +
3 +with lib;
4 +
5 +let
6 + sshKeys = rec {
7 + eelco = "ssh-dss AAAAB3NzaC1kc3MAAACBAOo3foMFsYvc+LEVVTAeXpaxdOFG6O2NE9coxZYN6UtwE477GwkvZ4uKymAekq3TB8I6dDg4QFfE27fIip/rQHJ/Rus+KsxwnTbwPzE0WcZVpkKQsepsoqLkfwMpiPfn5/oxcnJsimwRY/E95aJmmOHdGaYWrc0t4ARa+6teUgdFAAAAFQCSQq2Wil0/X4hDypGGUKlKvYyaWQAAAIAy/0fSDnz1tZOQBGq7q78y406HfWghErrVlrW9g+foJQG5pgXXcdJs9JCIrlaKivUKITDsYnQaCjrZaK8eHnc4ksbkSLfDOxFnR5814ulCftrgEDOv9K1UU3pYketjFMvQCA2U48lR6jG/99CPNXPH55QEFs8H97cIsdLQw9wM4gAAAIEAmzWZlXLzIf3eiHQggXqvw3+C19QvxQITcYHYVTx/XYqZi1VZ/fkY8bNmdcJsWFyOHgEhpEca+xM/SNvH/14rXDmt0wtclLEx/4GVLi59hQCnnKqv7HzJg8RF4v6XTiROBAEEdb4TaFuFn+JCvqPzilTzXTexvZKJECOvfYcY+10= eelco.dolstra@logicblox.com";
8 +
9 + rob = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHp3/OZP2nRS5bM9E1xN8Q2L398kC+m4guORjKsmGjwnnHvYnTml5puE2ogl8Wdenbk7hf82+vKyB+Tktrhx+IBSym4lY+czR6W+39hlPYdLbi980yxYT9KEMSyMWJEgPVJ1BZvHqsHQiad/L3eoPmAIMDmcn4mLh9rya5/oMW/ZgsA6j28ClvWkDRyaTmTLOa0Im4nLoSbdo8kJqU+JX/YcXlMKUvFfdMcj4T9YYwV98LPWHnEHFmjtBBUXRUAIESMXS6pm3Pep3czkKUL4UF0u9f17b40OWlLOF4IQWE2jM9yK09DiIQUzeU2XKRNW116DnmDL5QIRNrYnhkYeeQI3U6WnVTPdTU9kBVTDjhM+6U6/LClGJaWiglwwrzHtVELHgMi280qRefQEftb4CI/IbcPNAxetJevV68I5NAjfdnmMx8YbhfIiEqAJtBi4TvoH7HjDH+72+ZFjQ10fpz/p+DgUtiNlRKz8tXSZ+mbLuhmOJOxtGQTH3viYbSpG/4F9uKW1ekX0RMyRxVvpjMxHtCL4daJI4RTHFXy4R16OKAlYe7gs9sqv7O0IujLJPex/rnN2U4syGaSH5q3UnGxci6qgn8yLEhSP+Gj0xdv5H3fVjr/kNNZGWDOz6nDUaJT+eWlmWU7hOvm0ricrz9GEPUTQ0Rh70sWTQFq3poWQ== cardno:000606167509";
10 +
11 + provisioner = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKDSfHu0xHu2qtWjmCC92rTMZfwZNKXrsJvPCLSoWtzR eelco.dolstra@deploy";
12 +
13 + danny = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDlxgNIC4HbyLXB7jexKOEDkMRmiIfDnjOrx445otnR7w87hAehq8If+lnfK/ezGbss6yuWpTjjlc9gi03AxxL9KJW+xAe8avoC36/EeFfBJtRFMaUTjZ0XrcW09Yl8b9BWKypOXSOoF5JAJjV3yI160p+bzhSa584gY+Gj6A2SkVpnUPQfahm3gj0esqVBYVHfN1KXsHq8S89RxOr9HrRU5mTyZn0xO6YE3w+PG2f77vBhZMutt2+28xcMZCNO0HyY/WkWZo7tmJB/HLEENzz1OYiYCXjSEAgEgMbx69kG8r2fOLkrpbKLc/Wbg6fMsi7DM2G5egFPAEqxF1HXHf9T dan@dans-mbp.ws.tudelft.net";
14 +
15 + build-farm = "ssh-dss AAAAB3NzaC1kc3MAAACBAMHRjGSDaBp4Z30JF4S9ApabBCpdr57Ad0aD9oH2A/WEFnWYQSAzK4E/HHD2DV2XP1stNkZ1ks2v3F4Yu/veR+qVlUWbJW1RIIfuQgkG44K0R3C2qx4BAZUVYzju1NVCJbBOO6ipVY9cfmpokV52HZFhP/2HocTNLoav3F0AsbbJAAAAFQDaJiQdpJBEa4Wr5FfVl1kYqmQZJwAAAIEAwbern5XL+SNIMa+sJ3CBhrWyYExYWiUbdmhQEfyEAUmoPsEr1qpb+0WREic9Nrxz48QWZDK5xMvzZyQEkuAMJUBWcdm12rME7WMvg7OZGr9DADjAtfMfj3Ui2XvOuQ3ia/OTsMGkQTDWnkOM9Ni128SNSl9urFBlXATdGvo+468AAACBAK8s6LddhhkRqsF/l/L2ooS8c8A1rTFWAOy3/sgXFNvMyS/Mig2p966xRrRHr7Bc+H2SuKEE5WmLCXqymgxLHhrFU4zm/W/ej1yB1CAThd4xUfgJu4touJROjvcD1zzlmLeat0fp2k5mCuiLKcTKi0vxKWiiopF9nvBBK+7ODPC7 buildfarm@nixos";
16 +
17 + hydra-queue-runner = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCyM48VC5fpjJssLI8uolFscP4/iEoMHfkPoT9R3iE3OEjadmwa1XCAiXUoa7HSshw79SgPKF2KbGBPEVCascdAcErZKGHeHUzxj7v3IsNjObouUOBbJfpN4DR7RQT28PZRsh3TvTWjWnA9vIrSY/BvAK1uezFRuObvatqAPMrw4c0DK+JuGuCNkKDGHLXNSxYBc5Pmr1oSU7/BDiHVjjyLIsAMIc20+q8SjWswKqL1mY193mN7FpUMBtZrd0Za9fMFRII9AofEIDTOayvOZM6+/1dwRWZXM6jhE6kaPPF++yromHvDPBnd6FfwODKLvSF9BkA3pO5CqrD8zs7ETmrV hydra-queue-runner@chef";
18 +
19 + daniel_peebles = [
20 + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsDWV0eYYYpGL3GRFL68mG4aw5xTTiEWp4AyheJEA0WciZb8SZY/RxoYv3l7ccAQYOF7vf4qxRXb5KEGV0tZkcsAc7qSn4Hcg9sSVp2xb1sWsvhGIlJV87QBk0r2UVnom7xSncot67M2u2MUxGWNrTEbXir5FjUcYQYIInwiDhJ7jPZaZDYY4LGs8pBQaVYCPdfxAnsWqZgJnqjO9lwkK7OgJajEkMKhK3xixqFPhKUDiJ3MxmRewHelHTBcxN8ghz5G3Rb+qmfg2ZQGxQWHN3l7IFqrHEcGHQAiKoYPXd2aL6iHgojPHWiWT7efvLVC6nnqlCwHtWyErI+IEeXBF3 copumpkin@work"
21 + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCkoril5uKjJohHvqz9Ys9R2rBH95MUb4Rxo5kcuRvEIwMranQ7xP5eU7rZqfv7elE1DLfMs19via+btUX3w8o4juYxzXjafnH6Mck5hYdvxNnErW6gsp0vGDQ0ruRCQx3UmOuC5Ld/wXY7iMQqOlxeLZF2dVCKP1+BSs37wLC7scXYu0U+wODprVpAsZIOwLP85w/uCNlC8wbvNDWG+Hx+XD/ml2ezQiNBRnh7Qo3QKgpUvVBO0d9z84g92D2H9IA+pEpJiWFcYKGEowKSVQVFCi5LoWRiz8XLKL+JeBt5mmmqjmJua6o8lXV7+nba//KCIkG+IWS4nwKQlpzZXc4H copumpkin@home"
22 + ];
23 +
24 + graham_christensen = [
25 + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDUDEJaYzYDc87ZutbpWifJ7h2Zk7winbS7/0Qyy/+yGwwJ8qP+Mvje6IwDwsnQuNZ8XZLKdGq7iDaXIa8hfvT8+wbwURxlUvJhhp1eQ6dI1/n6vtVFN0nOnCSPHdgmAQsNoqbt3RG7gGAPsLwAyn2MMfmbsdkz9JF1p7Lja+brNHmXkaVCU4Jq90f5Qv+TrwJNN+VIy4yxU3m7zvQZg0A5cG0bR5SZDMzceL4AsCtxpV+HBiG9tcBETn/Cw60bl7b7cGQFuZRlrZBPyIoyZ3be6Bscv0lOZSMBJzWyYTeOxNbIT2rR5yv8aSW/taQIcJ6LkZYszT6xe+52x/iAXGlL grahamc"
26 + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDY8wRHQtq9uBzdiAYzpSNmF+nmIHmW+AOeBTDNmdva+CFGIBbB56q7w6GCOhfXs8edrPY4qOcQGaOD0ussIvHnqkVfw8e6CbxnpXKeAuIz7+1V72AhLPzOkif4yPrI6tSYF5nvzq6U4Yk1qFnXiLQjkA1s4EcZH6V0KbHMsu7Mtv3Irspdn8KUI3j2UwZcssFu1EuLHhLNussziRQK9tOg9ixb0U1WXuUJn7Noh9odTAsAt6jLFdr5eN/IINgC9WQqvY/W94Tc2/z5TWR7z382pEkMBR/3sf+nYKA82069tagkyrtJ/YXi00CWU4vjpnMvwPEYcmtCddfCPi8ZIUrn grahamc"
27 + ];
28 +
29 + mac_keys = [
30 + # Unclear what to do with Rob's key which doesn't match above, and
31 + # the other key, presumably Eelco's
32 + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDI6/qMXX80oWm+NyftRw45D+mRJwJQ6gexkUhp1OgZc3MuW6Zm2RO2IZHEjJLSMUndZebbznPmPPM58VxiyQnRYH2+hn+qCrwSsyCUxA8Gz6PpxeaeUMlpbsuXOPFbvBraDZEqIvx/gIK849nIahGz3EcfaY73lVRP+MrrVHBGyQmaOLoNfzrJp8rZfLqokQQXmG1d3DzjkIi87TZLgrdxQewpk/4eKBKf8FDnEYeV3ood78SPa3syS48al99Q7e8JyAEZJfyCQkUSUxgSizU5+se1A5seDJg2Vsqef1Ah23g/lTtSn93vtjjLvObvMJTSplBO8ttG/3ylIewWYER/ rbvermaa@nixos"
33 + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDfNUQXdhu4tIC+oDtq12aKRw3mfHa1nP/sMRkE379bnByQWqgpr8cCsXaNsZIrM49Iv/cP2JxZT1S3K4kfp6ouvNN+rYubOrpHLt+NWhiI+1s5IpgZv21Ln1kANjo4jzKLTRfoGv1gWILTG1KSD8oTev1kE1p3GJph5pTVilzAW1uXNhkSpYVMIw6HwqPR4QN1UliD5FvAdz6FJ16E7/xhaVWdeEOcsYw2uRXBaY/rXkKtikscZ99wnOiCf6Gph2ahLkmZ/I3QmNq+xH+Xq6vpx6Kky0MKNm2zlh5tRDOd7Wd5N3sQpGAtHL4qGObq0N3VCtuJIL5eVRpZ5l5yNLsp"
34 + ] ++ graham_christensen ++ daniel_peebles;
35 +};
36 + environment = concatStringsSep " "
37 + [
38 + "NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
39 + ];
40 +
41 + authorizedNixStoreKey = key:
42 + "command=\"${environment} ${config.nix.package}/bin/nix-store --serve --write\" ${key}";
43 +in
44 +
45 +{
46 + environment.systemPackages =
47 + [
48 + config.nix.package
49 + ];
50 +
51 + programs.bash.enable = true;
52 + programs.bash.enableCompletion = false;
53 +
54 + #services.activate-system.enable = true;
55 +
56 + services.nix-daemon.enable = true;
57 +
58 + nix.maxJobs = 4;
59 + nix.buildCores = 1;
60 + nix.gc.automatic = true;
61 + nix.gc.options = let
62 + gbFree = 25;
63 + in "--max-freed $((${toString gbFree} * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | awk '{ print $4 }')))";
64 +
65 + environment.etc."per-user/root/ssh/authorized_keys".text = concatStringsSep "\n"
66 + ([(authorizedNixStoreKey sshKeys.build-farm)
67 + (authorizedNixStoreKey sshKeys.hydra-queue-runner)
68 + ] ++ sshKeys.mac_keys);
69 +
70 +
71 + system.activationScripts.postActivation.text = ''
72 + printf "disabling spotlight indexing... "
73 + mdutil -i off -d / &> /dev/null
74 + mdutil -E / &> /dev/null
75 + echo "ok"
76 +
77 + printf "configuring ssh keys for hydra on the root account... "
78 + mkdir -p ~root/.ssh
79 + cp -f /etc/per-user/root/ssh/authorized_keys ~root/.ssh/authorized_keys
80 + chown root:wheel ~root ~root/.ssh ~root/.ssh/authorized_keys
81 + echo "ok"
82 + '';
83 +}
macs/host/default.nix new
+191
@@ -0,0 +1,191 @@
1 +
2 +{ lib, config, ... }:
3 +let
4 + inherit (lib) mkOption types;
5 +in {
6 + options = {
7 + macosGuest = {
8 + enable = mkOption {
9 + default = false;
10 + type = types.bool;
11 + description = ''
12 + Whether to enable the macOS guest, including networking and
13 + the QEMU VM.
14 + '';
15 + };
16 +
17 + network = {
18 + externalInterface = mkOption {
19 + type = types.str;
20 + description = ''
21 + Public network interface to forward traffic through.
22 + '';
23 + };
24 +
25 + interiorNetworkPrefix = mkOption {
26 + type = types.str;
27 + description = ''
28 + The first three octets of the network to use for the virtual
29 + machine. The VM always runs in a /24 network. If you use the
30 + value "192.168.1", the host will have IP 192.168.1.1 and the
31 + guest will have IP 192.168.1.2
32 + '';
33 +
34 + example = "192.168.1";
35 + };
36 + };
37 +
38 + guest = {
39 + sockets = mkOption {
40 + type = types.int;
41 + description = ''
42 + The number of physical CPU Sockets in the system.
43 +
44 + # lscpu
45 + Architecture: x86_64
46 + CPU op-mode(s): 32-bit, 64-bit
47 + Byte Order: Little Endian
48 + CPU(s): 4
49 + On-line CPU(s) list: 0-3
50 + Thread(s) per core: 2
51 + Core(s) per socket: 2
52 + Socket(s): 1 <------
53 + '';
54 + };
55 +
56 + cores = mkOption {
57 + type = types.int;
58 + description = ''
59 + The number of Cores per Socket.
60 +
61 + # lscpu
62 + Architecture: x86_64
63 + CPU op-mode(s): 32-bit, 64-bit
64 + Byte Order: Little Endian
65 + CPU(s): 4
66 + On-line CPU(s) list: 0-3
67 + Thread(s) per core: 2
68 + Core(s) per socket: 2 <------
69 + Socket(s): 1
70 + '';
71 + };
72 +
73 + threads = mkOption {
74 + type = types.int;
75 + description = ''
76 + The number of Threads per Core.
77 +
78 + # lscpu
79 + Architecture: x86_64
80 + CPU op-mode(s): 32-bit, 64-bit
81 + Byte Order: Little Endian
82 + CPU(s): 4
83 + On-line CPU(s) list: 0-3
84 + Thread(s) per core: 2 <------
85 + Core(s) per socket: 2
86 + Socket(s): 1
87 + '';
88 + };
89 +
90 + memoryInMegs = mkOption {
91 + type = types.int;
92 + description = ''
93 + I have no idea what "megs" is, but QEMU's documentatation
94 + says this is the number of megs. Save 1G or 2G or so for
95 + the host and ZFS.
96 + '';
97 + };
98 +
99 +
100 + MACAddress = mkOption {
101 + type = types.str;
102 + description = ''
103 + The MAC address to assign the guest's NIC.
104 + '';
105 +
106 + default = "52:54:00:c9:18:27";
107 + };
108 +
109 + persistentConfigDir = mkOption {
110 + type = types.str;
111 + description = ''
112 + A path on the guest to store secret, persistent
113 + configuration like SSH host keys.
114 +
115 + Host keys are generated on the host and copied to the VM
116 + to ensure they don't change on every boot.
117 + '';
118 + default = "/var/lib/macos-vm-persistent-config";
119 + };
120 +
121 + zvolName = mkOption {
122 + type = types.str;
123 + description = ''
124 + Name of the zvol containing the root disk image.
125 + '';
126 + example = "rpool/my-disk-image";
127 + };
128 +
129 + snapshotName = mkOption {
130 + type = types.str;
131 + description = ''
132 + Name of the snapshot on the zvolName.
133 +
134 + There must be a snapshot because the disk state is rolled
135 + back on every boot.
136 +
137 + The snapshot name is combined with zvolName like:
138 + zvolName@snapshotName
139 + '';
140 + example = "pristine";
141 + default = "pristine";
142 + };
143 +
144 + guestConfigDir = mkOption {
145 + type = types.path;
146 + description = ''
147 + A directory of configuration files to expose to the VM.
148 +
149 + At a minimum, it should contain an `apply.sh` file in the
150 + root. This is executed by the macOS VM on boot-up. Note
151 + the configuration will be mounted at /Volumes/CONFIG as a
152 + cdrom.
153 +
154 + At /Volumes/CONFIG/etc/ssh/ will be SSH host keys which
155 + should be copied to /etc/ssh/ on the host. Additionally,
156 + the script should finish by unmounting /Volumes/CONFIG
157 + otherwise it is possible for programs runnig on the guest
158 + to read the SSH host keys.
159 + '';
160 + };
161 +
162 + ovmfCodeFile = mkOption {
163 + type = types.path;
164 + description = ''
165 + Path to the OVMF Code File.
166 + '';
167 + };
168 +
169 + ovmfVarsFile = mkOption {
170 + type = types.path;
171 + description = ''
172 + Path to the OVMF Variable File.
173 + '';
174 + };
175 +
176 + cloverImage = mkOption {
177 + type = types.path;
178 + description = ''
179 + Path to the Clover bootloader.
180 + '';
181 + };
182 +
183 + };
184 + };
185 + };
186 +
187 + imports = [
188 + ./networking.nix
189 + ./qemu.nix
190 + ];
191 +}
macs/host/networking.nix new
+84
@@ -0,0 +1,84 @@
1 +{ lib, config, ... }:
2 +let
3 + inherit (lib) mkIf;
4 +
5 + subnetIP = "${config.macosGuest.network.interiorNetworkPrefix}.0";
6 + routerIP = "${config.macosGuest.network.interiorNetworkPrefix}.1";
7 + guestIP = "${config.macosGuest.network.interiorNetworkPrefix}.2";
8 + broadcastIP = "${config.macosGuest.network.interiorNetworkPrefix}.255";
9 +in {
10 + config = mkIf config.macosGuest.enable {
11 + boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = true;
12 + boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = true;
13 +
14 + networking.firewall.extraCommands = ''
15 + ip46tables -A nixos-fw -i tap0 -p udp --dport 53 -j nixos-fw-accept # knot dns / kresd
16 + '';
17 +
18 + networking.firewall.allowedTCPPorts = [
19 + 2200 # forwarded port to the guest
20 + ];
21 +
22 + networking.nat = {
23 + enable = true;
24 + externalInterface = config.macosGuest.network.externalInterface;
25 + internalInterfaces = [
26 + "tap0"
27 + ];
28 + internalIPs = [
29 + "${subnetIP}/24"
30 + ];
31 + forwardPorts = [
32 + {
33 + destination = "${guestIP}:22";
34 + proto = "tcp";
35 + sourcePort = 2200;
36 + }
37 + ];
38 + };
39 +
40 + networking.interfaces."tap0" = {
41 + virtual = true;
42 + ipv4.addresses = [
43 + {
44 + address = routerIP;
45 + prefixLength = 24;
46 + }
47 + ];
48 + };
49 +
50 + services.dhcpd4 = {
51 + enable = true;
52 + interfaces = [ "tap0" ];
53 + extraConfig = ''
54 + authoritative;
55 + subnet ${subnetIP} netmask 255.255.255.0 {
56 + option routers ${routerIP};
57 + option broadcast-address ${broadcastIP};
58 + option domain-name-servers ${routerIP};
59 +
60 + group {
61 + host builder {
62 + hardware ethernet ${config.macosGuest.guest.MACAddress};
63 + fixed-address ${guestIP};
64 + }
65 + }
66 + }
67 + '';
68 + };
69 +
70 + services.kresd = {
71 + enable = true;
72 + interfaces = [ "::1" "127.0.0.1" routerIP ];
73 + extraConfig = ''
74 + modules = {
75 + 'policy', -- Block queries to local zones/bad sites
76 + 'stats', -- Track internal statistics
77 + 'predict', -- Prefetch expiring/frequent records
78 + }
79 + -- Smaller cache size
80 + cache.size = 10 * MB
81 + '';
82 + };
83 + };
84 +}
macs/host/qemu.nix new
+73
@@ -0,0 +1,73 @@
1 +
2 +{ config, lib, pkgs, ... }:
3 +let
4 + inherit (config.macosGuest.guest) threads cores sockets memoryInMegs
5 + ovmfCodeFile ovmfVarsFile cloverImage zvolName snapshotName
6 + guestConfigDir persistentConfigDir;
7 + inherit (lib) mkIf;
8 +
9 + zvolDevice = "/dev/zvol/${zvolName}";
10 + snapshot = "${zvolName}@${snapshotName}";
11 +in {
12 + config = mkIf config.macosGuest.enable {
13 + systemd.services.create-macos-secrets = {
14 + path = with pkgs; [ openssh ];
15 +
16 + serviceConfig = {
17 + Type = "oneshot";
18 + RemainAfterExit = true;
19 + };
20 +
21 + script = ''
22 + if [ ! -f ${persistentConfigDir}/etc/ssh/ssh_host_ed25519_key ]; then
23 + mkdir -p ${persistentConfigDir}/etc/ssh
24 + ssh-keygen -A -f ${persistentConfigDir}
25 + fi
26 + '';
27 + };
28 +
29 + systemd.services."run-macos-vm" = {
30 + after = [ "create-macos-secrets.service" ];
31 + requires = [ "create-macos-secrets.service" ];
32 + wantedBy = [ "multi-user.target" ];
33 + path = with pkgs; [ zfs qemu cdrkit rsync findutils ];
34 +
35 + serviceConfig.PrivateTmp = true;
36 +
37 + preStart = ''
38 + zfs rollback ${snapshot}
39 +
40 + # Create a cloud-init style cdrom
41 + rm -rf /tmp/cdr
42 + cp -r ${persistentConfigDir} /tmp/cdr
43 + rsync -r ${guestConfigDir}/ /tmp/cdr
44 + cd /tmp/cdr
45 + find .
46 + genisoimage -v -J -r -V CONFIG -o /tmp/config.iso .
47 + '';
48 + postStop = "zfs rollback ${snapshot}";
49 + script = ''
50 + qemu-system-x86_64 \
51 + -enable-kvm \
52 + -cpu Penryn,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on,+aes,+xsave,+avx,+xsaveopt,avx2,+smep \
53 + -machine pc-q35-2.9 \
54 + -smp cpus=${toString (cores * threads * sockets)},cores=${toString cores},threads=${toString threads},sockets=${toString sockets} \
55 + -m ${toString memoryInMegs} \
56 + -usb -device usb-kbd -device usb-tablet \
57 + -device isa-applesmc,osk="ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc" \
58 + -drive if=pflash,format=raw,readonly,file=${ovmfCodeFile} \
59 + -drive if=pflash,format=raw,snapshot=on,file=${ovmfVarsFile} \
60 + -smbios type=2 \
61 + -device ich9-intel-hda -device hda-duplex \
62 + -device ide-drive,bus=ide.2,drive=Clover \
63 + -drive id=Clover,if=none,snapshot=on,format=qcow2,file='${cloverImage}' \
64 + -device ide-drive,bus=ide.1,drive=MacHDD \
65 + -drive id=MacHDD,cache=unsafe,if=none,file=${zvolDevice},format=raw \
66 + -device ide-drive,bus=ide.0,drive=config \
67 + -drive id=config,if=none,snapshot=on,media=cdrom,file=/tmp/config.iso \
68 + -netdev tap,id=net0,ifname=tap0,script=no,downscript=no -device e1000-82545em,netdev=net0,id=net0,mac=${config.macosGuest.guest.MACAddress} \
69 + -vnc 127.0.0.1:0
70 + '';
71 + };
72 + };
73 +}
macs/notes.md new
+385
@@ -0,0 +1,385 @@
1 +
2 +## Generating a new macOS disk image
3 +
4 +This is less practiced since it is only done rarely. These steps will
5 +likely require changes every time, as the OS upgrades happen.
6 +
7 +The following are just notes I took during this process.
8 +
9 +generate a disk image and clover image from
10 +https://github.com/kholia/OSX-KVM/tree/master/HighSierra
11 +
12 +I was at commit `3d995ed38ba72955c9355324ab92bd56d8bcf879` and
13 +downloaded `CloverISO-4699.tar.lzma` from SourceForge with sha256sum
14 +`d85ae93ef3aa3ef6e5b7074778cd3dbc2d74b4bdc5f6d4f6214ea213e0644602`
15 +and my High Sierra ISO was `macos-high-sierra-10.13.6-cdr.iso`
16 +
17 +I applied the following patch to OSX-KVM:
18 +
19 +```diff
20 +commit 223e3ebff7501219cf5ced8422ee2726a117a6aa
21 +Author: Graham Christensen <graham@grahamc.com>
22 +Date: Mon Oct 8 20:09:44 2018 +0000
23 +
24 + NixOS patches
25 +
26 +diff --git a/Clover.qcow2 b/Clover.qcow2
27 +index 8527b16..51e049c 100644
28 +Binary files a/Clover.qcow2 and b/Clover.qcow2 differ
29 +diff --git a/HighSierra/clover-image.sh b/HighSierra/clover-image.sh
30 +index 9300f7e..8dd1e32 100755
31 +--- a/HighSierra/clover-image.sh
32 ++++ b/HighSierra/clover-image.sh
33 +@@ -1,4 +1,5 @@
34 +-#!/bin/bash
35 ++#!/usr/bin/env nix-shell
36 ++#!nix-shell -i bash -p libguestfs
37 +
38 + # https://github.com/kraxel/imagefish
39 +
40 +diff --git a/HighSierra/clover/config.plist.stripped.qemu b/HighSierra/clover/config.plist.stripped.qemu
41 +index 79f7d7b..2159d89 100644
42 +--- a/HighSierra/clover/config.plist.stripped.qemu
43 ++++ b/HighSierra/clover/config.plist.stripped.qemu
44 +@@ -7,7 +7,7 @@
45 + <key>Arguments</key>
46 + <string></string>
47 + <key>DefaultVolume</key>
48 +- <string>clover</string>
49 ++ <string>system</string>
50 + <key>Log</key>
51 + <true/>
52 + <key>Secure</key>
53 +diff --git a/boot-macOS-HS.sh b/boot-macOS-HS.sh
54 +index 7e39eb8..b2f26d8 100755
55 +--- a/boot-macOS-HS.sh
56 ++++ b/boot-macOS-HS.sh
57 +@@ -1,4 +1,5 @@
58 +-#!/bin/bash
59 ++#!/usr/bin/env nix-shell
60 ++#!nix-shell -i bash -p qemu
61 +
62 + # See https://www.mail-archive.com/qemu-devel@nongnu.org/msg471657.html thread.
63 + #
64 +@@ -15,7 +16,7 @@ MY_OPTIONS="+aes,+xsave,+avx,+xsaveopt,avx2,+smep"
65 +
66 + qemu-system-x86_64 -enable-kvm -m 3072 -cpu Penryn,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on,$MY_OPTIONS\
67 + -machine pc-q35-2.9 \
68 +- -smp 4,cores=2 \
69 ++ -smp cpus=8,cores=4,threads=2,sockets=1 -m 14336 \
70 + -usb -device usb-kbd -device usb-tablet \
71 + -device isa-applesmc,osk="ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc" \
72 + -drive if=pflash,format=raw,readonly,file=OVMF_CODE.fd \
73 +@@ -29,4 +30,6 @@ qemu-system-x86_64 -enable-kvm -m 3072 -cpu Penryn,kvm=on,vendor=GenuineIntel,+i
74 + -device ide-drive,bus=ide.0,drive=MacDVD \
75 + -drive id=MacDVD,if=none,snapshot=on,media=cdrom,file=./'HighSierra-10.13.6.iso' \
76 + -netdev tap,id=net0,ifname=tap0,script=no,downscript=no -device e1000-82545em,netdev=net0,id=net0,mac=52:54:00:c9:18:27 \
77 +- -monitor stdio
78 ++ -monitor stdio \
79 ++ -vnc 127.0.0.1:0
80 ++
81 +```
82 +
83 +plus the patch
84 +
85 +```diff
86 +commit cee4519beb23a015f39116e178b3e0f642df6ed2
87 +Author: Graham Christensen <graham@grahamc.com>
88 +Date: Mon Oct 8 22:08:51 2018 +0000
89 +
90 + provision / ephemeral
91 +
92 +diff --git a/boot-macOS-HS-ephemeral.sh b/boot-macOS-HS-ephemeral.sh
93 +new file mode 100755
94 +index 0000000..1101977
95 +--- /dev/null
96 ++++ b/boot-macOS-HS-ephemeral.sh
97 +@@ -0,0 +1,35 @@
98 ++#!/usr/bin/env nix-shell
99 ++#!nix-shell -i bash -p qemu
100 ++
101 ++# See https://www.mail-archive.com/qemu-devel@nongnu.org/msg471657.html thread.
102 ++#
103 ++# The "pc-q35-2.4" machine type was changed to "pc-q35-2.9" on 06-August-2017.
104 ++#
105 ++# The "media=cdrom" part is needed to make Clover recognize the bootable ISO
106 ++# image.
107 ++
108 ++##################################################################################
109 ++# NOTE: Comment out the "MY_OPTIONS" line in case you are having booting problems!
110 ++##################################################################################
111 ++
112 ++MY_OPTIONS="+aes,+xsave,+avx,+xsaveopt,avx2,+smep"
113 ++
114 ++qemu-system-x86_64 -enable-kvm -m 3072 -cpu Penryn,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on,$MY_OPTIONS\
115 ++ -machine pc-q35-2.9 \
116 ++ -smp cpus=8,cores=4,threads=2,sockets=1 -m 14336 \
117 ++ -usb -device usb-kbd -device usb-tablet \
118 ++ -device isa-applesmc,osk="ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc" \
119 ++ -drive if=pflash,format=raw,readonly,file=OVMF_CODE.fd \
120 ++ -drive if=pflash,format=raw,file=OVMF_VARS-1024x768.fd \
121 ++ -smbios type=2 \
122 ++ -snapshot \
123 ++ -device ich9-intel-hda -device hda-duplex \
124 ++ -device ide-drive,bus=ide.2,drive=Clover \
125 ++ -drive id=Clover,if=none,snapshot=on,format=qcow2,file=./'Clover.qcow2' \
126 ++ -device ide-drive,bus=ide.1,drive=MacHDD \
127 ++ -drive id=MacHDD,if=none,snapshot=on,file=./mac_hdd.img,format=qcow2 \
128 ++ -device ide-drive,bus=ide.0,drive=MacDVD \
129 ++ -drive id=MacDVD,if=none,snapshot=on,media=cdrom,file=./'HighSierra-10.13.6.iso' \
130 ++ -netdev tap,id=net0,ifname=tap0,script=no,downscript=no -device e1000-82545em,netdev=net0,id=net0,mac=52:54:00:c9:18:27 \
131 ++ -vnc 127.0.0.1:0
132 ++
133 +diff --git a/boot-macOS-HS.sh b/boot-macOS-HS-provision.sh
134 +similarity index 100%
135 +rename from boot-macOS-HS.sh
136 +rename to boot-macOS-HS-provision.sh
137 +```
138 +
139 +calculate your own `-smp` line like this:
140 +
141 + - cores: # of cores per socket
142 + - threads: # of threads per core, ie: hyperthreading? set to 2, none? set to 1
143 + - sockets: # of physical sockets in the system
144 + - cpus = * cores * threads * sockets
145 +
146 +
147 +generate your own Clover.qcow2:
148 +
149 +```
150 +[nix-shell:~/OSX-KVM/HighSierra]# ./clover-image.sh --iso ./clover-ext/Clover-v2.4k-4699-X64.iso --cfg clover/config.plist.stripped.qemu --img Clover.qcow2
151 +### copy files from iso
152 +### creating and adding disk image
153 +# disk-create Clover.qcow2 qcow2 256M
154 +# add Clover.qcow2
155 +# run
156 +### partition disk image
157 +# part-init /dev/sda gpt
158 +# part-add /dev/sda p 2048 200000
159 +# part-add /dev/sda p 202048 -2048
160 +# part-set-gpt-type /dev/sda 1 C12A7328-F81F-11D2-BA4B-00A0C93EC93B
161 +# part-set-bootable /dev/sda 1 true
162 +# mkfs vfat /dev/sda1 label:EFI
163 +# mkfs vfat /dev/sda2 label:clover
164 +# mount /dev/sda2 /
165 +# mkdir /ESP
166 +# mount /dev/sda1 /ESP
167 +### copy files to disk image
168 +'clover/config.plist.stripped.qemu' -> '/run/user/0/clover-image.sh-2833/config.plist'
169 +# mkdir /ESP/EFI
170 +# mkdir /ESP/EFI/CLOVER
171 +# copy-in /run/user/0/clover-image.sh-2833/EFI/BOOT /ESP/EFI
172 +# copy-in /run/user/0/clover-image.sh-2833/EFI/CLOVER/CLOVERX64.efi /ESP/EFI/CLOVER
173 +# copy-in /run/user/0/clover-image.sh-2833/EFI/CLOVER/drivers64UEFI /ESP/EFI/CLOVER
174 +# copy-in /run/user/0/clover-image.sh-2833/EFI/CLOVER/drivers-Off/drivers64UEFI/PartitionDxe-64.efi /ESP/EFI/CLOVER/drivers64UEFI
175 +# copy-in apfs.efi /ESP/EFI/CLOVER/drivers64UEFI
176 +# copy-in /run/user/0/clover-image.sh-2833/EFI/CLOVER/tools /ESP/EFI/CLOVER
177 +# copy-in /run/user/0/clover-image.sh-2833/config.plist /ESP/EFI/CLOVER
178 +# -*- OsxAptioFix v3 -*-
179 +# copy-in /run/user/0/clover-image.sh-2833/EFI/CLOVER/drivers-Off/drivers64UEFI/OsxAptioFix3Drv-64.efi /ESP/EFI/CLOVER/drivers64UEFI
180 +# ls /ESP/EFI/CLOVER/drivers64UEFI
181 +DataHubDxe-64.efi
182 +FSInject-64.efi
183 +OsxAptioFix3Drv-64.efi
184 +PartitionDxe-64.efi
185 +SMCHelper-64.efi
186 +VBoxHfs-64.efi
187 +apfs.efi
188 +# umount-all
189 +### cleaning up ...
190 +```
191 +
192 +[nix-shell:~/OSX-KVM/HighSierra]# cp Clover.qcow2 ../
193 +
194 +
195 +then:
196 +
197 +[root@nixos:~/OSX-KVM]# nix-shell -p qemu
198 +
199 +[nix-shell:~/OSX-KVM]# qemu-img create -f qcow2 mac_hdd.img 128G
200 +Formatting 'mac_hdd.img', fmt=qcow2 size=137438953472 cluster_size=65536 lazy_refcounts=off refcount_bits=16
201 +
202 +then:
203 +
204 +[root@nixos:~/OSX-KVM]# ./boot-macOS-HS.sh
205 +QEMU 3.0.0 monitor - type 'help' for more information
206 +(qemu)
207 +
208 +
209 +then, use tigervnc's vncviewer. If you're running this on a remote
210 +machine you can port-forward the VNC port via
211 +`ssh -L 5900:localhost:5900 root@10.5.3.153`.
212 +
213 +1. boot the install disk (only option)
214 +2. select "English" langage
215 +3. select "Disk Utility"
216 +4. Find the "QEMU HARDDISK Media" disk which is about 130GB
217 +5. click Erase, name: system (exactly `system`), format: Mac OS Extended (Journaled), scheme: GUID Partition Map, click Done
218 +6. exit Disk Utility
219 +7. select "Install macOS"
220 +8. select "system" as the target disk
221 +9. install will proceed and automatically reboot to the new root disk
222 +and continue installation. this takes about 20-30 minutes.
223 +10. Once the install process gets to the "Welcome" screen where you
224 +select a physical location, Ctrl-C the QEMU process, copy the disk
225 +image to another location for safe keeping. This duplicated image will
226 +be used for future fresh re-setting-up like major upgrades:
227 +`cp mac_hdd.img mac-hdd-1-installed-not-set-up.img` save this
228 +somewhere for long-term storage.
229 +11. re-run:
230 +
231 +[root@nixos:~/OSX-KVM]# ./boot-macOS-HS-provision.sh
232 +QEMU 3.0.0 monitor - type 'help' for more information
233 +(qemu)
234 +(qemu) usb_desc_get_descriptor: 2 unknown type 33 (len 10)
235 +usb_desc_get_descriptor: 1 unknown type 33 (len 10)
236 +qemu-system-x86_64: terminating on signal 2
237 +
238 +[root@nixos:~/OSX-KVM]# cp mac_hdd.img mac-hdd-1-installed-not-set-up.img
239 +
240 +[root@nixos:~/OSX-KVM]# ./boot-macOS-HS-provision.sh
241 +QEMU 3.0.0 monitor - type 'help' for more information
242 +(qemu)
243 +
244 +and reconnect over vnc
245 +
246 +12. Select "United States"
247 +13. Select "US" Keyboard
248 +14. When asked to sign in with an Apple ID, click "Set Up Later"
249 +which is probably near the top
250 +15. create a user:
251 + full name: nixos
252 + account name: nixos
253 + password: generate a new one each time, note: nixos is not a good password =)
254 + hint: set no hint
255 +16. select "customize setup"
256 +17. don't enable location services
257 +18. select your timezone: UTC - United Kingdom
258 +19. untick "share mac analytics" and "share crash data"
259 +20. You'll get to the desktop and it'll try to config the keyboard,
260 +press `z` then `/` then select `ANSI` and click Done
261 +21. Click the magnifying glass in the top bar
262 +22. Type "term" and press enter on Terminal
263 +23. Run `sudo systemsetup -setremotelogin on` to turn on SSH.
264 + IMPORTANT: DO NOT TEST SSH AT THIS STAGE!
265 +Testing SSH now would cause the image to generate an SSH host key, and
266 +cause it to be fixed in a generic disk image too soon.
267 +24. Disable the protections preventing you from running unsigned
268 +software: `sudo spctl --master-disable`
269 +25. Enable automaticly mounting ISOs even before users log in,
270 + (should be one line):
271 + `sudo defaults write
272 + /Library/Preferences/SystemConfiguration/autodiskmount
273 + AutomountDisksWithoutUserLogin -bool YES`
274 +26. Load the auto-run script, add the following to
275 + /Library/LaunchDaemons/org.nixos.bootup.plist:
276 +
277 +<?xml version="1.0" encoding="UTF-8"?>
278 +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
279 +<plist version="1.0">
280 +<dict>
281 + <key>Label</key>
282 + <string>org.nixos.bootup</string>
283 + <key>ProgramArguments</key>
284 + <array>
285 + <string>bash</string>
286 + <string>/Volumes/CONFIG/apply.sh</string>
287 + </array>
288 + <key>StandardOutPath</key>
289 + <string>/tmp/apply.stdout</string>
290 + <key>StandardErrorPath</key>
291 + <string>/tmp/apply.stderr</string>
292 + <key>RunAtLoad</key>
293 + <true/>
294 + <key>StartOnMount</key>
295 + <true/>
296 +</dict>
297 +</plist>
298 +
299 +Copy-paste it, or if that doesn't work (it doesn't for me,) use a
300 +pastebin. It is annoying to get this wrong, so be careful.
301 +
302 +It might be here already:
303 +https://gist.github.com/grahamc/126b1a28d50d99db315fb5b6fce551c7
304 +
305 +27. Via the apple menu, select Shut Down
306 +28. untick "Reopen windowsn when logging back in"
307 +29. shut down
308 +30. When the computer is shut down, duplicate mac_hd.img again:
309 +`cp mac_hdd.img mac-hdd-2-initial-setup.img` and back this image up
310 +as well. This image is used as the basis for ofborg and hydra
311 +builders.
312 +
313 +---
314 +
315 +Specializing the image
316 +
317 +Try to minimize specialization here
318 +
319 +1. Run `./boot-macOS-HS-provision.sh`
320 +2.
321 +
322 +From now on, we'll be running ./boot-macOS-HS-ephemeral.sh which will
323 +not write to mac_hdd.img. This means that the OS can update the disk
324 +and even persist data across reboots, however all changes go away when
325 +qemu restarts.
326 +
327 +You can now SSH to the host running `./boot-macOS-HS-ephemeral.sh` via
328 +`ssh -p 2200 nixos@10.5.3.153` for provisioning.
329 +
330 +
331 +---
332 +
333 +nixos module for running:
334 +
335 +
336 +activation-time import:
337 +
338 +1. create a zvol for the disk image based on the `import/hash-name` of the
339 + image file in the store
340 +
341 + [nix-shell:~]# zfs create -V $(qemu-img info ./OSX-KVM/mac-hdd-2-initial-setup.img --output=json | jq '."virtual-size"') rpool/imported-disk
342 +
343 +2. qemu-img dd the data from the `.qcow2` to the zvol
344 +
345 + qemu-img dd if=./OSX-KVM/mac-hdd-2-initial-setup.img -f qcow2 of=/dev/zvol/rpool/imported-disk CoC-O raw bs=250000000
346 +
347 + ^ takes ~5min
348 +
349 +3. snapshot zvol to `import/hash-name:import`
350 +
351 +[nix-shell:~]# zfs snapshot rpool/imported-disk@import
352 +
353 +[nix-shell:~]# zfs list -t snapshot
354 +NAME USED AVAIL REFER MOUNTPOINT
355 +rpool/imported-disk@import 0B - 14.5G -
356 +
357 +
358 +
359 +
360 +4. For each `import/*` see if their path is live, if not: delete the
361 + snapshot and zvol (via: `nix-store --query --roots /nix/store/hash-name`
362 +
363 +
364 +run-time code:
365 +
366 +pre-start: roll-back `execute/hash-name` to the snapshot for
367 + `import/hash-name:import`
368 + gene
369 +
370 + zfs rollback rpool/imported-disk@import
371 +
372 + start: execute qemu with the parameters like this:
373 +
374 +<disk type='block' device='disk'>
375 + <driver name='qemu' type='raw' cache='none'/>
376 + <source dev='/dev/zd0'/>
377 + <target dev='vda' bus='virtio'/>
378 + <alias name='virtio-disk0'/>
379 + <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
380 +</disk>
381 +
382 +
383 +sudo cp /Volumes/CONFIG/etc/ssh/ssh_host_* /etc/ssh/
384 +sudo chown root:root /etc/ssh/ssh_host_*_key
385 +sudo umount /Volumes/CONFIG