@cryptotaxi247 / infra / commits / 3893d599

Move channel generation to bastion

Channel redirects are now stored in S3. Fixes #72. Fixes #73.

Eelco Dolstra committed Jan 24, 2019 at 19:08 UTC 3893d5993c5214ec220be4b3ceab86c49b84eec6
5 files changed +23 -170
bastion/network.nix
+16
@@ -83,6 +83,12 @@ in
83 vpc = true;
84 };
85
86 + resources.ebsVolumes.scratch =
87 + { tags.Name = "Scratch space for the channel generator";
88 + inherit region zone accessKeyId;
89 + size = 64;
90 + };
91 +
92 bastion =
93 { config, pkgs, resources, ... }:
94
@@ -104,6 +110,7 @@ in
110 imports =
111 [ ../modules/common.nix
112 ../modules/tarball-mirror.nix
113 + ../modules/hydra-mirror.nix
114 ];
115
116 users.extraUsers.tarball-mirror.openssh.authorizedKeys.keys = [ sshKeys.eelco ];
@@ -130,5 +137,14 @@ in
137 ''
138 AcceptEnv AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY FASTLY_API_KEY
139 '';
140 +
141 + fileSystems."/scratch" =
142 + { autoFormat = true;
143 + fsType = "ext4";
144 + device = "/dev/xvdh";
145 + ec2.disk = resources.ebsVolumes.scratch;
146 + };
147 +
148 + systemd.tmpfiles.rules = [ "d /scratch/hydra-mirror 0755 hydra-mirror users 10d" ];
149 };
150 }
modules/common.nix
+1 -1
@@ -12,7 +12,7 @@ with lib;
12
13 nix.useSandbox = true;
14 nix.buildCores = 0;
15 - nix.nixPath = [ "nixpkgs=channel:nixos-18.03-small" ];
15 + nix.nixPath = [ "nixpkgs=channel:nixos-18.09-small" ];
16
17 environment.systemPackages =
18 [ pkgs.emacs
modules/hydra-mirror.nix renamed
+1 -1
@@ -26,6 +26,7 @@ let
26 exec mirror-nixos-branch ${channelName} https://hydra.nixos.org/job/${mainJob}/latest-finished
27 ''; # */
28 serviceConfig.User = "hydra-mirror";
29 + environment.TMPDIR = "/scratch/hydra-mirror";
30 };
31 };
32
@@ -55,5 +56,4 @@ in
56 (makeUpdateChannel "nixpkgs-17.09-darwin" "nixpkgs/nixpkgs-17.09-darwin/darwin-tested")
57 (makeUpdateChannel "nixpkgs-unstable" "nixpkgs/trunk/unstable")
58 ];
58 -
59 }
nixos-org/network.nix
+1 -168
@@ -13,13 +13,6 @@ in
13 size = 1024;
14 };
15
16 - # FIXME: remove
17 - resources.ebsVolumes.data =
18 - { tags.Name = "Misc. NixOS.org data";
19 - inherit region zone accessKeyId;
20 - size = 10;
21 - };
22 -
16 resources.ebsVolumes.data-new =
17 { tags.Name = "Misc. NixOS.org data";
18 inherit region zone accessKeyId;
@@ -35,166 +28,6 @@ in
28 { inherit region accessKeyId;
29 };
30
38 - resources.s3Buckets.nixpkgs-tarballs =
39 - { config, ... }:
40 - let
41 - allowUser = name: arn:
42 - [
43 - {
44 - Sid = "${name}AllowUpload";
45 - Effect = "Allow";
46 - Principal.AWS = arn;
47 - Action = [ "s3:PutObject" "s3:PutObjectAcl" ];
48 - Resource = [ "${config.arn}/*" ];
49 - }
50 - {
51 - Sid = "${name}AllowUpload2";
52 - Effect = "Allow";
53 - Principal.AWS = arn;
54 - Action = [ "s3:ListBucket" ];
55 - Resource = [ "${config.arn}" ];
56 - }
57 - ];
58 - in
59 - { inherit region accessKeyId;
60 - name = "nixpkgs-tarballs";
61 - # All files are readable but not listable.
62 - # The s3-upload-tarballs user can upload files.
63 - policy = builtins.toJSON
64 - { Version = "2008-10-17";
65 - Statement =
66 - [
67 - {
68 - Sid = "AllowPublicRead";
69 - Effect = "Allow";
70 - Principal.AWS = "*";
71 - Action = [ "s3:GetObject" ];
72 - Resource = [ "${config.arn}/*" ];
73 - }
74 - {
75 - Sid = "AllowUpload";
76 - Effect = "Allow";
77 - Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-tarballs";
78 - Action = [ "s3:PutObject" "s3:PutObjectAcl" ];
79 - Resource = [ "${config.arn}/*" ];
80 - }
81 - {
82 - Sid = "AllowUpload2";
83 - Effect = "Allow";
84 - Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-tarballs";
85 - Action = [ "s3:ListBucket" ];
86 - Resource = [ "${config.arn}" ];
87 - }
88 - ]
89 - ++ allowUser "Copumpkin" "arn:aws:iam::390897850978:root"
90 - ++ allowUser "Shlevy" "arn:aws:iam::976576280863:user/shlevy"
91 - ++ allowUser "Daiderd" "arn:aws:iam::014292808257:user/lnl7"
92 - ;
93 - };
94 - website.enabled = true;
95 - };
96 -
97 - resources.s3Buckets.nix-cache =
98 - { config, ... }:
99 - { inherit accessKeyId;
100 - region = "us-east-1";
101 - name = "nix-cache";
102 - policy = builtins.toJSON {
103 - Version = "2008-10-17";
104 - Statement =
105 - [
106 - { Sid = "AllowPublicRead";
107 - Effect = "Allow";
108 - Principal.AWS = "*";
109 - Action = [ "s3:GetObject" ];
110 - Resource = [ "${config.arn}/*" ];
111 - }
112 - { Sid = "AllowUploadDebuginfoWrite";
113 - Effect = "Allow";
114 - Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-releases";
115 - Action = [ "s3:PutObject" "s3:PutObjectAcl" ];
116 - Resource = [ "${config.arn}/debuginfo/*" ];
117 - }
118 - { Sid = "AllowUploadDebuginfoRead";
119 - Effect = "Allow";
120 - Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-releases";
121 - Action = [ "s3:GetObject" ];
122 - Resource = [ "${config.arn}/*" ];
123 - }
124 - { Sid = "AllowUploadDebuginfoRead2";
125 - Effect = "Allow";
126 - Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-releases";
127 - Action = [ "s3:ListBucket" "s3:GetBucketLocation" ];
128 - Resource = [ "${config.arn}" ];
129 - }
130 - ];
131 - };
132 - };
133 -
134 - /*
135 - resources.s3Buckets.nix-test-cache =
136 - { config, ... }:
137 - { inherit region accessKeyId;
138 - name = "nix-test-cache";
139 - policy =
140 - ''
141 - {
142 - "Version": "2008-10-17",
143 - "Statement": [
144 - {
145 - "Sid": "AllowPublicRead",
146 - "Effect": "Allow",
147 - "Principal": {"AWS": "*"},
148 - "Action": ["s3:GetObject"],
149 - "Resource": ["${config.arn}/*"]
150 - },
151 - {
152 - "Sid": "AllowPublicList",
153 - "Effect": "Allow",
154 - "Principal": {"AWS": "*"},
155 - "Action": ["s3:ListBucket"],
156 - "Resource": ["${config.arn}"]
157 - }
158 - ]
159 - }
160 - '';
161 - };
162 - */
163 -
164 - resources.s3Buckets.nix-releases =
165 - { config, ... }:
166 - { inherit accessKeyId;
167 - name = "nix-releases";
168 - region = "eu-west-1";
169 - policy = builtins.toJSON
170 - { Version = "2008-10-17";
171 - Statement =
172 - [
173 - {
174 - Sid = "AllowPublicRead";
175 - Effect = "Allow";
176 - Principal.AWS = "*";
177 - Action = [ "s3:GetObject" ];
178 - Resource = [ "${config.arn}/*" ];
179 - }
180 - {
181 - Sid = "AllowPublicList";
182 - Effect = "Allow";
183 - Principal.AWS = "*";
184 - Action = [ "s3:ListBucket" ];
185 - Resource = [ "${config.arn}" ];
186 - }
187 - {
188 - Sid = "AllowUpload";
189 - Effect = "Allow";
190 - Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-releases";
191 - Action = [ "s3:PutObject" "s3:PutObjectAcl" ];
192 - Resource = [ "${config.arn}/*" ];
193 - }
194 - ];
195 - };
196 - };
197 -
31 resources.vpc.nixos-org-vpc =
32 {
33 inherit region accessKeyId;
@@ -308,7 +141,7 @@ in
141
142 system.stateVersion = "17.09";
143
311 - imports = [ ./webserver.nix ./hydra-mirror.nix ];
144 + imports = [ ./webserver.nix ];
145 };
146
147 }
terraform/releases.tf
+4
@@ -1,6 +1,10 @@
1 resource "aws_s3_bucket" "releases" {
2 bucket = "nix-releases"
3
4 + website {
5 + index_document = "index.html"
6 + }
7 +
8 cors_rule {
9 allowed_headers = ["*"]
10 allowed_methods = ["HEAD", "GET"]