refactor(ci): libp2p conformance is separate job
this ensures the libp2p experiment runs independently and its failure does not impact the result of job that tests stable features on http port
Marcin Rataj committed
Sep 6, 2023 at 01:33 UTC
1efd9d47aa6ff05072bdfc9115a02797a4d50e40
1 file changed
+83
-72
.github/workflows/gateway-conformance.yml
+83
-72
@@ -17,7 +17,24 @@ defaults:
17
run:
18
shell: bash
19
20
+env:
21
+ # hostnames expected by https://github.com/ipfs/gateway-conformance
22
+ GATEWAY_PUBLIC_GATEWAYS: |
23
+ {
24
+ "example.com": {
25
+ "UseSubdomains": true,
26
+ "InlineDNSLink": true,
27
+ "Paths": ["/ipfs", "/ipns"]
28
+ },
29
+ "localhost": {
30
+ "UseSubdomains": true,
31
+ "InlineDNSLink": true,
32
+ "Paths": ["/ipfs", "/ipns"]
33
+ }
34
+ }
35
+
36
jobs:
37
+ # Testing all gateway features via TCP port specified in Addresses.Gateway
38
gateway-conformance:
39
runs-on: ubuntu-latest
40
timeout-minutes: 10
@@ -30,11 +47,14 @@ jobs:
47
48
# 2. Build the kubo-gateway
49
- name: Setup Go
33
- uses: actions/setup-go@v3
50
+ uses: actions/setup-go@v4
51
with:
52
go-version: 1.20.x
53
+ - uses: protocol/cache-go-action@v1
54
+ with:
55
+ name: ${{ github.job }}
56
- name: Checkout kubo-gateway
37
- uses: actions/checkout@v3
57
+ uses: actions/checkout@v4
58
with:
59
path: kubo-gateway
60
- name: Build kubo-gateway
@@ -43,26 +63,9 @@ jobs:
63
64
# 3. Init the kubo-gateway
65
- name: Init kubo-gateway
46
- env:
47
- GATEWAY_PUBLIC_GATEWAYS: |
48
- {
49
- "example.com": {
50
- "UseSubdomains": true,
51
- "InlineDNSLink": true,
52
- "Paths": ["/ipfs", "/ipns"]
53
- },
54
- "localhost": {
55
- "UseSubdomains": true,
56
- "InlineDNSLink": true,
57
- "Paths": ["/ipfs", "/ipns"]
58
- }
59
- }
66
run: |
61
- ./ipfs init --profile=test
67
+ ./ipfs init -e
68
./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
63
- ./ipfs config --json Experimental.GatewayOverLibp2p true
64
- ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080"
65
- ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
69
working-directory: kubo-gateway/cmd/ipfs
70
71
# 4. Populate the Kubo gateway with the gateway-conformance fixtures
@@ -88,28 +91,7 @@ jobs:
91
# 5. Start the kubo-gateway
92
- name: Start kubo-gateway
93
run: |
91
- ./ipfs daemon &
92
- endpoint="http://127.0.0.1:5001/api/v0/version"
93
- max_retries=5
94
- retry_interval=3
95
-
96
- check_endpoint() {
97
- curl -X POST --silent --fail "$endpoint" > /dev/null
98
- return $?
99
- }
100
-
101
- retries=0
102
- while ! check_endpoint; do
103
- retries=$((retries+1))
104
-
105
- if [ $retries -ge $max_retries ]; then
106
- echo "daemon took too long to start"
107
- exit 1
108
- fi
109
-
110
- sleep $retry_interval
111
- done
112
- echo "daemon started and ready to receive API calls"
94
+ ./ipfs daemon --offline &
95
working-directory: kubo-gateway/cmd/ipfs
96
97
# 6. Run the gateway-conformance tests
@@ -140,48 +122,77 @@ jobs:
122
name: gateway-conformance.json
123
path: output.json
124
143
- # 8. Setup a kubo http-p2p-proxy to run gateway conformance tests
125
+ # Testing trustless gateway feature subset exposed as libp2p protocol
126
+ gateway-conformance-libp2p-experiment:
127
+ runs-on: ubuntu-latest
128
+ timeout-minutes: 10
129
+ steps:
130
+ # 1. Download the gateway-conformance fixtures
131
+ - name: Download gateway-conformance fixtures
132
+ uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.3
133
+ with:
134
+ output: fixtures
135
+
136
+ # 2. Build the kubo-gateway
137
+ - name: Setup Go
138
+ uses: actions/setup-go@v4
139
+ with:
140
+ go-version: 1.20.x
141
+ - uses: protocol/cache-go-action@v1
142
+ with:
143
+ name: ${{ github.job }}
144
+ - name: Checkout kubo-gateway
145
+ uses: actions/checkout@v3
146
+ with:
147
+ path: kubo-gateway
148
+ - name: Build kubo-gateway
149
+ run: make build
150
+ working-directory: kubo-gateway
151
+
152
+ # 3. Init the kubo-gateway
153
+ - name: Init kubo-gateway
154
+ run: |
155
+ ./ipfs init --profile=test
156
+ ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
157
+ ./ipfs config --json Experimental.GatewayOverLibp2p true
158
+ ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080"
159
+ ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
160
+ working-directory: kubo-gateway/cmd/ipfs
161
+
162
+ # 4. Populate the Kubo gateway with the gateway-conformance fixtures
163
+ - name: Import fixtures
164
+ run: |
165
+ # Import car files
166
+ find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \;
167
+
168
+ # 5. Start the kubo-gateway
169
+ - name: Start kubo-gateway
170
+ run: |
171
+ ( ./ipfs daemon & ) | sed '/Daemon is ready/q'
172
+ while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done
173
+ working-directory: kubo-gateway/cmd/ipfs
174
+
175
+ # 6. Setup a kubo http-p2p-proxy to expose libp2p protocol as a regular HTTP port for gateway conformance tests
176
- name: Init p2p-proxy kubo node
177
env:
178
IPFS_PATH: "~/.kubo-p2p-proxy"
179
run: |
148
- ./ipfs init --profile=test
180
+ ./ipfs init --profile=test -e
181
./ipfs config --json Experimental.Libp2pStreamMounting true
182
./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081"
183
./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002"
184
working-directory: kubo-gateway/cmd/ipfs
185
154
- # 9. Start the kubo http-p2p-proxy
186
+ # 7. Start the kubo http-p2p-proxy
187
- name: Start kubo http-p2p-proxy
188
env:
189
IPFS_PATH: "~/.kubo-p2p-proxy"
190
run: |
159
- ./ipfs daemon &
160
-
161
- endpoint="http://127.0.0.1:5002/api/v0/version"
162
- max_retries=5
163
- retry_interval=3
164
-
165
- check_endpoint() {
166
- curl -X POST --silent --fail "$endpoint" > /dev/null
167
- return $?
168
- }
169
-
170
- retries=0
171
- while ! check_endpoint; do
172
- retries=$((retries+1))
173
-
174
- if [ $retries -ge $max_retries ]; then
175
- echo "daemon took too long to start"
176
- exit 1
177
- fi
178
-
179
- sleep $retry_interval
180
- done
181
- echo "daemon started and ready to receive API calls"
191
+ ( ./ipfs daemon & ) | sed '/Daemon is ready/q'
192
+ while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done
193
working-directory: kubo-gateway/cmd/ipfs
194
184
- # 10. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p
195
+ # 8. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p
196
- name: Start http-over-libp2p forwarding proxy
197
run: |
198
gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="<id>")
@@ -189,7 +200,7 @@ jobs:
200
./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8082 /p2p/$gatewayNodeId
201
working-directory: kubo-gateway/cmd/ipfs
202
192
- # 11. Run the gateway-conformance tests over libp2p
203
+ # 9. Run the gateway-conformance tests over libp2p
204
- name: Run gateway-conformance tests over libp2p
205
uses: ipfs/gateway-conformance/.github/actions/test@v0.3
206
with:
@@ -200,7 +211,7 @@ jobs:
211
markdown: output.md
212
args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
213
203
- # 11. Upload the results
214
+ # 10. Upload the results
215
- name: Upload MD summary
216
if: failure() || success()
217
run: cat output.md >> $GITHUB_STEP_SUMMARY
@@ -215,4 +226,4 @@ jobs:
226
uses: actions/upload-artifact@v3
227
with:
228
name: gateway-conformance-libp2p.json
218
- path: output.json
\ No newline at end of file
229
+ path: output.json