feat: X-Ipfs-Roots for smarter HTTP caches (#8720)
Marcin Rataj committed
Mar 1, 2022 at 18:04 UTC
caba3b264340d77f0848bd5362472822e95ea101
3 files changed
+204
core/corehttp/gateway_handler.go
+54
@@ -327,6 +327,13 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
327
w.Header().Set("X-IPFS-Path", urlPath)
328
w.Header().Set("Etag", responseEtag)
329
330
+ if rootCids, err := i.buildIpfsRootsHeader(urlPath, r); err == nil {
331
+ w.Header().Set("X-Ipfs-Roots", rootCids)
332
+ } else { // this should never happen, as we resolved the urlPath already
333
+ webError(w, "error while resolving X-Ipfs-Roots", err, http.StatusInternalServerError)
334
+ return
335
+ }
336
+
337
// set these headers _after_ the error, for we may just not have it
338
// and don't want the client to cache a 500 response...
339
// and only if it's /ipfs!
@@ -391,6 +398,9 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
398
internalWebError(w, files.ErrNotReader)
399
return
400
}
401
+ // static index.html → no need to generate dynamic dir-index-html
402
+ // replace mutable DirIndex Etag with immutable dir CID
403
+ w.Header().Set("Etag", `"`+resolvedPath.Cid().String()+`"`)
404
405
logger.Debugw("serving index.html file", "path", idxPath)
406
// write to request
@@ -785,6 +795,50 @@ func (i *gatewayHandler) addUserHeaders(w http.ResponseWriter) {
795
}
796
}
797
798
+// Set X-Ipfs-Roots with logical CID array for efficient HTTP cache invalidation.
799
+func (i *gatewayHandler) buildIpfsRootsHeader(contentPath string, r *http.Request) (string, error) {
800
+ /*
801
+ These are logical roots where each CID represent one path segment
802
+ and resolves to either a directory or the root block of a file.
803
+ The main purpose of this header is allow HTTP caches to do smarter decisions
804
+ around cache invalidation (eg. keep specific subdirectory/file if it did not change)
805
+
806
+ A good example is Wikipedia, which is HAMT-sharded, but we only care about
807
+ logical roots that represent each segment of the human-readable content
808
+ path:
809
+
810
+ Given contentPath = /ipns/en.wikipedia-on-ipfs.org/wiki/Block_of_Wikipedia_in_Turkey
811
+ rootCidList is a generated by doing `ipfs resolve -r` on each sub path:
812
+ /ipns/en.wikipedia-on-ipfs.org → bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze
813
+ /ipns/en.wikipedia-on-ipfs.org/wiki/ → bafybeihn2f7lhumh4grizksi2fl233cyszqadkn424ptjajfenykpsaiw4
814
+ /ipns/en.wikipedia-on-ipfs.org/wiki/Block_of_Wikipedia_in_Turkey → bafkreibn6euazfvoghepcm4efzqx5l3hieof2frhp254hio5y7n3hv5rma
815
+
816
+ The result is an ordered array of values:
817
+ X-Ipfs-Roots: bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze,bafybeihn2f7lhumh4grizksi2fl233cyszqadkn424ptjajfenykpsaiw4,bafkreibn6euazfvoghepcm4efzqx5l3hieof2frhp254hio5y7n3hv5rma
818
+
819
+ Note that while the top one will change every time any article is changed,
820
+ the last root (responsible for specific article) may not change at all.
821
+ */
822
+ var sp strings.Builder
823
+ var pathRoots []string
824
+ pathSegments := strings.Split(contentPath[6:], "/")
825
+ sp.WriteString(contentPath[:5]) // /ipfs or /ipns
826
+ for _, root := range pathSegments {
827
+ if root == "" {
828
+ continue
829
+ }
830
+ sp.WriteString("/")
831
+ sp.WriteString(root)
832
+ resolvedSubPath, err := i.api.ResolvePath(r.Context(), ipath.New(sp.String()))
833
+ if err != nil {
834
+ return "", err
835
+ }
836
+ pathRoots = append(pathRoots, resolvedSubPath.Cid().String())
837
+ }
838
+ rootCidList := strings.Join(pathRoots, ",") // convention from rfc2616#sec4.2
839
+ return rootCidList, nil
840
+}
841
+
842
func webError(w http.ResponseWriter, message string, err error, defaultCode int) {
843
if _, ok := err.(resolver.ErrNoLink); ok {
844
webErrorWithCode(w, message, err, http.StatusNotFound)
test/sharness/t0116-gateway-cache.sh
new
+150
@@ -0,0 +1,150 @@
1
+#!/usr/bin/env bash
2
+
3
+test_description="Test HTTP Gateway Cache Control Support"
4
+
5
+. lib/test-lib.sh
6
+
7
+test_init_ipfs
8
+test_launch_ipfs_daemon_without_network
9
+
10
+# Cache control support is based on logical roots (each path segment == one logical root).
11
+# To maximize the test surface, we want to test:
12
+# - /ipfs/ content path
13
+# - /ipns/ content path
14
+# - at least 3 levels
15
+# - separate tests for a directory listing and a file
16
+# - have implicit index.html for a good measure
17
+# /ipns/root1/root2/root3/ (/ipns/root1/root2/root3/index.html)
18
+
19
+# Note: we cover important edge case here:
20
+# ROOT3_CID - dir listing (dir-index-html response)
21
+# ROOT4_CID - index.html returned as a root response (dir/), instead of generated dir-index-html
22
+# FILE_CID - index.html returned directly, as a file
23
+
24
+test_expect_success "Add the test directory" '
25
+ mkdir -p root2/root3/root4 &&
26
+ echo "hello" > root2/root3/root4/index.html &&
27
+ ROOT1_CID=$(ipfs add -Qrw --cid-version 1 root2)
28
+ ROOT2_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2 | cut -d "/" -f3)
29
+ ROOT3_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3 | cut -d "/" -f3)
30
+ ROOT4_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3/root4 | cut -d "/" -f3)
31
+ FILE_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3/root4/index.html | cut -d "/" -f3)
32
+'
33
+
34
+test_expect_success "Prepare IPNS unixfs content path for testing" '
35
+ TEST_IPNS_ID=$(ipfs key gen --ipns-base=base36 --type=ed25519 cache_test_key | head -n1 | tr -d "\n")
36
+ ipfs name publish --key cache_test_key --allow-offline -Q "/ipfs/$ROOT1_CID" > name_publish_out &&
37
+ test_check_peerid "${TEST_IPNS_ID}" &&
38
+ ipfs name resolve "${TEST_IPNS_ID}" > output &&
39
+ printf "/ipfs/%s\n" "$ROOT1_CID" > expected &&
40
+ test_cmp expected output
41
+'
42
+
43
+# GET /ipfs/
44
+ test_expect_success "GET for /ipfs/ unixfs dir listing succeeds" '
45
+ curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/" >/dev/null 2>curl_ipfs_dir_listing_output &&
46
+ cat curl_ipfs_dir_listing_output
47
+ '
48
+ test_expect_success "GET for /ipfs/ unixfs dir with index.html succeeds" '
49
+ curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/" >/dev/null 2>curl_ipfs_dir_index.html_output &&
50
+ cat curl_ipfs_dir_index.html_output
51
+ '
52
+ test_expect_success "GET for /ipfs/ unixfs file succeeds" '
53
+ curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/index.html" >/dev/null 2>curl_ipfs_file_output &&
54
+ cat curl_ipfs_file_output
55
+ '
56
+# GET /ipns/
57
+ test_expect_success "GET for /ipns/ unixfs dir listing succeeds" '
58
+ curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/" >/dev/null 2>curl_ipns_dir_listing_output &&
59
+ cat curl_ipns_dir_listing_output
60
+ '
61
+ test_expect_success "GET for /ipns/ unixfs dir with index.html succeeds" '
62
+ curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/" >/dev/null 2>curl_ipns_dir_index.html_output &&
63
+ cat curl_ipns_dir_index.html_output
64
+ '
65
+ test_expect_success "GET for /ipns/ unixfs file succeeds" '
66
+ curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" >/dev/null 2>curl_ipns_file_output &&
67
+ cat curl_ipns_file_output
68
+ '
69
+
70
+# X-Ipfs-Path
71
+
72
+ ## dir generated listing
73
+ test_expect_success "GET /ipfs/ dir listing response has original content path in X-Ipfs-Path" '
74
+ grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3" curl_ipfs_dir_listing_output
75
+ '
76
+ test_expect_success "GET /ipns/ dir listing response has original content path in X-Ipfs-Path" '
77
+ grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3" curl_ipns_dir_listing_output
78
+ '
79
+
80
+ ## dir static index.html
81
+ test_expect_success "GET /ipfs/ dir index.html response has original content path in X-Ipfs-Path" '
82
+ grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/" curl_ipfs_dir_index.html_output
83
+ '
84
+ test_expect_success "GET /ipns/ dir index.html response has original content path in X-Ipfs-Path" '
85
+ grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/" curl_ipns_dir_index.html_output
86
+ '
87
+
88
+ # file
89
+ test_expect_success "GET /ipfs/ file response has original content path in X-Ipfs-Path" '
90
+ grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/index.html" curl_ipfs_file_output
91
+ '
92
+ test_expect_success "GET /ipns/ file response has original content path in X-Ipfs-Path" '
93
+ grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" curl_ipns_file_output
94
+ '
95
+
96
+# X-Ipfs-Roots
97
+
98
+ ## dir generated listing
99
+ test_expect_success "GET /ipfs/ dir listing response has logical CID roots in X-Ipfs-Roots" '
100
+ grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipfs_dir_listing_output
101
+ '
102
+ test_expect_success "GET /ipns/ dir listing response has logical CID roots in X-Ipfs-Roots" '
103
+ grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipns_dir_listing_output
104
+ '
105
+
106
+ ## dir static index.html
107
+ test_expect_success "GET /ipfs/ dir index.html response has logical CID roots in X-Ipfs-Roots" '
108
+ grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipfs_dir_index.html_output
109
+ '
110
+ test_expect_success "GET /ipns/ dir index.html response has logical CID roots in X-Ipfs-Roots" '
111
+ grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipns_dir_index.html_output
112
+ '
113
+
114
+ ## file
115
+ test_expect_success "GET /ipfs/ file response has logical CID roots in X-Ipfs-Roots" '
116
+ grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipfs_file_output
117
+ '
118
+ test_expect_success "GET /ipns/ file response has logical CID roots in X-Ipfs-Roots" '
119
+ grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipns_file_output
120
+ '
121
+
122
+# Etag
123
+
124
+ ## dir generated listing
125
+ test_expect_success "GET /ipfs/ dir response has special Etag for generated dir listing" '
126
+ grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipfs_dir_listing_output
127
+ '
128
+ test_expect_success "GET /ipns/ dir response has special Etag for generated dir listing" '
129
+ grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipns_dir_listing_output
130
+ '
131
+
132
+ ## dir static index.html should use CID of the index.html file for improved HTTP caching
133
+ test_expect_success "GET /ipfs/ dir index.html response has dir CID as Etag" '
134
+ grep "< Etag: \"${ROOT4_CID}\"" curl_ipfs_dir_index.html_output
135
+ '
136
+ test_expect_success "GET /ipns/ dir index.html response has dir CID as Etag" '
137
+ grep "< Etag: \"${ROOT4_CID}\"" curl_ipns_dir_index.html_output
138
+ '
139
+
140
+ ## file
141
+ test_expect_success "GET /ipfs/ response has CID as Etag for a file" '
142
+ grep "< Etag: \"${FILE_CID}\"" curl_ipfs_file_output
143
+ '
144
+ test_expect_success "GET /ipns/ response has CID as Etag for a file" '
145
+ grep "< Etag: \"${FILE_CID}\"" curl_ipns_file_output
146
+ '
147
+
148
+test_kill_ipfs_daemon
149
+
150
+test_done