| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "fmt" |
| 6 | "io" |
| 7 | "log" |
| 8 | "net/http" |
| 9 | "net/url" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | |
| 15 | "github.com/ipfs/go-cid" |
| 16 | "github.com/ipfs/kubo/test/cli/harness" |
| 17 | carstore "github.com/ipld/go-car/v2/blockstore" |
| 18 | "github.com/libp2p/go-libp2p" |
| 19 | "github.com/libp2p/go-libp2p/core/peer" |
| 20 | libp2phttp "github.com/libp2p/go-libp2p/p2p/http" |
| 21 | "github.com/stretchr/testify/assert" |
| 22 | "github.com/stretchr/testify/require" |
| 23 | ) |
| 24 | |
| 25 | func TestContentBlocking(t *testing.T) { |
| 26 | // NOTE: we can't run this with t.Parallel() because we set IPFS_NS_MAP |
| 27 | // and running in parallel could impact other tests |
| 28 | |
| 29 | const blockedMsg = "blocked and cannot be provided" |
| 30 | const statusExpl = "specific HTTP error code is expected" |
| 31 | const bodyExpl = "Error message informing about content block is expected" |
| 32 | |
| 33 | h := harness.NewT(t) |
| 34 | |
| 35 | // Init IPFS_PATH |
| 36 | node := h.NewNode().Init("--empty-repo", "--profile=test") |
| 37 | |
| 38 | // Create CIDs we use in test |
| 39 | h.WriteFile("parent-dir/blocked-subdir/indirectly-blocked-file.txt", "indirectly blocked file content") |
| 40 | allowedParentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", "--pin=false", filepath.Join(h.Dir, "parent-dir")).Stdout.Trimmed() |
| 41 | blockedSubDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", "--pin=false", filepath.Join(h.Dir, "parent-dir", "blocked-subdir")).Stdout.Trimmed() |
| 42 | node.IPFS("block", "rm", blockedSubDirCID) |
| 43 | |
| 44 | h.WriteFile("directly-blocked-file.txt", "directly blocked file content") |
| 45 | blockedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() |
| 46 | |
| 47 | h.WriteFile("not-blocked-file.txt", "not blocked file content") |
| 48 | allowedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "not-blocked-file.txt")).Stdout.Trimmed() |
| 49 | |
| 50 | // Create denylist at $IPFS_PATH/denylists/test.deny |
| 51 | denylistTmp := h.WriteToTemp("name: test list\n---\n" + |
| 52 | "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n" + // Double hash (sha256) CID block: base58btc(sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR)) |
| 53 | "//gW813G35CnLsy7gRYYHuf63hrz71U1xoLFDVeV7actx6oX\n" + // Double hash (blake3) Path block under blake3 root CID: base58btc(blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path)) |
| 54 | "//8526ba05eec55e28f8db5974cc891d0d92c8af69d386fc6464f1e9f372caf549\n" + // Legacy CID double-hash block: sha256(bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq/) |
| 55 | "//e5b7d2ce2594e2e09901596d8e1f29fa249b74c8c9e32ea01eda5111e4d33f07\n" + // Legacy Path double-hash block: sha256(bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath) |
| 56 | "/ipfs/" + blockedCID + "\n" + // block specific CID |
| 57 | "/ipfs/" + allowedParentDirCID + "/blocked-subdir*\n" + // block only specific subpath |
| 58 | "/ipns/blocked-cid.example.com\n" + |
| 59 | "/ipns/blocked-dnslink.example.com\n") |
| 60 | |
| 61 | if err := os.MkdirAll(filepath.Join(node.Dir, "denylists"), 0o777); err != nil { |
| 62 | log.Panicf("failed to create denylists dir: %s", err.Error()) |
| 63 | } |
| 64 | if err := os.Rename(denylistTmp, filepath.Join(node.Dir, "denylists", "test.deny")); err != nil { |
| 65 | log.Panicf("failed to create test denylist: %s", err.Error()) |
| 66 | } |
| 67 | |
| 68 | // Add two entries to namesys resolution cache |
| 69 | // /ipns/blocked-cid.example.com point at a blocked CID (to confirm blocking impacts /ipns resolution) |
| 70 | // /ipns/blocked-dnslink.example.com with safe CID (to test blocking of /ipns/ paths) |
| 71 | os.Setenv("IPFS_NS_MAP", "blocked-cid.example.com:/ipfs/"+blockedCID+",blocked-dnslink.example.com/ipns/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") |
| 72 | defer os.Unsetenv("IPFS_NS_MAP") |
| 73 | |
| 74 | // Enable GatewayOverLibp2p as we want to test denylist there too |
| 75 | node.IPFS("config", "--json", "Experimental.GatewayOverLibp2p", "true") |
| 76 | |
| 77 | // Start daemon, it should pick up denylist from $IPFS_PATH/denylists/test.deny |
| 78 | node.StartDaemon() // we need online mode for GatewayOverLibp2p tests |
| 79 | t.Cleanup(func() { node.StopDaemon() }) |
| 80 | client := node.GatewayClient() |
| 81 | |
| 82 | // First, confirm gateway works |
| 83 | t.Run("Gateway Allows CID that is not blocked", func(t *testing.T) { |
| 84 | t.Parallel() |
| 85 | resp := client.Get("/ipfs/" + allowedCID) |
| 86 | assert.Equal(t, http.StatusOK, resp.StatusCode) |
| 87 | assert.Equal(t, "not blocked file content", resp.Body) |
| 88 | }) |
| 89 | |
| 90 | // Then, does the most basic blocking case work? |
| 91 | t.Run("Gateway Denies directly blocked CID", func(t *testing.T) { |
| 92 | t.Parallel() |
| 93 | resp := client.Get("/ipfs/" + blockedCID) |
| 94 | assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) |
| 95 | assert.NotEqual(t, "directly blocked file content", resp.Body) |
| 96 | assert.Contains(t, resp.Body, blockedMsg, bodyExpl) |
| 97 | }) |
| 98 | |
| 99 | // Confirm parent of blocked subpath is not blocked |
| 100 | t.Run("Gateway Allows parent Path that is not blocked", func(t *testing.T) { |
| 101 | t.Parallel() |
| 102 | resp := client.Get("/ipfs/" + allowedParentDirCID) |
| 103 | assert.Equal(t, http.StatusOK, resp.StatusCode) |
| 104 | }) |
| 105 | |
| 106 | // Confirm CAR responses skip blocked subpaths |
| 107 | t.Run("Gateway returns CAR without blocked subpath", func(t *testing.T) { |
| 108 | resp := client.Get("/ipfs/" + allowedParentDirCID + "/subdir?format=car") |
| 109 | assert.Equal(t, http.StatusOK, resp.StatusCode) |
| 110 | |
| 111 | bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil) |
| 112 | assert.NoError(t, err) |
| 113 | |
| 114 | has, err := bs.Has(context.Background(), cid.MustParse(blockedSubDirCID)) |
| 115 | assert.NoError(t, err) |
| 116 | assert.False(t, has) |
| 117 | }) |
| 118 | |
| 119 | /* TODO: this was already broken in 0.26, but we should fix it |
| 120 | t.Run("Gateway returns CAR without directly blocked CID", func(t *testing.T) { |
| 121 | allowedDirWithDirectlyBlockedCID := node.IPFS("add", "--raw-leaves", "-Q", "-rw", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() |
| 122 | resp := client.Get("/ipfs/" + allowedDirWithDirectlyBlockedCID + "?format=car") |
| 123 | assert.Equal(t, http.StatusOK, resp.StatusCode) |
| 124 | |
| 125 | bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil) |
| 126 | assert.NoError(t, err) |
| 127 | |
| 128 | has, err := bs.Has(context.Background(), cid.MustParse(blockedCID)) |
| 129 | assert.NoError(t, err) |
| 130 | assert.False(t, has, "Returned CAR should not include blockedCID") |
| 131 | }) |
| 132 | */ |
| 133 | |
| 134 | // Confirm CAR responses skip blocked subpaths |
| 135 | t.Run("Gateway returns CAR without blocked subpath", func(t *testing.T) { |
| 136 | resp := client.Get("/ipfs/" + allowedParentDirCID + "/subdir?format=car") |
| 137 | assert.Equal(t, http.StatusOK, resp.StatusCode) |
| 138 | |
| 139 | bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil) |
| 140 | assert.NoError(t, err) |
| 141 | |
| 142 | has, err := bs.Has(context.Background(), cid.MustParse(blockedSubDirCID)) |
| 143 | assert.NoError(t, err) |
| 144 | assert.False(t, has, "Returned CAR should not include blockedSubDirCID") |
| 145 | }) |
| 146 | |
| 147 | // Ok, now the full list of test cases we want to cover in both CLI and Gateway |
| 148 | testCases := []struct { |
| 149 | name string |
| 150 | path string |
| 151 | }{ |
| 152 | { |
| 153 | name: "directly blocked file CID", |
| 154 | path: "/ipfs/" + blockedCID, |
| 155 | }, |
| 156 | { |
| 157 | name: "indirectly blocked file (on a blocked subpath)", |
| 158 | path: "/ipfs/" + allowedParentDirCID + "/blocked-subdir/indirectly-blocked-file.txt", |
| 159 | }, |
| 160 | { |
| 161 | name: "/ipns path that resolves to a blocked CID", |
| 162 | path: "/ipns/blocked-cid.example.com", |
| 163 | }, |
| 164 | { |
| 165 | name: "/ipns Path that is blocked by DNSLink name", |
| 166 | path: "/ipns/blocked-dnslink.example.com", |
| 167 | }, |
| 168 | { |
| 169 | name: "double-hash CID block (sha256-multihash)", |
| 170 | path: "/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR", |
| 171 | }, |
| 172 | { |
| 173 | name: "double-hash Path block (blake3-multihash)", |
| 174 | path: "/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path", |
| 175 | }, |
| 176 | { |
| 177 | name: "legacy CID double-hash block (sha256)", |
| 178 | path: "/ipfs/bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq", |
| 179 | }, |
| 180 | |
| 181 | { |
| 182 | name: "legacy Path double-hash block (sha256)", |
| 183 | path: "/ipfs/bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath", |
| 184 | }, |
| 185 | } |
| 186 | |
| 187 | // Which specific cliCmds we test against testCases |
| 188 | cliCmds := [][]string{ |
| 189 | {"block", "get"}, |
| 190 | {"block", "stat"}, |
| 191 | {"dag", "get"}, |
| 192 | {"dag", "export"}, |
| 193 | {"dag", "stat"}, |
| 194 | {"cat"}, |
| 195 | {"ls"}, |
| 196 | {"get"}, |
| 197 | {"refs"}, |
| 198 | } |
| 199 | |
| 200 | expectedMsg := blockedMsg |
| 201 | for _, testCase := range testCases { |
| 202 | |
| 203 | // Confirm that denylist is active for every command in 'cliCmds' x 'testCases' |
| 204 | for _, cmd := range cliCmds { |
| 205 | cliTestName := fmt.Sprintf("CLI '%s' denies %s", strings.Join(cmd, " "), testCase.name) |
| 206 | t.Run(cliTestName, func(t *testing.T) { |
| 207 | t.Parallel() |
| 208 | args := append(cmd, testCase.path) |
| 209 | cmd := node.RunIPFS(args...) |
| 210 | stdout := cmd.Stdout.Trimmed() |
| 211 | stderr := cmd.Stderr.Trimmed() |
| 212 | if !strings.Contains(stderr, expectedMsg) { |
| 213 | t.Errorf("Expected STDERR error message %q, but got: %q", expectedMsg, stderr) |
| 214 | if stdout != "" { |
| 215 | t.Errorf("Expected STDOUT to be empty, but got: %q", stdout) |
| 216 | } |
| 217 | } |
| 218 | }) |
| 219 | } |
| 220 | |
| 221 | // Confirm that denylist is active for every content path in 'testCases' |
| 222 | gwTestName := fmt.Sprintf("Gateway denies %s", testCase.name) |
| 223 | t.Run(gwTestName, func(t *testing.T) { |
| 224 | resp := client.Get(testCase.path) |
| 225 | assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) |
| 226 | assert.Contains(t, resp.Body, blockedMsg, bodyExpl) |
| 227 | }) |
| 228 | |
| 229 | } |
| 230 | |
| 231 | // Extra edge cases on subdomain gateway |
| 232 | |
| 233 | t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain redirect)", func(t *testing.T) { |
| 234 | t.Parallel() |
| 235 | |
| 236 | gwURL, _ := url.Parse(node.GatewayURL()) |
| 237 | resp := client.Get("/ipns/blocked-dnslink.example.com", func(r *http.Request) { |
| 238 | r.Host = "localhost:" + gwURL.Port() |
| 239 | }) |
| 240 | |
| 241 | assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) |
| 242 | assert.Contains(t, resp.Body, blockedMsg, bodyExpl) |
| 243 | }) |
| 244 | |
| 245 | t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, no TLS)", func(t *testing.T) { |
| 246 | t.Parallel() |
| 247 | |
| 248 | gwURL, _ := url.Parse(node.GatewayURL()) |
| 249 | resp := client.Get("/", func(r *http.Request) { |
| 250 | r.Host = "blocked-dnslink.example.com.ipns.localhost:" + gwURL.Port() |
| 251 | }) |
| 252 | |
| 253 | assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) |
| 254 | assert.Contains(t, resp.Body, blockedMsg, bodyExpl) |
| 255 | }) |
| 256 | |
| 257 | t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, inlined for TLS)", func(t *testing.T) { |
| 258 | t.Parallel() |
| 259 | |
| 260 | gwURL, _ := url.Parse(node.GatewayURL()) |
| 261 | resp := client.Get("/", func(r *http.Request) { |
| 262 | // Inlined DNSLink to fit in single DNS label for TLS interop: |
| 263 | // https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header |
| 264 | r.Host = "blocked--dnslink-example-com.ipns.localhost:" + gwURL.Port() |
| 265 | }) |
| 266 | |
| 267 | assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) |
| 268 | assert.Contains(t, resp.Body, blockedMsg, bodyExpl) |
| 269 | }) |
| 270 | |
| 271 | // We need to confirm denylist is active when gateway is run in NoFetch |
| 272 | // mode (which usually swaps blockservice to a read-only one, and that swap |
| 273 | // may cause denylists to not be applied, as it is a separate code path) |
| 274 | t.Run("GatewayNoFetch", func(t *testing.T) { |
| 275 | // NOTE: we don't run this in parallel, as it requires restart with different config |
| 276 | |
| 277 | // Switch gateway to NoFetch mode |
| 278 | node.StopDaemon() |
| 279 | node.IPFS("config", "--json", "Gateway.NoFetch", "true") |
| 280 | node.StartDaemon() |
| 281 | |
| 282 | // update client, as the port of test node might've changed after restart |
| 283 | client = node.GatewayClient() |
| 284 | |
| 285 | // First, confirm gateway works |
| 286 | t.Run("Allows CID that is not blocked", func(t *testing.T) { |
| 287 | resp := client.Get("/ipfs/" + allowedCID) |
| 288 | assert.Equal(t, http.StatusOK, resp.StatusCode) |
| 289 | assert.Equal(t, "not blocked file content", resp.Body) |
| 290 | }) |
| 291 | |
| 292 | // Then, does the most basic blocking case work? |
| 293 | t.Run("Denies directly blocked CID", func(t *testing.T) { |
| 294 | resp := client.Get("/ipfs/" + blockedCID) |
| 295 | assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) |
| 296 | assert.NotEqual(t, "directly blocked file content", resp.Body) |
| 297 | assert.Contains(t, resp.Body, blockedMsg, bodyExpl) |
| 298 | }) |
| 299 | |
| 300 | // Restore default |
| 301 | node.StopDaemon() |
| 302 | node.IPFS("config", "--json", "Gateway.NoFetch", "false") |
| 303 | node.StartDaemon() |
| 304 | client = node.GatewayClient() |
| 305 | }) |
| 306 | |
| 307 | // We need to confirm denylist is active on the |
| 308 | // trustless gateway exposed over libp2p |
| 309 | // when Experimental.GatewayOverLibp2p=true |
| 310 | // (https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p) |
| 311 | // NOTE: this type of gateway is hardcoded to be NoFetch: it does not fetch |
| 312 | // data that is not in local store, so we only need to run it once: a |
| 313 | // simple smoke-test for allowed CID and blockedCID. |
| 314 | t.Run("GatewayOverLibp2p", func(t *testing.T) { |
| 315 | t.Parallel() |
| 316 | |
| 317 | // Create libp2p client that connects to our node over |
| 318 | // /http1.1 and then talks gateway semantics over the /ipfs/gateway sub-protocol |
| 319 | clientHost, err := libp2p.New(libp2p.NoListenAddrs) |
| 320 | require.NoError(t, err) |
| 321 | err = clientHost.Connect(context.Background(), peer.AddrInfo{ |
| 322 | ID: node.PeerID(), |
| 323 | Addrs: node.SwarmAddrs(), |
| 324 | }) |
| 325 | require.NoError(t, err) |
| 326 | |
| 327 | libp2pClient, err := (&libp2phttp.Host{StreamHost: clientHost}).NamespacedClient("/ipfs/gateway", peer.AddrInfo{ID: node.PeerID()}) |
| 328 | require.NoError(t, err) |
| 329 | |
| 330 | t.Run("Serves Allowed CID", func(t *testing.T) { |
| 331 | t.Parallel() |
| 332 | resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", allowedCID)) |
| 333 | require.NoError(t, err) |
| 334 | defer resp.Body.Close() |
| 335 | assert.Equal(t, http.StatusOK, resp.StatusCode) |
| 336 | body, err := io.ReadAll(resp.Body) |
| 337 | require.NoError(t, err) |
| 338 | require.Equal(t, string(body), "not blocked file content", bodyExpl) |
| 339 | }) |
| 340 | |
| 341 | t.Run("Denies Blocked CID", func(t *testing.T) { |
| 342 | t.Parallel() |
| 343 | resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", blockedCID)) |
| 344 | require.NoError(t, err) |
| 345 | defer resp.Body.Close() |
| 346 | assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) |
| 347 | body, err := io.ReadAll(resp.Body) |
| 348 | require.NoError(t, err) |
| 349 | assert.NotEqual(t, string(body), "directly blocked file content") |
| 350 | assert.Contains(t, string(body), blockedMsg, bodyExpl) |
| 351 | }) |
| 352 | |
| 353 | t.Run("Denies Blocked CID as CAR", func(t *testing.T) { |
| 354 | t.Parallel() |
| 355 | resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=car", blockedCID)) |
| 356 | require.NoError(t, err) |
| 357 | defer resp.Body.Close() |
| 358 | assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) |
| 359 | body, err := io.ReadAll(resp.Body) |
| 360 | require.NoError(t, err) |
| 361 | assert.NotContains(t, string(body), "directly blocked file content") |
| 362 | assert.Contains(t, string(body), blockedMsg, bodyExpl) |
| 363 | }) |
| 364 | }) |
| 365 | } |