add pinning support to the urlstore
fixes #5833 License: MIT Signed-off-by: Steven Allen <steven@stebalien.com>
Steven Allen committed
Dec 10, 2018 at 18:30 UTC
25a0db4f692ab5501879fbc0a6129430f9a18c0e
2 files changed
+21
-9
core/commands/urlstore.go
+18
-4
@@ -7,6 +7,7 @@ import (
7
8
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
9
filestore "github.com/ipfs/go-ipfs/filestore"
10
+ pin "github.com/ipfs/go-ipfs/pin"
11
12
chunk "gx/ipfs/QmR4QQVkBZsZENRjYFVi8dEtPL3daZRNKk24m4r6WKJHNm/go-ipfs-chunker"
13
cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
@@ -36,9 +37,6 @@ control.
37
The file is added using raw-leaves but otherwise using the default
38
settings for 'ipfs add'.
39
39
-The file is not pinned, so this command should be followed by an 'ipfs
40
-pin add'.
41
-
40
This command is considered temporary until a better solution can be
41
found. It may disappear or the semantics can change at any
42
time.
@@ -46,6 +44,7 @@ time.
44
},
45
Options: []cmdkit.Option{
46
cmdkit.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
47
+ cmdkit.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true),
48
},
49
Arguments: []cmdkit.Argument{
50
cmdkit.StringArg("url", true, false, "URL to add to IPFS"),
@@ -73,6 +72,7 @@ time.
72
}
73
74
useTrickledag, _ := req.Options[trickleOptionName].(bool)
75
+ dopin, _ := req.Options[pinOptionName].(bool)
76
77
hreq, err := http.NewRequest("GET", url, nil)
78
if err != nil {
@@ -87,6 +87,11 @@ time.
87
return fmt.Errorf("expected code 200, got: %d", hres.StatusCode)
88
}
89
90
+ if dopin {
91
+ // Take the pinlock
92
+ defer n.Blockstore.PinLock().Unlock()
93
+ }
94
+
95
chk := chunk.NewSizeSplitter(hres.Body, chunk.DefaultBlockSize)
96
prefix := cid.NewPrefixV1(cid.DagProtobuf, mh.SHA2_256)
97
dbp := &ihelper.DagBuilderParams{
@@ -102,13 +107,22 @@ time.
107
if useTrickledag {
108
layout = trickle.Layout
109
}
110
+
111
root, err := layout(dbp.New(chk))
112
if err != nil {
113
return err
114
}
115
116
+ c := root.Cid()
117
+ if dopin {
118
+ n.Pinning.PinWithMode(c, pin.Recursive)
119
+ if err := n.Pinning.Flush(); err != nil {
120
+ return err
121
+ }
122
+ }
123
+
124
return cmds.EmitOnce(res, &BlockStat{
111
- Key: root.Cid().String(),
125
+ Key: c.String(),
126
Size: int(hres.ContentLength),
127
})
128
},
test/sharness/t0272-urlstore.sh
+3
-5
@@ -46,9 +46,8 @@ test_expect_success "enable urlstore" '
46
test_launch_ipfs_daemon --offline
47
48
test_expect_success "add files using gateway address via url store" '
49
- HASH1=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) &&
50
- HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a) &&
51
- ipfs pin add $HASH1 $HASH2
49
+ HASH1=$(ipfs urlstore add --pin=false http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) &&
50
+ HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a)
51
'
52
53
test_expect_success "make sure hashes are different" '
@@ -99,7 +98,6 @@ test_expect_success "gatway no longer has files" '
98
cat <<EOF | sort > verify_expect_2
99
error zb2rhX1q5oFFzEkPNsTe1Y8osUdFqSQGjUWRZsqC9fbY6WVSk 262144 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 0
100
error zb2rhYbKFn1UWGHXaAitcdVTkDGTykX8RFpGWzRFuLpoe9VE4 237856 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 262144
102
-error zb2rhjddJ5DNzBrFu8G6CP1ApY25BukwCeskXHzN1H18CiVVZ 2222 http://127.0.0.1:$GWAY_PORT/ipfs/QmcHm3BL2cXuQ6rJdKQgPrmT9suqGkfy2KzH3MkXPEBXU6 0
101
EOF
102
103
test_expect_success "ipfs filestore verify is correct" '
@@ -113,7 +111,7 @@ test_expect_success "files can not be retrieved via the urlstore" '
111
'
112
113
test_expect_success "remove broken files" '
116
- ipfs pin rm $HASH1 $HASH2 &&
114
+ ipfs pin rm $HASH2 &&
115
ipfs repo gc > /dev/null
116
'
117